Restructuring

Import flask app as well as new makefile and entirely new directory
structure.

scanner/.gitignore

@@ -1,4 +1 @@
-upload.sh
 *.pyc
-cache/*
-test/*

scanner/Makefile

@@ -0,0 +1,37 @@
+.PHONY: deploy scan
+
+include ../deployment-config.mk
+include floatapp/app.cfg
+
+SSH_OPTS := -q -o ControlMaster=auto -o ControlPath=.ssh-deployment.sock
+
+scan:
+	@echo "    SCAN    $(WEB_SERVER)"
+	@curl "$(WEB_SERVER_URL)/scan?username=$(subst $\",,$(ADMIN_USERNAME))&password=$(subst $\",,$(ADMIN_USERNAME))"
+
+deploy:
+	@echo "    SSH     $(WEB_SERVER)"
+	@ssh $(SSH_OPTS) -Nf $(WEB_SERVER)
+	
+	@echo "    RSYNC   . $(WEB_SERVER):$(FLASK_PATH)"
+	@ssh -t $(SSH_OPTS) $(WEB_SERVER) "sudo -u $(FLASK_USER) -v"
+	@rsync -aizm --delete-excluded --exclude=.ssh-deployment.sock --exclude=Makefile --exclude="*.pyc" \
+		--filter="P floatapp/auth.txt" --filter="P albums/" --filter="P cache/" --exclude=.gitignore --exclude="*.swp" \
+		--rsh="ssh $(SSH_OPTS)" --rsync-path="sudo -n -u $(FLASK_USER) rsync" \
+		. "$(WEB_SERVER):$(FLASK_PATH)" 
+	
+	@echo "    CHMOD   750/640 $(WEB_SERVER):$(FLASK_PATH)/*"
+	@ssh -t $(SSH_OPTS) $(WEB_SERVER) "sudo find -P '$(FLASK_PATH)' \! -path '$(FLASK_PATH)/albums/*' \! -path '$(FLASK_PATH)/cache/*' -type f -exec chmod 640 {} \;; \
+					sudo find -P '$(FLASK_PATH)' \! -path '$(FLASK_PATH)/albums/*' \! -path '$(FLASK_PATH)/cache/*' -type d -exec chmod 750 {} \;; \
+					sudo chmod 750 $(FLASK_PATH)/main.py"
+	
+	@echo "    CHOWN   $(FLASK_USER):$(HTDOCS_USER) $(WEB_SERVER):$(FLASK_PATH){,albums,cache}"
+	@ssh -t $(SSH_OPTS) $(WEB_SERVER) "sudo chown $(FLASK_USER):$(HTDOCS_USER) '$(FLASK_PATH)' '$(FLASK_PATH)/albums' '$(FLASK_PATH)/cache'"
+	@echo "    CHMOD   710 $(WEB_SERVER):$(FLASK_PATH)"
+	@ssh -t $(SSH_OPTS) $(WEB_SERVER) "sudo chmod 710 '$(FLASK_PATH)'"
+	
+	@echo "    UWSGI   restart $(WEB_SERVER)"
+	@ssh -t $(SSH_OPTS) $(WEB_SERVER) "sudo /etc/init.d/uwsgi restart"
+	
+	@echo "    SSH     $(WEB_SERVER)"
+	@ssh -O exit $(SSH_OPTS) $(WEB_SERVER)

scanner/TreeWalker.py

@@ -88,12 +88,6 @@ class TreeWalker:
 		fp = open(os.path.join(self.cache_path, "all_photos.json"), 'w')
 		json.dump(photo_list, fp, cls=PhotoAlbumEncoder)
 		fp.close()
-		photo_list.reverse()
-		message("caching", "latest photos path list")
-		fp = open(os.path.join(self.cache_path, "latest_photos.json"), 'w')
-		json.dump(photo_list[0:27], fp, cls=PhotoAlbumEncoder)
-		fp.close()
-		
 	def remove_stale(self):
 		message("cleanup", "building stale list")
 		all_cache_entries = { "all_photos.json": True, "latest_photos.json": True }

scanner/floatapp/__init__.py

@@ -0,0 +1,10 @@
+from flask import Flask
+from flask_login import LoginManager
+import os.path
+
+app = Flask(__name__)
+app.config.from_pyfile(os.path.join(os.path.dirname(os.path.abspath(__file__)), "app.cfg"))
+login_manager = LoginManager()
+import login
+login_manager.setup_app(app)
+import endpoints

scanner/floatapp/app.cfg

@@ -0,0 +1,13 @@
+ADMIN_USERNAME = "misterscanner"
+ADMIN_PASSWORD = "ilovescanning"
+
+PHOTO_USERNAME = "photos" # The GUI currently hardcodes 'photos', so don't change this
+PHOTO_PASSWORD = "myphotopassword"
+
+ALBUM_PATH = "/var/www/uwsgi/photofloat/albums"
+ALBUM_ACCEL = "/internal-albums"
+CACHE_PATH = "/var/www/uwsgi/photofloat/cache"
+CACHE_ACCEL = "/internal-cache"
+
+SECRET_KEY = "johlahba7shahquoequ7iod0eiGhiephahve0foo2ahshaiko9nahp0Tohch" # Replace this with something big
+DEBUG = False

scanner/floatapp/auth.txt

@@ -0,0 +1 @@
+path/to/some/place

scanner/floatapp/endpoints.py

@@ -0,0 +1,118 @@
+from floatapp import app
+from floatapp.login import admin_required, login_required, is_authenticated, query_is_photo_user, query_is_admin_user, photo_user, admin_user
+from floatapp.jsonp import jsonp
+from process import send_process
+from TreeWalker import TreeWalker
+from flask import Response, abort, json, request, jsonify
+from flask_login import login_user, current_user
+from random import shuffle
+import os
+from mimetypes import guess_type
+
+cwd = os.path.dirname(os.path.abspath(__file__))
+
+@app.route("/scan")
+@admin_required
+def scan_photos():
+	global cwd
+	response = send_process([ "stdbuf", "-oL", os.path.abspath(os.path.join(cwd, "../main.py")),
+				os.path.abspath(app.config["ALBUM_PATH"]), os.path.abspath(app.config["CACHE_PATH"]) ],
+				os.path.join(cwd, "scanner.pid"))
+	response.headers.add("X-Accel-Buffering", "no")
+	response.cache_control.no_cache = True
+	return response
+
+@app.route("/auth")
+def login():
+	success = False
+	if current_user.is_authenticated():
+		success = True
+	elif query_is_photo_user(request.form) or query_is_photo_user(request.args):
+		success = login_user(photo_user, remember=True)
+	elif query_is_admin_user(request.form) or query_is_admin_user(request.args):
+		success = login_user(admin_user, remember=True)
+	if not success:
+		abort(403)
+	return ""
+
+def cache_base(path):
+	path = path.replace('/', '-').replace(' ', '_').replace('(', '').replace('&', '').replace(',', '').replace(')', '').replace('#', '').replace('[', '').replace(']', '').replace('"', '').replace("'", '').replace('_-_', '-').lower()
+	while path.find("--") != -1:
+		path = path.replace("--", "-")
+	while path.find("__") != -1:
+		path = path.replace("__", "_")
+	if len(path) == 0:
+		path = "root"
+	return path
+
+auth_list = [ ]
+def read_auth_list():
+	global auth_list, cwd
+	f = open(os.path.join(cwd, "auth.txt"), "r")
+	paths = [ ]
+	for path in f:
+		path = path.strip()
+		paths.append(path)
+		paths.append(cache_base(path))
+	f.close()
+	auth_list = paths
+
+# TODO: Make this run via inotify
+read_auth_list()
+
+def check_permissions(path):
+	if not is_authenticated():
+		for auth_path in auth_list:
+			if path.startswith(auth_path):
+				abort(403)
+
+@app.route("/albums/<path:path>")
+def albums(path):
+	check_permissions(path)
+	return accel_redirect(app.config["ALBUM_ACCEL"], app.config["ALBUM_PATH"], path)
+
+@app.route("/cache/<path:path>")
+def cache(path):
+	check_permissions(path)
+	return accel_redirect(app.config["CACHE_ACCEL"], app.config["CACHE_PATH"], path)
+
+def accel_redirect(internal, real, relative_name):
+	real_path = os.path.join(real, relative_name)
+	internal_path = os.path.join(internal, relative_name)
+	if not os.path.isfile(real_path):
+		abort(404)
+	mimetype = None
+	types = guess_type(real_path)
+	if len(types) != 0:
+		mimetype = types[0]
+	response = Response(mimetype=mimetype)
+	response.headers.add("X-Accel-Redirect", internal_path)
+	response.cache_control.public = True
+	if mimetype == "application/json":
+		response.cache_control.max_age = 3600
+	else:
+		response.cache_control.max_age = 29030400
+	return response
+
+@app.route("/photos")
+@jsonp
+def photos():
+	f = open(os.path.join(app.config["CACHE_PATH"], "all_photos.json"), "r")
+	photos = json.load(f)
+	f.close()
+	if not is_authenticated():
+		def allowed(photo):
+			for auth_path in auth_list:
+				if photo.startswith(auth_path):
+					return False
+			return True
+		photos = [photo for photo in photos if allowed(photo)]
+	count = int(request.args.get("count", len(photos)))
+	random = request.args.get("random") == "true"
+	if random:
+		shuffle(photos)
+	else:
+		photos.reverse()
+	response = jsonify(photos=photos[0:count])
+	response.cache_control.no_cache = True
+	return response

scanner/floatapp/jsonp.py

@@ -0,0 +1,18 @@
+import json
+from functools import wraps
+from flask import redirect, request, current_app
+import re
+
+jsonp_validator = re.compile("^[a-zA-Z0-9_\-.]{1,128}$")
+
+def jsonp(f):
+	"""Wraps JSONified output for JSONP"""
+	@wraps(f)
+	def decorated_function(*args, **kwargs):
+		callback = request.args.get('callback', False)
+		if callback and jsonp_validator.match(callback):
+			content = str(callback) + '(' + str(f(*args,**kwargs).data) + ')'
+			return current_app.response_class(content, mimetype='application/javascript')
+		else:
+			return f(*args, **kwargs)
+	return decorated_function

scanner/floatapp/login.py

@@ -0,0 +1,53 @@
+from floatapp import app, login_manager
+from flask import request, abort
+from flask_login import current_user, UserMixin
+from functools import wraps
+
+class User(UserMixin):
+	def __init__(self, id, admin=False):
+		self.admin = admin
+		self.id = id
+
+photo_user = User("user")
+admin_user = User("admin", True)
+
+@login_manager.user_loader
+def load_user(id):
+	if id == "user":
+		return photo_user
+	elif id == "admin":
+		return admin_user
+	return None
+
+@login_manager.unauthorized_handler
+def unauthorized():
+	return abort(403)
+
+def login_required(fn):
+	@wraps(fn)
+	def decorated_view(*args, **kwargs):
+		if query_is_admin_user(request.args) or query_is_photo_user(request.args) or current_user.is_authenticated():
+			return fn(*args, **kwargs)
+		return app.login_manager.unauthorized()
+	return decorated_view
+
+def admin_required(fn):
+	@wraps(fn)
+	def decorated_view(*args, **kwargs):
+		if query_is_admin_user(request.args) or (current_user.is_authenticated() and current_user.admin):
+			return fn(*args, **kwargs)
+		return app.login_manager.unauthorized()
+	return decorated_view
+
+def query_is_photo_user(query):
+	username = query.get("username", None)
+	password = query.get("password", None)
+	return username == app.config["PHOTO_USERNAME"] and password == app.config["PHOTO_PASSWORD"]
+
+def query_is_admin_user(query):
+	username = query.get("username", None)
+	password = query.get("password", None)
+	return username == app.config["ADMIN_USERNAME"] and password == app.config["ADMIN_PASSWORD"]
+
+def is_authenticated():
+	return query_is_admin_user(request.args) or query_is_photo_user(request.args) or current_user.is_authenticated()

scanner/floatapp/process.py

@@ -0,0 +1,52 @@
+from flask import Response
+import subprocess
+import os
+import sys
+
+class ProcessWrapper(object):
+	def __init__(self, process, done):
+		self.process = process
+		self.done = done
+	def close(self):
+		self.done()
+		if self.process.returncode is not None:
+			return
+		self.process.stdout.close()
+		self.process.terminate()
+		self.process.wait()
+	def __iter__(self):
+		return self
+	def __del__(self):
+		self.close()
+	def next(self):
+		try:
+			data = self.process.stdout.readline()
+		except:
+			self.close()
+			raise StopIteration()
+		if data:
+			return data
+		self.close()
+		raise StopIteration()
+
+def send_process(args, pid_file):
+	def setup_proc():
+		f = open(pid_file, "w")
+		f.write(str(os.getpid()))
+		f.close()
+		os.close(0)
+		os.dup2(1, 2)
+	def tear_down_proc():
+		try:
+			os.unlink(pid_file)
+		except:
+			pass
+	if os.path.exists(pid_file):
+		f = open(pid_file, "r")
+		pid = f.read()
+		f.close()
+		if os.path.exists("/proc/%s/status" % pid):
+			return Response("Scanner is already running.\n", mimetype="text/plain")
+	process = subprocess.Popen(args, close_fds=True, stdout=subprocess.PIPE, preexec_fn=setup_proc)
+	response = ProcessWrapper(process, tear_down_proc)
+	return Response(response, direct_passthrough=True, mimetype="text/plain")

scanner/main.py

@@ -1,8 +1,9 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
 
 from TreeWalker import TreeWalker
 from CachePath import message
 import sys
+import os
 
 def main():
 	reload(sys)
@@ -12,6 +13,7 @@ def main():
 		print "usage: %s ALBUM_PATH CACHE_PATH" % sys.argv[0]
 		return
 	try:
+		os.umask(022)
 		TreeWalker(sys.argv[1], sys.argv[2])
 	except KeyboardInterrupt:
 		message("keyboard", "CTRL+C pressed, quitting.")