Restructuring

Import flask app as well as new makefile and entirely new directory
structure.
This commit is contained in:
Jason A. Donenfeld 2013-04-29 11:05:09 +02:00
parent 9c8beb0cc5
commit d33715066a
53 changed files with 9957 additions and 9541 deletions

1
.gitignore vendored Normal file
View File

@ -0,0 +1 @@
*.swp

8
deployment-config.mk Normal file
View File

@ -0,0 +1,8 @@
WEB_SERVER := metheny.zx2c4.com
WEB_SERVER_URL := http://photos.jasondonenfeld.com
HTDOCS_PATH := /var/www/htdocs/photos.jasondonenfeld.com
HTDOCS_USER := nginx
FLASK_USER := photofloat
FLASK_PATH := /var/www/uwsgi/photofloat

3
scanner/.gitignore vendored
View File

@ -1,4 +1 @@
upload.sh
*.pyc
cache/*
test/*

37
scanner/Makefile Normal file
View File

@ -0,0 +1,37 @@
.PHONY: deploy scan
include ../deployment-config.mk
include floatapp/app.cfg
SSH_OPTS := -q -o ControlMaster=auto -o ControlPath=.ssh-deployment.sock
scan:
@echo " SCAN $(WEB_SERVER)"
@curl "$(WEB_SERVER_URL)/scan?username=$(subst $\",,$(ADMIN_USERNAME))&password=$(subst $\",,$(ADMIN_USERNAME))"
deploy:
@echo " SSH $(WEB_SERVER)"
@ssh $(SSH_OPTS) -Nf $(WEB_SERVER)
@echo " RSYNC . $(WEB_SERVER):$(FLASK_PATH)"
@ssh -t $(SSH_OPTS) $(WEB_SERVER) "sudo -u $(FLASK_USER) -v"
@rsync -aizm --delete-excluded --exclude=.ssh-deployment.sock --exclude=Makefile --exclude="*.pyc" \
--filter="P floatapp/auth.txt" --filter="P albums/" --filter="P cache/" --exclude=.gitignore --exclude="*.swp" \
--rsh="ssh $(SSH_OPTS)" --rsync-path="sudo -n -u $(FLASK_USER) rsync" \
. "$(WEB_SERVER):$(FLASK_PATH)"
@echo " CHMOD 750/640 $(WEB_SERVER):$(FLASK_PATH)/*"
@ssh -t $(SSH_OPTS) $(WEB_SERVER) "sudo find -P '$(FLASK_PATH)' \! -path '$(FLASK_PATH)/albums/*' \! -path '$(FLASK_PATH)/cache/*' -type f -exec chmod 640 {} \;; \
sudo find -P '$(FLASK_PATH)' \! -path '$(FLASK_PATH)/albums/*' \! -path '$(FLASK_PATH)/cache/*' -type d -exec chmod 750 {} \;; \
sudo chmod 750 $(FLASK_PATH)/main.py"
@echo " CHOWN $(FLASK_USER):$(HTDOCS_USER) $(WEB_SERVER):$(FLASK_PATH){,albums,cache}"
@ssh -t $(SSH_OPTS) $(WEB_SERVER) "sudo chown $(FLASK_USER):$(HTDOCS_USER) '$(FLASK_PATH)' '$(FLASK_PATH)/albums' '$(FLASK_PATH)/cache'"
@echo " CHMOD 710 $(WEB_SERVER):$(FLASK_PATH)"
@ssh -t $(SSH_OPTS) $(WEB_SERVER) "sudo chmod 710 '$(FLASK_PATH)'"
@echo " UWSGI restart $(WEB_SERVER)"
@ssh -t $(SSH_OPTS) $(WEB_SERVER) "sudo /etc/init.d/uwsgi restart"
@echo " SSH $(WEB_SERVER)"
@ssh -O exit $(SSH_OPTS) $(WEB_SERVER)

View File

@ -88,12 +88,6 @@ class TreeWalker:
fp = open(os.path.join(self.cache_path, "all_photos.json"), 'w')
json.dump(photo_list, fp, cls=PhotoAlbumEncoder)
fp.close()
photo_list.reverse()
message("caching", "latest photos path list")
fp = open(os.path.join(self.cache_path, "latest_photos.json"), 'w')
json.dump(photo_list[0:27], fp, cls=PhotoAlbumEncoder)
fp.close()
def remove_stale(self):
message("cleanup", "building stale list")
all_cache_entries = { "all_photos.json": True, "latest_photos.json": True }

View File

@ -0,0 +1,10 @@
from flask import Flask
from flask_login import LoginManager
import os.path
app = Flask(__name__)
app.config.from_pyfile(os.path.join(os.path.dirname(os.path.abspath(__file__)), "app.cfg"))
login_manager = LoginManager()
import login
login_manager.setup_app(app)
import endpoints

13
scanner/floatapp/app.cfg Normal file
View File

@ -0,0 +1,13 @@
ADMIN_USERNAME = "misterscanner"
ADMIN_PASSWORD = "ilovescanning"
PHOTO_USERNAME = "photos" # The GUI currently hardcodes 'photos', so don't change this
PHOTO_PASSWORD = "myphotopassword"
ALBUM_PATH = "/var/www/uwsgi/photofloat/albums"
ALBUM_ACCEL = "/internal-albums"
CACHE_PATH = "/var/www/uwsgi/photofloat/cache"
CACHE_ACCEL = "/internal-cache"
SECRET_KEY = "johlahba7shahquoequ7iod0eiGhiephahve0foo2ahshaiko9nahp0Tohch" # Replace this with something big
DEBUG = False

View File

@ -0,0 +1 @@
path/to/some/place

View File

@ -0,0 +1,118 @@
from floatapp import app
from floatapp.login import admin_required, login_required, is_authenticated, query_is_photo_user, query_is_admin_user, photo_user, admin_user
from floatapp.jsonp import jsonp
from process import send_process
from TreeWalker import TreeWalker
from flask import Response, abort, json, request, jsonify
from flask_login import login_user, current_user
from random import shuffle
import os
from mimetypes import guess_type
cwd = os.path.dirname(os.path.abspath(__file__))
@app.route("/scan")
@admin_required
def scan_photos():
global cwd
response = send_process([ "stdbuf", "-oL", os.path.abspath(os.path.join(cwd, "../main.py")),
os.path.abspath(app.config["ALBUM_PATH"]), os.path.abspath(app.config["CACHE_PATH"]) ],
os.path.join(cwd, "scanner.pid"))
response.headers.add("X-Accel-Buffering", "no")
response.cache_control.no_cache = True
return response
@app.route("/auth")
def login():
success = False
if current_user.is_authenticated():
success = True
elif query_is_photo_user(request.form) or query_is_photo_user(request.args):
success = login_user(photo_user, remember=True)
elif query_is_admin_user(request.form) or query_is_admin_user(request.args):
success = login_user(admin_user, remember=True)
if not success:
abort(403)
return ""
def cache_base(path):
path = path.replace('/', '-').replace(' ', '_').replace('(', '').replace('&', '').replace(',', '').replace(')', '').replace('#', '').replace('[', '').replace(']', '').replace('"', '').replace("'", '').replace('_-_', '-').lower()
while path.find("--") != -1:
path = path.replace("--", "-")
while path.find("__") != -1:
path = path.replace("__", "_")
if len(path) == 0:
path = "root"
return path
auth_list = [ ]
def read_auth_list():
global auth_list, cwd
f = open(os.path.join(cwd, "auth.txt"), "r")
paths = [ ]
for path in f:
path = path.strip()
paths.append(path)
paths.append(cache_base(path))
f.close()
auth_list = paths
# TODO: Make this run via inotify
read_auth_list()
def check_permissions(path):
if not is_authenticated():
for auth_path in auth_list:
if path.startswith(auth_path):
abort(403)
@app.route("/albums/<path:path>")
def albums(path):
check_permissions(path)
return accel_redirect(app.config["ALBUM_ACCEL"], app.config["ALBUM_PATH"], path)
@app.route("/cache/<path:path>")
def cache(path):
check_permissions(path)
return accel_redirect(app.config["CACHE_ACCEL"], app.config["CACHE_PATH"], path)
def accel_redirect(internal, real, relative_name):
real_path = os.path.join(real, relative_name)
internal_path = os.path.join(internal, relative_name)
if not os.path.isfile(real_path):
abort(404)
mimetype = None
types = guess_type(real_path)
if len(types) != 0:
mimetype = types[0]
response = Response(mimetype=mimetype)
response.headers.add("X-Accel-Redirect", internal_path)
response.cache_control.public = True
if mimetype == "application/json":
response.cache_control.max_age = 3600
else:
response.cache_control.max_age = 29030400
return response
@app.route("/photos")
@jsonp
def photos():
f = open(os.path.join(app.config["CACHE_PATH"], "all_photos.json"), "r")
photos = json.load(f)
f.close()
if not is_authenticated():
def allowed(photo):
for auth_path in auth_list:
if photo.startswith(auth_path):
return False
return True
photos = [photo for photo in photos if allowed(photo)]
count = int(request.args.get("count", len(photos)))
random = request.args.get("random") == "true"
if random:
shuffle(photos)
else:
photos.reverse()
response = jsonify(photos=photos[0:count])
response.cache_control.no_cache = True
return response

18
scanner/floatapp/jsonp.py Normal file
View File

@ -0,0 +1,18 @@
import json
from functools import wraps
from flask import redirect, request, current_app
import re
jsonp_validator = re.compile("^[a-zA-Z0-9_\-.]{1,128}$")
def jsonp(f):
"""Wraps JSONified output for JSONP"""
@wraps(f)
def decorated_function(*args, **kwargs):
callback = request.args.get('callback', False)
if callback and jsonp_validator.match(callback):
content = str(callback) + '(' + str(f(*args,**kwargs).data) + ')'
return current_app.response_class(content, mimetype='application/javascript')
else:
return f(*args, **kwargs)
return decorated_function

53
scanner/floatapp/login.py Normal file
View File

@ -0,0 +1,53 @@
from floatapp import app, login_manager
from flask import request, abort
from flask_login import current_user, UserMixin
from functools import wraps
class User(UserMixin):
def __init__(self, id, admin=False):
self.admin = admin
self.id = id
photo_user = User("user")
admin_user = User("admin", True)
@login_manager.user_loader
def load_user(id):
if id == "user":
return photo_user
elif id == "admin":
return admin_user
return None
@login_manager.unauthorized_handler
def unauthorized():
return abort(403)
def login_required(fn):
@wraps(fn)
def decorated_view(*args, **kwargs):
if query_is_admin_user(request.args) or query_is_photo_user(request.args) or current_user.is_authenticated():
return fn(*args, **kwargs)
return app.login_manager.unauthorized()
return decorated_view
def admin_required(fn):
@wraps(fn)
def decorated_view(*args, **kwargs):
if query_is_admin_user(request.args) or (current_user.is_authenticated() and current_user.admin):
return fn(*args, **kwargs)
return app.login_manager.unauthorized()
return decorated_view
def query_is_photo_user(query):
username = query.get("username", None)
password = query.get("password", None)
return username == app.config["PHOTO_USERNAME"] and password == app.config["PHOTO_PASSWORD"]
def query_is_admin_user(query):
username = query.get("username", None)
password = query.get("password", None)
return username == app.config["ADMIN_USERNAME"] and password == app.config["ADMIN_PASSWORD"]
def is_authenticated():
return query_is_admin_user(request.args) or query_is_photo_user(request.args) or current_user.is_authenticated()

View File

@ -0,0 +1,52 @@
from flask import Response
import subprocess
import os
import sys
class ProcessWrapper(object):
def __init__(self, process, done):
self.process = process
self.done = done
def close(self):
self.done()
if self.process.returncode is not None:
return
self.process.stdout.close()
self.process.terminate()
self.process.wait()
def __iter__(self):
return self
def __del__(self):
self.close()
def next(self):
try:
data = self.process.stdout.readline()
except:
self.close()
raise StopIteration()
if data:
return data
self.close()
raise StopIteration()
def send_process(args, pid_file):
def setup_proc():
f = open(pid_file, "w")
f.write(str(os.getpid()))
f.close()
os.close(0)
os.dup2(1, 2)
def tear_down_proc():
try:
os.unlink(pid_file)
except:
pass
if os.path.exists(pid_file):
f = open(pid_file, "r")
pid = f.read()
f.close()
if os.path.exists("/proc/%s/status" % pid):
return Response("Scanner is already running.\n", mimetype="text/plain")
process = subprocess.Popen(args, close_fds=True, stdout=subprocess.PIPE, preexec_fn=setup_proc)
response = ProcessWrapper(process, tear_down_proc)
return Response(response, direct_passthrough=True, mimetype="text/plain")

View File

@ -1,8 +1,9 @@
#!/usr/bin/env python
#!/usr/bin/env python2
from TreeWalker import TreeWalker
from CachePath import message
import sys
import os
def main():
reload(sys)
@ -12,6 +13,7 @@ def main():
print "usage: %s ALBUM_PATH CACHE_PATH" % sys.argv[0]
return
try:
os.umask(022)
TreeWalker(sys.argv[1], sys.argv[2])
except KeyboardInterrupt:
message("keyboard", "CTRL+C pressed, quitting.")

1
web/.gitignore vendored
View File

@ -1 +0,0 @@
upload.sh

View File

@ -1,27 +0,0 @@
AddOutputFilterByType DEFLATE text/text text/html text/plain text/xml text/css application/x-javascript application/javascript application/json
<FilesMatch "\.(jpg|otf|ico)$">
Header set Cache-Control "max-age=29030400, public"
</FilesMatch>
<FilesMatch "\.(css|js)$">
Header set Cache-Control "max-age=5184000, public"
</FilesMatch>
<FilesMatch "index.html">
Header set Cache-Control "max-age=2678400, public"
</FilesMatch>
<FilesMatch "\.json$">
Header set Cache-Control "max-age=3600, public"
</FilesMatch>
<FilesMatch "Makefile">
deny from all
</FilesMatch>
RewriteEngine On
RewriteBase /
RewriteRule ^redirect\.php$ - [L]
RewriteCond %{QUERY_STRING} _escaped_fragment_=
RewriteRule . staticrender.php [L]
RewriteCond %{REQUEST_FILENAME} !-f
RewriteCond %{REQUEST_FILENAME} !-d
RewriteRule . /redirect.php [L]

View File

@ -7,36 +7,63 @@ CSS_MIN = $(CSS_DIR)/styles.min.css
JS_MIN_FILES := $(sort $(patsubst %.js, %.min.js, $(filter-out %.min.js, $(wildcard $(JS_DIR)/*.js))))
CSS_MIN_FILES := $(sort $(patsubst %.css, %.min.css, $(filter-out %.min.css, $(wildcard $(CSS_DIR)/*.css))))
JS_COMPILER = utils/google-compiler --warning_level QUIET
CSS_COMPILER = utils/yuicompressor --type css
JS_COMPILER := java -jar bin/closure-compiler.jar --warning_level QUIET
CSS_COMPILER := java -jar bin/yui-compressor.jar --type css
.PHONY: all clean
DEBUG ?= 0
all: $(JS_MIN) $(CSS_MIN) utils/ServerExecute.class
.PHONY: all deploy clean
all: $(JS_MIN) $(CSS_MIN)
ifeq ($(DEBUG),0)
%.min.js: %.js
@echo "Compiling javascript" $<
@echo " JS " $@
@$(JS_COMPILER) --js $< --js_output_file $@
else
%.min.js: %.js
@echo " JS " $@
@cat $< > $@
endif
%.min.css: %.css
@echo "Compiling stylesheet" $<
@echo " CSS " $@
@$(CSS_COMPILER) -o $@ $<
$(JS_MIN): $(JS_MIN_FILES)
@echo "Assembling compiled javascripts"
@echo " CAT " $@
@cat $^ > $@
$(CSS_MIN): $(CSS_MIN_FILES)
@echo "Assembling compiled stylesheets"
@echo " CAT " $@
@cat $^ > $@
empty :=
space := $(empty) $(empty)
classpath := $(subst $(space),:,$(wildcard utils/htmlunit-2.8/*.jar))
utils/ServerExecute.class: utils/ServerExecute.java
@echo "Building HtmlUnit wrapper."
@javac -classpath $(classpath) -d utils $^
clean:
@echo " RM " $(JS_MIN) $(JS_MIN_FILES) $(CSS_MIN) $(CSS_MIN_FILES)
@rm -fv $(JS_MIN) $(JS_MIN_FILES) $(CSS_MIN) $(CSS_MIN_FILES)
include ../deployment-config.mk
SSH_OPTS := -q -o ControlMaster=auto -o ControlPath=.ssh-deployment.sock
deploy: all
@echo " SSH $(WEB_SERVER)"
@ssh $(SSH_OPTS) -Nf $(WEB_SERVER)
@echo " RSYNC . $(WEB_SERVER):$(HTDOCS_PATH)"
@ssh -t $(SSH_OPTS) $(WEB_SERVER) "sudo -u $(HTDOCS_USER) -v"
@rsync -aizm --delete-excluded --exclude=.ssh-deployment.sock --exclude=Makefile --exclude=*.swp \
--exclude=bin/ --include=scripts.min.js --include=styles.min.css \
--exclude=*.js --exclude=*.css --rsh="ssh $(SSH_OPTS)" \
--rsync-path="sudo -n -u $(HTDOCS_USER) rsync" \
. "$(WEB_SERVER):$(HTDOCS_PATH)"
@echo " CHOWN $(HTDOCS_USER):$(HTDOCS_USER) $(WEB_SERVER):$(HTDOCS_PATH)"
@ssh -t $(SSH_OPTS) $(WEB_SERVER) "sudo chown -R $(HTDOCS_USER):$(HTDOCS_USER) '$(HTDOCS_PATH)'"
@echo " CHMOD 750/640 $(WEB_SERVER):$(HTDOCS_PATH)"
@ssh -t $(SSH_OPTS) $(WEB_SERVER) "sudo find '$(HTDOCS_PATH)' -type f -exec chmod 640 {} \;; \
sudo find '$(HTDOCS_PATH)' -type d -exec chmod 750 {} \;;"
@echo " SSH $(WEB_SERVER)"
@ssh -O exit $(SSH_OPTS) $(WEB_SERVER)

Binary file not shown.

View File

@ -1,3 +0,0 @@
<FilesMatch "(?<!min)\.css">
deny from all
</FilesMatch>

View File

@ -5,7 +5,7 @@
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="fragment" content="!" />
<meta name="medium" content="image" />
<title>PhotoFloat</title>
<title>Photos</title>
<link href="css/styles.min.css" rel="stylesheet" type="text/css" />
<script type="text/javascript" src="js/scripts.min.js"></script>
</head>

View File

@ -1,3 +0,0 @@
<FilesMatch "(?<!min)\.js">
deny from all
</FilesMatch>

File diff suppressed because it is too large Load Diff

9597
web/js/000-jquery-1.9.1.js Normal file

File diff suppressed because it is too large Load Diff

View File

@ -297,7 +297,7 @@
// vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
// vvvvvvvvvvvvvvvvvvv REMOVE IF NOT SUPPORTING IE6/7/8 vvvvvvvvvvvvvvvvvvv
// vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
$.browser.msie && !supports_onhashchange && (function(){
(navigator.appName == 'Microsoft Internet Explorer') && !supports_onhashchange && (function(){
// Not only do IE6/7 need the "magical" Iframe treatment, but so does IE8
// when running in "IE7 compatibility" mode.

View File

@ -92,7 +92,7 @@
$.ajax({
type: "GET",
dataType: "text",
url: "auth?password=" + password,
url: "auth?username=photos&password=" + password,
success: function() {
result(true);
},

View File

@ -1,6 +1,6 @@
$(window).load(function () {
window._gaq = window._gaq || [];
window._gaq.push(['_setAccount', 'UA-XXXXXX-X']);
window._gaq.push(['_setAccount', 'UA-XXXXXX-XXX']);
var ga = document.createElement('script');
ga.type = 'text/javascript';
ga.async = true;

View File

@ -1,38 +0,0 @@
<?php
function cachePath($path) {
if ($path[0] == '/')
$path = substr($path, 1);
if ($path[strlen($path) - 1] == '/')
$path = substr($path, 0, strlen($path) - 1);
$path = str_replace('_-_', '-', str_replace('/', '-', str_replace(' ', '_', str_replace('(', '', str_replace(')', '', str_replace('#', '', str_replace('[', '', str_replace(']', '', str_replace('&', '', str_replace(',', '', str_replace('"', '', str_replace("'", '', strtolower($path)))))))))))));
while (strpos($path, "--") !== false)
$path = str_replace("--", "-", $path);
while (strpos($path, "__") !== false)
$path = str_replace("__", "_", $path);
if (strlen(path) == 0)
$path = "root";
return $path;
}
$url = str_replace("\b", "", str_replace("\r", "", str_replace("\n", "", $_SERVER["SCRIPT_URL"])));
if ($url[strlen($url) - 1] == '/')
$url = substr($url, 0, strlen($url) - 1);
if (strpos(strtolower($url), ".php") == strlen($url) - 4) {
$url = substr($url, 0, strlen($url) - 4);
$index = strrpos($url, "/");
$redirect = "/#!/".cachePath(substr($url, 0, $index))."/".cachePath(substr($url, $index));
} else if (strpos(strtolower($url), ".jpg") == strlen($url) - 4) {
$index = strrpos($url, "/");
$redirect = "/#!/".cachePath(substr($url, 0, $index))."/".cachePath(substr($url, $index));
} else if (strpos($url, "/cache/") === 0 || strpos($url, "/albums/") === 0 || strpos($url, "/img/") === 0 || strpos($url, "/img/") === 0 || strpos($url, "/js/") === 0 || strpos($url, "/css/") === 0) {
header("HTTP/1.1 404 Not Found");
exit();
} else
$redirect = "/#!/".cachePath($url);
header("HTTP/1.1 301 Moved Permanently");
header("Location: $redirect");
?>

View File

@ -1,6 +0,0 @@
<?php
if ($_ENV["SCRIPT_URL"] == $_ENV["SCRIPT_NAME"])
die("Infinite loop.");
putenv('LANG=en_US.UTF-8');
passthru("utils/serverexecute ".escapeshellarg($_ENV["SCRIPT_URI"]."#!".$_GET["_escaped_fragment_"]));
?>

View File

@ -1 +0,0 @@
*.class

View File

@ -1 +0,0 @@
deny from all

View File

@ -1,19 +0,0 @@
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
public class ServerExecute {
public static void main(String[] args) {
if (args.length != 1) {
System.err.println("You must give a url as an argument.");
return;
}
try {
final WebClient webClient = new WebClient();
HtmlPage page = webClient.getPage(args[0]);
webClient.waitForBackgroundJavaScript(2000);
System.out.println(page.asXml());
} catch (Exception e) {
e.printStackTrace();
}
}
}

View File

@ -1,3 +0,0 @@
#!/bin/sh
me=$(dirname "$0")
java -jar "$me/google-compiler.jar" $@

Binary file not shown.

Binary file not shown.

View File

@ -1,3 +0,0 @@
#!/bin/sh
cd $(dirname $0)
java -Xmx128m -classpath $(for i in htmlunit-2.8/*; do echo $i; done|tr '\n' ':') ServerExecute $@

View File

@ -1,3 +0,0 @@
#!/bin/sh
me=$(dirname "$0")
java -jar "$me/yuicompressor-2.4.6.jar" $@