Restructuring

Import flask app as well as new makefile and entirely new directory
structure.
This commit is contained in:
Jason A. Donenfeld
2013-04-29 11:05:09 +02:00
parent 9c8beb0cc5
commit d33715066a
53 changed files with 9957 additions and 9541 deletions
+1
View File
@@ -0,0 +1 @@
*.swp
+8
View File
@@ -0,0 +1,8 @@
WEB_SERVER := metheny.zx2c4.com
WEB_SERVER_URL := http://photos.jasondonenfeld.com
HTDOCS_PATH := /var/www/htdocs/photos.jasondonenfeld.com
HTDOCS_USER := nginx
FLASK_USER := photofloat
FLASK_PATH := /var/www/uwsgi/photofloat
-3
View File
@@ -1,4 +1 @@
upload.sh
*.pyc
cache/*
test/*
+37
View File
@@ -0,0 +1,37 @@
.PHONY: deploy scan
include ../deployment-config.mk
include floatapp/app.cfg
SSH_OPTS := -q -o ControlMaster=auto -o ControlPath=.ssh-deployment.sock
scan:
@echo " SCAN $(WEB_SERVER)"
@curl "$(WEB_SERVER_URL)/scan?username=$(subst $\",,$(ADMIN_USERNAME))&password=$(subst $\",,$(ADMIN_USERNAME))"
deploy:
@echo " SSH $(WEB_SERVER)"
@ssh $(SSH_OPTS) -Nf $(WEB_SERVER)
@echo " RSYNC . $(WEB_SERVER):$(FLASK_PATH)"
@ssh -t $(SSH_OPTS) $(WEB_SERVER) "sudo -u $(FLASK_USER) -v"
@rsync -aizm --delete-excluded --exclude=.ssh-deployment.sock --exclude=Makefile --exclude="*.pyc" \
--filter="P floatapp/auth.txt" --filter="P albums/" --filter="P cache/" --exclude=.gitignore --exclude="*.swp" \
--rsh="ssh $(SSH_OPTS)" --rsync-path="sudo -n -u $(FLASK_USER) rsync" \
. "$(WEB_SERVER):$(FLASK_PATH)"
@echo " CHMOD 750/640 $(WEB_SERVER):$(FLASK_PATH)/*"
@ssh -t $(SSH_OPTS) $(WEB_SERVER) "sudo find -P '$(FLASK_PATH)' \! -path '$(FLASK_PATH)/albums/*' \! -path '$(FLASK_PATH)/cache/*' -type f -exec chmod 640 {} \;; \
sudo find -P '$(FLASK_PATH)' \! -path '$(FLASK_PATH)/albums/*' \! -path '$(FLASK_PATH)/cache/*' -type d -exec chmod 750 {} \;; \
sudo chmod 750 $(FLASK_PATH)/main.py"
@echo " CHOWN $(FLASK_USER):$(HTDOCS_USER) $(WEB_SERVER):$(FLASK_PATH){,albums,cache}"
@ssh -t $(SSH_OPTS) $(WEB_SERVER) "sudo chown $(FLASK_USER):$(HTDOCS_USER) '$(FLASK_PATH)' '$(FLASK_PATH)/albums' '$(FLASK_PATH)/cache'"
@echo " CHMOD 710 $(WEB_SERVER):$(FLASK_PATH)"
@ssh -t $(SSH_OPTS) $(WEB_SERVER) "sudo chmod 710 '$(FLASK_PATH)'"
@echo " UWSGI restart $(WEB_SERVER)"
@ssh -t $(SSH_OPTS) $(WEB_SERVER) "sudo /etc/init.d/uwsgi restart"
@echo " SSH $(WEB_SERVER)"
@ssh -O exit $(SSH_OPTS) $(WEB_SERVER)
-6
View File
@@ -88,12 +88,6 @@ class TreeWalker:
fp = open(os.path.join(self.cache_path, "all_photos.json"), 'w')
json.dump(photo_list, fp, cls=PhotoAlbumEncoder)
fp.close()
photo_list.reverse()
message("caching", "latest photos path list")
fp = open(os.path.join(self.cache_path, "latest_photos.json"), 'w')
json.dump(photo_list[0:27], fp, cls=PhotoAlbumEncoder)
fp.close()
def remove_stale(self):
message("cleanup", "building stale list")
all_cache_entries = { "all_photos.json": True, "latest_photos.json": True }
+10
View File
@@ -0,0 +1,10 @@
from flask import Flask
from flask_login import LoginManager
import os.path
app = Flask(__name__)
app.config.from_pyfile(os.path.join(os.path.dirname(os.path.abspath(__file__)), "app.cfg"))
login_manager = LoginManager()
import login
login_manager.setup_app(app)
import endpoints
+13
View File
@@ -0,0 +1,13 @@
ADMIN_USERNAME = "misterscanner"
ADMIN_PASSWORD = "ilovescanning"
PHOTO_USERNAME = "photos" # The GUI currently hardcodes 'photos', so don't change this
PHOTO_PASSWORD = "myphotopassword"
ALBUM_PATH = "/var/www/uwsgi/photofloat/albums"
ALBUM_ACCEL = "/internal-albums"
CACHE_PATH = "/var/www/uwsgi/photofloat/cache"
CACHE_ACCEL = "/internal-cache"
SECRET_KEY = "johlahba7shahquoequ7iod0eiGhiephahve0foo2ahshaiko9nahp0Tohch" # Replace this with something big
DEBUG = False
+1
View File
@@ -0,0 +1 @@
path/to/some/place
+118
View File
@@ -0,0 +1,118 @@
from floatapp import app
from floatapp.login import admin_required, login_required, is_authenticated, query_is_photo_user, query_is_admin_user, photo_user, admin_user
from floatapp.jsonp import jsonp
from process import send_process
from TreeWalker import TreeWalker
from flask import Response, abort, json, request, jsonify
from flask_login import login_user, current_user
from random import shuffle
import os
from mimetypes import guess_type
cwd = os.path.dirname(os.path.abspath(__file__))
@app.route("/scan")
@admin_required
def scan_photos():
global cwd
response = send_process([ "stdbuf", "-oL", os.path.abspath(os.path.join(cwd, "../main.py")),
os.path.abspath(app.config["ALBUM_PATH"]), os.path.abspath(app.config["CACHE_PATH"]) ],
os.path.join(cwd, "scanner.pid"))
response.headers.add("X-Accel-Buffering", "no")
response.cache_control.no_cache = True
return response
@app.route("/auth")
def login():
success = False
if current_user.is_authenticated():
success = True
elif query_is_photo_user(request.form) or query_is_photo_user(request.args):
success = login_user(photo_user, remember=True)
elif query_is_admin_user(request.form) or query_is_admin_user(request.args):
success = login_user(admin_user, remember=True)
if not success:
abort(403)
return ""
def cache_base(path):
path = path.replace('/', '-').replace(' ', '_').replace('(', '').replace('&', '').replace(',', '').replace(')', '').replace('#', '').replace('[', '').replace(']', '').replace('"', '').replace("'", '').replace('_-_', '-').lower()
while path.find("--") != -1:
path = path.replace("--", "-")
while path.find("__") != -1:
path = path.replace("__", "_")
if len(path) == 0:
path = "root"
return path
auth_list = [ ]
def read_auth_list():
global auth_list, cwd
f = open(os.path.join(cwd, "auth.txt"), "r")
paths = [ ]
for path in f:
path = path.strip()
paths.append(path)
paths.append(cache_base(path))
f.close()
auth_list = paths
# TODO: Make this run via inotify
read_auth_list()
def check_permissions(path):
if not is_authenticated():
for auth_path in auth_list:
if path.startswith(auth_path):
abort(403)
@app.route("/albums/<path:path>")
def albums(path):
check_permissions(path)
return accel_redirect(app.config["ALBUM_ACCEL"], app.config["ALBUM_PATH"], path)
@app.route("/cache/<path:path>")
def cache(path):
check_permissions(path)
return accel_redirect(app.config["CACHE_ACCEL"], app.config["CACHE_PATH"], path)
def accel_redirect(internal, real, relative_name):
real_path = os.path.join(real, relative_name)
internal_path = os.path.join(internal, relative_name)
if not os.path.isfile(real_path):
abort(404)
mimetype = None
types = guess_type(real_path)
if len(types) != 0:
mimetype = types[0]
response = Response(mimetype=mimetype)
response.headers.add("X-Accel-Redirect", internal_path)
response.cache_control.public = True
if mimetype == "application/json":
response.cache_control.max_age = 3600
else:
response.cache_control.max_age = 29030400
return response
@app.route("/photos")
@jsonp
def photos():
f = open(os.path.join(app.config["CACHE_PATH"], "all_photos.json"), "r")
photos = json.load(f)
f.close()
if not is_authenticated():
def allowed(photo):
for auth_path in auth_list:
if photo.startswith(auth_path):
return False
return True
photos = [photo for photo in photos if allowed(photo)]
count = int(request.args.get("count", len(photos)))
random = request.args.get("random") == "true"
if random:
shuffle(photos)
else:
photos.reverse()
response = jsonify(photos=photos[0:count])
response.cache_control.no_cache = True
return response
+18
View File
@@ -0,0 +1,18 @@
import json
from functools import wraps
from flask import redirect, request, current_app
import re
jsonp_validator = re.compile("^[a-zA-Z0-9_\-.]{1,128}$")
def jsonp(f):
"""Wraps JSONified output for JSONP"""
@wraps(f)
def decorated_function(*args, **kwargs):
callback = request.args.get('callback', False)
if callback and jsonp_validator.match(callback):
content = str(callback) + '(' + str(f(*args,**kwargs).data) + ')'
return current_app.response_class(content, mimetype='application/javascript')
else:
return f(*args, **kwargs)
return decorated_function
+53
View File
@@ -0,0 +1,53 @@
from floatapp import app, login_manager
from flask import request, abort
from flask_login import current_user, UserMixin
from functools import wraps
class User(UserMixin):
def __init__(self, id, admin=False):
self.admin = admin
self.id = id
photo_user = User("user")
admin_user = User("admin", True)
@login_manager.user_loader
def load_user(id):
if id == "user":
return photo_user
elif id == "admin":
return admin_user
return None
@login_manager.unauthorized_handler
def unauthorized():
return abort(403)
def login_required(fn):
@wraps(fn)
def decorated_view(*args, **kwargs):
if query_is_admin_user(request.args) or query_is_photo_user(request.args) or current_user.is_authenticated():
return fn(*args, **kwargs)
return app.login_manager.unauthorized()
return decorated_view
def admin_required(fn):
@wraps(fn)
def decorated_view(*args, **kwargs):
if query_is_admin_user(request.args) or (current_user.is_authenticated() and current_user.admin):
return fn(*args, **kwargs)
return app.login_manager.unauthorized()
return decorated_view
def query_is_photo_user(query):
username = query.get("username", None)
password = query.get("password", None)
return username == app.config["PHOTO_USERNAME"] and password == app.config["PHOTO_PASSWORD"]
def query_is_admin_user(query):
username = query.get("username", None)
password = query.get("password", None)
return username == app.config["ADMIN_USERNAME"] and password == app.config["ADMIN_PASSWORD"]
def is_authenticated():
return query_is_admin_user(request.args) or query_is_photo_user(request.args) or current_user.is_authenticated()
+52
View File
@@ -0,0 +1,52 @@
from flask import Response
import subprocess
import os
import sys
class ProcessWrapper(object):
def __init__(self, process, done):
self.process = process
self.done = done
def close(self):
self.done()
if self.process.returncode is not None:
return
self.process.stdout.close()
self.process.terminate()
self.process.wait()
def __iter__(self):
return self
def __del__(self):
self.close()
def next(self):
try:
data = self.process.stdout.readline()
except:
self.close()
raise StopIteration()
if data:
return data
self.close()
raise StopIteration()
def send_process(args, pid_file):
def setup_proc():
f = open(pid_file, "w")
f.write(str(os.getpid()))
f.close()
os.close(0)
os.dup2(1, 2)
def tear_down_proc():
try:
os.unlink(pid_file)
except:
pass
if os.path.exists(pid_file):
f = open(pid_file, "r")
pid = f.read()
f.close()
if os.path.exists("/proc/%s/status" % pid):
return Response("Scanner is already running.\n", mimetype="text/plain")
process = subprocess.Popen(args, close_fds=True, stdout=subprocess.PIPE, preexec_fn=setup_proc)
response = ProcessWrapper(process, tear_down_proc)
return Response(response, direct_passthrough=True, mimetype="text/plain")
+3 -1
View File
@@ -1,8 +1,9 @@
#!/usr/bin/env python
#!/usr/bin/env python2
from TreeWalker import TreeWalker
from CachePath import message
import sys
import os
def main():
reload(sys)
@@ -12,6 +13,7 @@ def main():
print "usage: %s ALBUM_PATH CACHE_PATH" % sys.argv[0]
return
try:
os.umask(022)
TreeWalker(sys.argv[1], sys.argv[2])
except KeyboardInterrupt:
message("keyboard", "CTRL+C pressed, quitting.")
-1
View File
@@ -1 +0,0 @@
upload.sh
-27
View File
@@ -1,27 +0,0 @@
AddOutputFilterByType DEFLATE text/text text/html text/plain text/xml text/css application/x-javascript application/javascript application/json
<FilesMatch "\.(jpg|otf|ico)$">
Header set Cache-Control "max-age=29030400, public"
</FilesMatch>
<FilesMatch "\.(css|js)$">
Header set Cache-Control "max-age=5184000, public"
</FilesMatch>
<FilesMatch "index.html">
Header set Cache-Control "max-age=2678400, public"
</FilesMatch>
<FilesMatch "\.json$">
Header set Cache-Control "max-age=3600, public"
</FilesMatch>
<FilesMatch "Makefile">
deny from all
</FilesMatch>
RewriteEngine On
RewriteBase /
RewriteRule ^redirect\.php$ - [L]
RewriteCond %{QUERY_STRING} _escaped_fragment_=
RewriteRule . staticrender.php [L]
RewriteCond %{REQUEST_FILENAME} !-f
RewriteCond %{REQUEST_FILENAME} !-d
RewriteRule . /redirect.php [L]
+42 -15
View File
@@ -7,36 +7,63 @@ CSS_MIN = $(CSS_DIR)/styles.min.css
JS_MIN_FILES := $(sort $(patsubst %.js, %.min.js, $(filter-out %.min.js, $(wildcard $(JS_DIR)/*.js))))
CSS_MIN_FILES := $(sort $(patsubst %.css, %.min.css, $(filter-out %.min.css, $(wildcard $(CSS_DIR)/*.css))))
JS_COMPILER = utils/google-compiler --warning_level QUIET
CSS_COMPILER = utils/yuicompressor --type css
JS_COMPILER := java -jar bin/closure-compiler.jar --warning_level QUIET
CSS_COMPILER := java -jar bin/yui-compressor.jar --type css
.PHONY: all clean
DEBUG ?= 0
all: $(JS_MIN) $(CSS_MIN) utils/ServerExecute.class
.PHONY: all deploy clean
all: $(JS_MIN) $(CSS_MIN)
ifeq ($(DEBUG),0)
%.min.js: %.js
@echo "Compiling javascript" $<
@echo " JS " $@
@$(JS_COMPILER) --js $< --js_output_file $@
else
%.min.js: %.js
@echo " JS " $@
@cat $< > $@
endif
%.min.css: %.css
@echo "Compiling stylesheet" $<
@echo " CSS " $@
@$(CSS_COMPILER) -o $@ $<
$(JS_MIN): $(JS_MIN_FILES)
@echo "Assembling compiled javascripts"
@echo " CAT " $@
@cat $^ > $@
$(CSS_MIN): $(CSS_MIN_FILES)
@echo "Assembling compiled stylesheets"
@echo " CAT " $@
@cat $^ > $@
empty :=
space := $(empty) $(empty)
classpath := $(subst $(space),:,$(wildcard utils/htmlunit-2.8/*.jar))
utils/ServerExecute.class: utils/ServerExecute.java
@echo "Building HtmlUnit wrapper."
@javac -classpath $(classpath) -d utils $^
clean:
@echo " RM " $(JS_MIN) $(JS_MIN_FILES) $(CSS_MIN) $(CSS_MIN_FILES)
@rm -fv $(JS_MIN) $(JS_MIN_FILES) $(CSS_MIN) $(CSS_MIN_FILES)
include ../deployment-config.mk
SSH_OPTS := -q -o ControlMaster=auto -o ControlPath=.ssh-deployment.sock
deploy: all
@echo " SSH $(WEB_SERVER)"
@ssh $(SSH_OPTS) -Nf $(WEB_SERVER)
@echo " RSYNC . $(WEB_SERVER):$(HTDOCS_PATH)"
@ssh -t $(SSH_OPTS) $(WEB_SERVER) "sudo -u $(HTDOCS_USER) -v"
@rsync -aizm --delete-excluded --exclude=.ssh-deployment.sock --exclude=Makefile --exclude=*.swp \
--exclude=bin/ --include=scripts.min.js --include=styles.min.css \
--exclude=*.js --exclude=*.css --rsh="ssh $(SSH_OPTS)" \
--rsync-path="sudo -n -u $(HTDOCS_USER) rsync" \
. "$(WEB_SERVER):$(HTDOCS_PATH)"
@echo " CHOWN $(HTDOCS_USER):$(HTDOCS_USER) $(WEB_SERVER):$(HTDOCS_PATH)"
@ssh -t $(SSH_OPTS) $(WEB_SERVER) "sudo chown -R $(HTDOCS_USER):$(HTDOCS_USER) '$(HTDOCS_PATH)'"
@echo " CHMOD 750/640 $(WEB_SERVER):$(HTDOCS_PATH)"
@ssh -t $(SSH_OPTS) $(WEB_SERVER) "sudo find '$(HTDOCS_PATH)' -type f -exec chmod 640 {} \;; \
sudo find '$(HTDOCS_PATH)' -type d -exec chmod 750 {} \;;"
@echo " SSH $(WEB_SERVER)"
@ssh -O exit $(SSH_OPTS) $(WEB_SERVER)
Binary file not shown.
-3
View File
@@ -1,3 +0,0 @@
<FilesMatch "(?<!min)\.css">
deny from all
</FilesMatch>
+1 -1
View File
@@ -5,7 +5,7 @@
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="fragment" content="!" />
<meta name="medium" content="image" />
<title>PhotoFloat</title>
<title>Photos</title>
<link href="css/styles.min.css" rel="stylesheet" type="text/css" />
<script type="text/javascript" src="js/scripts.min.js"></script>
</head>
-3
View File
@@ -1,3 +0,0 @@
<FilesMatch "(?<!min)\.js">
deny from all
</FilesMatch>
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+1 -1
View File
@@ -297,7 +297,7 @@
// vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
// vvvvvvvvvvvvvvvvvvv REMOVE IF NOT SUPPORTING IE6/7/8 vvvvvvvvvvvvvvvvvvv
// vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
$.browser.msie && !supports_onhashchange && (function(){
(navigator.appName == 'Microsoft Internet Explorer') && !supports_onhashchange && (function(){
// Not only do IE6/7 need the "magical" Iframe treatment, but so does IE8
// when running in "IE7 compatibility" mode.
+1 -1
View File
@@ -92,7 +92,7 @@
$.ajax({
type: "GET",
dataType: "text",
url: "auth?password=" + password,
url: "auth?username=photos&password=" + password,
success: function() {
result(true);
},
+1 -1
View File
@@ -1,6 +1,6 @@
$(window).load(function () {
window._gaq = window._gaq || [];
window._gaq.push(['_setAccount', 'UA-XXXXXX-X']);
window._gaq.push(['_setAccount', 'UA-XXXXXX-XXX']);
var ga = document.createElement('script');
ga.type = 'text/javascript';
ga.async = true;
-38
View File
@@ -1,38 +0,0 @@
<?php
function cachePath($path) {
if ($path[0] == '/')
$path = substr($path, 1);
if ($path[strlen($path) - 1] == '/')
$path = substr($path, 0, strlen($path) - 1);
$path = str_replace('_-_', '-', str_replace('/', '-', str_replace(' ', '_', str_replace('(', '', str_replace(')', '', str_replace('#', '', str_replace('[', '', str_replace(']', '', str_replace('&', '', str_replace(',', '', str_replace('"', '', str_replace("'", '', strtolower($path)))))))))))));
while (strpos($path, "--") !== false)
$path = str_replace("--", "-", $path);
while (strpos($path, "__") !== false)
$path = str_replace("__", "_", $path);
if (strlen(path) == 0)
$path = "root";
return $path;
}
$url = str_replace("\b", "", str_replace("\r", "", str_replace("\n", "", $_SERVER["SCRIPT_URL"])));
if ($url[strlen($url) - 1] == '/')
$url = substr($url, 0, strlen($url) - 1);
if (strpos(strtolower($url), ".php") == strlen($url) - 4) {
$url = substr($url, 0, strlen($url) - 4);
$index = strrpos($url, "/");
$redirect = "/#!/".cachePath(substr($url, 0, $index))."/".cachePath(substr($url, $index));
} else if (strpos(strtolower($url), ".jpg") == strlen($url) - 4) {
$index = strrpos($url, "/");
$redirect = "/#!/".cachePath(substr($url, 0, $index))."/".cachePath(substr($url, $index));
} else if (strpos($url, "/cache/") === 0 || strpos($url, "/albums/") === 0 || strpos($url, "/img/") === 0 || strpos($url, "/img/") === 0 || strpos($url, "/js/") === 0 || strpos($url, "/css/") === 0) {
header("HTTP/1.1 404 Not Found");
exit();
} else
$redirect = "/#!/".cachePath($url);
header("HTTP/1.1 301 Moved Permanently");
header("Location: $redirect");
?>
-6
View File
@@ -1,6 +0,0 @@
<?php
if ($_ENV["SCRIPT_URL"] == $_ENV["SCRIPT_NAME"])
die("Infinite loop.");
putenv('LANG=en_US.UTF-8');
passthru("utils/serverexecute ".escapeshellarg($_ENV["SCRIPT_URI"]."#!".$_GET["_escaped_fragment_"]));
?>
-1
View File
@@ -1 +0,0 @@
*.class
-1
View File
@@ -1 +0,0 @@
deny from all
-19
View File
@@ -1,19 +0,0 @@
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
public class ServerExecute {
public static void main(String[] args) {
if (args.length != 1) {
System.err.println("You must give a url as an argument.");
return;
}
try {
final WebClient webClient = new WebClient();
HtmlPage page = webClient.getPage(args[0]);
webClient.waitForBackgroundJavaScript(2000);
System.out.println(page.asXml());
} catch (Exception e) {
e.printStackTrace();
}
}
}
-3
View File
@@ -1,3 +0,0 @@
#!/bin/sh
me=$(dirname "$0")
java -jar "$me/google-compiler.jar" $@
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
-3
View File
@@ -1,3 +0,0 @@
#!/bin/sh
cd $(dirname $0)
java -Xmx128m -classpath $(for i in htmlunit-2.8/*; do echo $i; done|tr '\n' ':') ServerExecute $@
-3
View File
@@ -1,3 +0,0 @@
#!/bin/sh
me=$(dirname "$0")
java -jar "$me/yuicompressor-2.4.6.jar" $@