diff --git a/.gitignore b/.gitignore index e3740eb..5b5dd2a 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ scanner/floatapp/app.cfg *.pyc *.min.css *.min.js +.vscode \ No newline at end of file diff --git a/scanner/CachePath.py b/scanner/CachePath.py index a94364b..09ae0d8 100644 --- a/scanner/CachePath.py +++ b/scanner/CachePath.py @@ -1,49 +1,90 @@ import os.path from datetime import datetime + def message(category, text): - if message.level <= 0: - sep = " " - else: - sep = "--" - print "%s %s%s[%s]%s%s" % (datetime.now().isoformat(), max(0, message.level) * " |", sep, category, max(1, (14 - len(category))) * " ", text) + if message.level <= 0: + sep = " " + else: + sep = "--" + print "%s %s%s[%s]%s%s" % ( + datetime.now().isoformat(), + max(0, message.level) * " |", + sep, + category, + max(1, (14 - len(category))) * " ", + text) message.level = -1 + + def next_level(): - message.level += 1 + message.level += 1 + + def back_level(): - message.level -= 1 + message.level -= 1 + + def set_cache_path_base(base): - trim_base.base = base + trim_base.base = base + + def untrim_base(path): - return os.path.join(trim_base.base, path) + return os.path.join(trim_base.base, path) + + def trim_base_custom(path, base): - if path.startswith(base): - path = path[len(base):] - if path.startswith('/'): - path = path[1:] - return path + if path.startswith(base): + path = path[len(base):] + if path.startswith('/'): + path = path[1:] + return path + + def trim_base(path): - return trim_base_custom(path, trim_base.base) + return trim_base_custom(path, trim_base.base) + + def cache_base(path, filepath=False): - if len(path) == 0: - return "root" - elif filepath and len(path.split(os.sep)) < 2: - path = "root-" + path - path = trim_base(path).replace('/', '-').replace(' ', '_').replace('(', '').replace('&', '').replace(',', '').replace(')', '').replace('#', '').replace('[', '').replace(']', '').replace('"', '').replace("'", '').replace('_-_', '-').lower() - while path.find("--") != -1: - path = path.replace("--", "-") - while path.find("__") != -1: - path = path.replace("__", "_") - return path + if len(path) == 0: + return "root" + elif filepath and len(path.split(os.sep)) < 2: + path = "root-" + path + path = trim_base(path).replace( + '/', '-').replace( + ' ', '_').replace( + '(', '').replace( + '&', '').replace( + ',', '').replace( + ')', '').replace( + '#', '').replace( + '[', '').replace( + ']', '').replace( + '"', '').replace( + "'", '').replace( + '_-_', '-').lower() + while path.find("--") != -1: + path = path.replace("--", "-") + while path.find("__") != -1: + path = path.replace("__", "_") + return path + + def json_cache(path): - return cache_base(path) + ".json" + return cache_base(path) + ".json" + + def image_cache(path, size, square=False): - if square: - suffix = str(size) + "s" - else: - suffix = str(size) - return cache_base(path, True) + "_" + suffix + ".jpg" + if square: + suffix = str(size) + "s" + else: + suffix = str(size) + return cache_base(path, True) + "_" + suffix + ".jpg" + + def video_cache(path): - return cache_base(path, True) + ".mp4" + return cache_base(path, True) + ".mp4" + + def file_mtime(path): - return datetime.fromtimestamp(int(os.path.getmtime(path))) + return datetime.fromtimestamp(int(os.path.getmtime(path))) diff --git a/scanner/PhotoAlbum.py b/scanner/PhotoAlbum.py index a1f14a3..50ef978 100644 --- a/scanner/PhotoAlbum.py +++ b/scanner/PhotoAlbum.py @@ -10,585 +10,754 @@ import gc import tempfile from VideoToolWrapper import * + def make_photo_thumbs(self, original_path, thumb_path, size): - # The pool methods use a queue.Queue to pass tasks to the worker processes. - # Everything that goes through the queue.Queue must be pickable, and since - # self._photo_thumbnail is not defined at the top level, it's not pickable. - # This is why we have this "dummy" function, so that it's pickable. - self._photo_thumbnail(original_path, thumb_path, size[0], size[1]) + # The pool methods use a queue.Queue to pass tasks to the worker processes. + # Everything that goes through the queue.Queue must be pickable, and since + # self._photo_thumbnail is not defined at the top level, it's not pickable. + # This is why we have this "dummy" function, so that it's pickable. + self._photo_thumbnail(original_path, thumb_path, size[0], size[1]) + class Album(object): - def __init__(self, path): - self._path = trim_base(path) - self._photos = list() - self._albums = list() - self._photos_sorted = True - self._albums_sorted = True - @property - def photos(self): - return self._photos - @property - def albums(self): - return self._albums - @property - def path(self): - return self._path - def __str__(self): - return self.path - @property - def cache_path(self): - return json_cache(self.path) - @property - def date(self): - self._sort() - if len(self._photos) == 0 and len(self._albums) == 0: - return datetime(1900, 1, 1) - elif len(self._photos) == 0: - return self._albums[-1].date - elif len(self._albums) == 0: - return self._photos[-1].date - return max(self._photos[-1].date, self._albums[-1].date) - def __cmp__(self, other): - return cmp(self.date, other.date) - def add_photo(self, photo): - self._photos.append(photo) - self._photos_sorted = False - def add_album(self, album): - self._albums.append(album) - self._albums_sorted = False - def _sort(self): - if not self._photos_sorted: - self._photos.sort() - self._photos_sorted = True - if not self._albums_sorted: - self._albums.sort() - self._albums_sorted = True - @property - def empty(self): - if len(self._photos) != 0: - return False - if len(self._albums) == 0: - return True - for album in self._albums: - if not album.empty: - return False - return True - - def cache(self, base_dir): - self._sort() - fp = open(os.path.join(base_dir, self.cache_path), 'w') - json.dump(self, fp, cls=PhotoAlbumEncoder) - fp.close() - @staticmethod - def from_cache(path): - fp = open(path, "r") - dictionary = json.load(fp) - fp.close() - return Album.from_dict(dictionary) - @staticmethod - def from_dict(dictionary, cripple=True): - album = Album(dictionary["path"]) - for photo in dictionary["photos"]: - album.add_photo(Photo.from_dict(photo, untrim_base(album.path))) - if not cripple: - for subalbum in dictionary["albums"]: - album.add_album(Album.from_dict(subalbum), cripple) - album._sort() - return album - def to_dict(self, cripple=True): - self._sort() - subalbums = [] - if cripple: - for sub in self._albums: - if not sub.empty: - subalbums.append({ "path": trim_base_custom(sub.path, self._path), "date": sub.date }) - else: - for sub in self._albums: - if not sub.empty: - subalbums.append(sub) - return { "path": self.path, "date": self.date, "albums": subalbums, "photos": self._photos } - def photo_from_path(self, path): - for photo in self._photos: - if trim_base(path) == photo._path: - return photo - return None + def __init__(self, path): + self._path = trim_base(path) + self._photos = list() + self._albums = list() + self._photos_sorted = True + self._albums_sorted = True + + @property + def photos(self): + return self._photos + + @property + def albums(self): + return self._albums + + @property + def path(self): + return self._path + + def __str__(self): + return self.path + + @property + def cache_path(self): + return json_cache(self.path) + + @property + def date(self): + self._sort() + if len(self._photos) == 0 and len(self._albums) == 0: + return datetime(1900, 1, 1) + elif len(self._photos) == 0: + return self._albums[-1].date + elif len(self._albums) == 0: + return self._photos[-1].date + return max(self._photos[-1].date, self._albums[-1].date) + + def __cmp__(self, other): + return cmp(self.date, other.date) + + def add_photo(self, photo): + self._photos.append(photo) + self._photos_sorted = False + + def add_album(self, album): + self._albums.append(album) + self._albums_sorted = False + + def _sort(self): + if not self._photos_sorted: + self._photos.sort() + self._photos_sorted = True + if not self._albums_sorted: + self._albums.sort() + self._albums_sorted = True + + @property + def empty(self): + if len(self._photos) != 0: + return False + if len(self._albums) == 0: + return True + for album in self._albums: + if not album.empty: + return False + return True + + def cache(self, base_dir): + self._sort() + fp = open(os.path.join(base_dir, self.cache_path), 'w') + json.dump(self, fp, cls=PhotoAlbumEncoder) + fp.close() + + @staticmethod + def from_cache(path): + fp = open(path, "r") + dictionary = json.load(fp) + fp.close() + return Album.from_dict(dictionary) + + @staticmethod + def from_dict(dictionary, cripple=True): + album = Album(dictionary["path"]) + for photo in dictionary["photos"]: + album.add_photo(Photo.from_dict(photo, untrim_base(album.path))) + if not cripple: + for subalbum in dictionary["albums"]: + album.add_album(Album.from_dict(subalbum), cripple) + album._sort() + return album + + def to_dict(self, cripple=True): + self._sort() + subalbums = [] + if cripple: + for sub in self._albums: + if not sub.empty: + subalbums.append({ + "path": trim_base_custom(sub.path, self._path), + "date": sub.date + }) + else: + for sub in self._albums: + if not sub.empty: + subalbums.append(sub) + return { + "path": self.path, + "date": self.date, + "albums": subalbums, + "photos": self._photos + } + + def photo_from_path(self, path): + for photo in self._photos: + if trim_base(path) == photo._path: + return photo + return None + class Photo(object): - thumb_sizes = [ (75, True), (150, True), (640, False), (1024, False), (1600, False) ] - def __init__(self, path, thumb_path=None, attributes=None): - self._path = trim_base(path) - self.is_valid = True - image = None - try: - mtime = file_mtime(path) - except KeyboardInterrupt: - raise - except: - self.is_valid = False - return - if attributes is not None and attributes["dateTimeFile"] >= mtime: - self._attributes = attributes - return - self._attributes = {} - self._attributes["dateTimeFile"] = mtime - self._attributes["mediaType"] = "photo" - - try: - image = Image.open(path) - except KeyboardInterrupt: - raise - except: - self._video_metadata(path) + thumb_sizes = [ + (75, True), (150, True), (640, False), (1024, False), (1600, False)] - if isinstance(image, Image.Image): - self._photo_metadata(image) - self._photo_thumbnails(path, thumb_path) - elif self._attributes["mediaType"] == "video": - self._video_thumbnails(thumb_path, path) - self._video_transcode(thumb_path, path) - else: - self.is_valid = False - return - - def _photo_metadata(self, image): - self._attributes["size"] = image.size - self._orientation = 1 - try: - info = image._getexif() - except KeyboardInterrupt: - raise - except: - return - if not info: - return - - exif = {} - for tag, value in info.items(): - decoded = TAGS.get(tag, tag) - if (isinstance(value, tuple) or isinstance(value, list)) and (isinstance(decoded, str) or isinstance(decoded, unicode)) and decoded.startswith("DateTime") and len(value) >= 1: - value = value[0] - if isinstance(value, str) or isinstance(value, unicode): - value = value.strip().partition("\x00")[0] - if (isinstance(decoded, str) or isinstance(decoded, unicode)) and decoded.startswith("DateTime"): - try: - value = datetime.strptime(value, '%Y:%m:%d %H:%M:%S') - except KeyboardInterrupt: - raise - except: - continue - exif[decoded] = value - - if "Orientation" in exif: - self._orientation = exif["Orientation"]; - if self._orientation in range(5, 9): - self._attributes["size"] = (self._attributes["size"][1], self._attributes["size"][0]) - if self._orientation - 1 < len(self._photo_metadata.orientation_list): - self._attributes["orientation"] = self._photo_metadata.orientation_list[self._orientation - 1] - if "Make" in exif: - self._attributes["make"] = exif["Make"] - if "Model" in exif: - self._attributes["model"] = exif["Model"] - if "ApertureValue" in exif: - self._attributes["aperture"] = exif["ApertureValue"] - elif "FNumber" in exif: - self._attributes["aperture"] = exif["FNumber"] - if "FocalLength" in exif: - self._attributes["focalLength"] = exif["FocalLength"] - if "ISOSpeedRatings" in exif: - self._attributes["iso"] = exif["ISOSpeedRatings"] - if "ISO" in exif: - self._attributes["iso"] = exif["ISO"] - if "PhotographicSensitivity" in exif: - self._attributes["iso"] = exif["PhotographicSensitivity"] - if "ExposureTime" in exif: - self._attributes["exposureTime"] = exif["ExposureTime"] - if "Flash" in exif and exif["Flash"] in self._photo_metadata.flash_dictionary: - try: - self._attributes["flash"] = self._photo_metadata.flash_dictionary[exif["Flash"]] - except KeyboardInterrupt: - raise - except: - pass - if "LightSource" in exif and exif["LightSource"] in self._photo_metadata.light_source_dictionary: - try: - self._attributes["lightSource"] = self._photo_metadata.light_source_dictionary[exif["LightSource"]] - except KeyboardInterrupt: - raise - except: - pass - if "ExposureProgram" in exif and exif["ExposureProgram"] < len(self._photo_metadata.exposure_list): - self._attributes["exposureProgram"] = self._photo_metadata.exposure_list[exif["ExposureProgram"]] - if "SpectralSensitivity" in exif: - self._attributes["spectralSensitivity"] = exif["SpectralSensitivity"] - if "MeteringMode" in exif and exif["MeteringMode"] < len(self._photo_metadata.metering_list): - self._attributes["meteringMode"] = self._photo_metadata.metering_list[exif["MeteringMode"]] - if "SensingMethod" in exif and exif["SensingMethod"] < len(self._photo_metadata.sensing_method_list): - self._attributes["sensingMethod"] = self._photo_metadata.sensing_method_list[exif["SensingMethod"]] - if "SceneCaptureType" in exif and exif["SceneCaptureType"] < len(self._photo_metadata.scene_capture_type_list): - self._attributes["sceneCaptureType"] = self._photo_metadata.scene_capture_type_list[exif["SceneCaptureType"]] - if "SubjectDistanceRange" in exif and exif["SubjectDistanceRange"] < len(self._photo_metadata.subject_distance_range_list): - self._attributes["subjectDistanceRange"] = self._photo_metadata.subject_distance_range_list[exif["SubjectDistanceRange"]] - if "ExposureCompensation" in exif: - self._attributes["exposureCompensation"] = exif["ExposureCompensation"] - if "ExposureBiasValue" in exif: - self._attributes["exposureCompensation"] = exif["ExposureBiasValue"] - if "DateTimeOriginal" in exif: - try: - self._attributes["dateTimeOriginal"] = datetime.strptime(exif["DateTimeOriginal"], '%Y:%m:%d %H:%M:%S') - except KeyboardInterrupt: - raise - except TypeError: - self._attributes["dateTimeOriginal"] = exif["DateTimeOriginal"] - if "DateTime" in exif: - try: - self._attributes["dateTime"] = datetime.strptime(exif["DateTime"], '%Y:%m:%d %H:%M:%S') - except KeyboardInterrupt: - raise - except TypeError: - self._attributes["dateTime"] = exif["DateTime"] - - _photo_metadata.flash_dictionary = {0x0: "No Flash", 0x1: "Fired",0x5: "Fired, Return not detected",0x7: "Fired, Return detected",0x8: "On, Did not fire",0x9: "On, Fired",0xd: "On, Return not detected",0xf: "On, Return detected",0x10: "Off, Did not fire",0x14: "Off, Did not fire, Return not detected",0x18: "Auto, Did not fire",0x19: "Auto, Fired",0x1d: "Auto, Fired, Return not detected",0x1f: "Auto, Fired, Return detected",0x20: "No flash function",0x30: "Off, No flash function",0x41: "Fired, Red-eye reduction",0x45: "Fired, Red-eye reduction, Return not detected",0x47: "Fired, Red-eye reduction, Return detected",0x49: "On, Red-eye reduction",0x4d: "On, Red-eye reduction, Return not detected",0x4f: "On, Red-eye reduction, Return detected",0x50: "Off, Red-eye reduction",0x58: "Auto, Did not fire, Red-eye reduction",0x59: "Auto, Fired, Red-eye reduction",0x5d: "Auto, Fired, Red-eye reduction, Return not detected",0x5f: "Auto, Fired, Red-eye reduction, Return detected"} - _photo_metadata.light_source_dictionary = {0: "Unknown", 1: "Daylight", 2: "Fluorescent", 3: "Tungsten (incandescent light)", 4: "Flash", 9: "Fine weather", 10: "Cloudy weather", 11: "Shade", 12: "Daylight fluorescent (D 5700 - 7100K)", 13: "Day white fluorescent (N 4600 - 5400K)", 14: "Cool white fluorescent (W 3900 - 4500K)", 15: "White fluorescent (WW 3200 - 3700K)", 17: "Standard light A", 18: "Standard light B", 19: "Standard light C", 20: "D55", 21: "D65", 22: "D75", 23: "D50", 24: "ISO studio tungsten"} - _photo_metadata.metering_list = ["Unknown", "Average", "Center-weighted average", "Spot", "Multi-spot", "Multi-segment", "Partial"] - _photo_metadata.exposure_list = ["Not Defined", "Manual", "Program AE", "Aperture-priority AE", "Shutter speed priority AE", "Creative (Slow speed)", "Action (High speed)", "Portrait", "Landscape", "Bulb"] - _photo_metadata.orientation_list = ["Horizontal (normal)", "Mirror horizontal", "Rotate 180", "Mirror vertical", "Mirror horizontal and rotate 270 CW", "Rotate 90 CW", "Mirror horizontal and rotate 90 CW", "Rotate 270 CW"] - _photo_metadata.sensing_method_list = ["Not defined", "One-chip color area sensor", "Two-chip color area sensor", "Three-chip color area sensor", "Color sequential area sensor", "Trilinear sensor", "Color sequential linear sensor"] - _photo_metadata.scene_capture_type_list = ["Standard", "Landscape", "Portrait", "Night scene"] - _photo_metadata.subject_distance_range_list = ["Unknown", "Macro", "Close view", "Distant view"] + def __init__(self, path, thumb_path=None, attributes=None): + self._path = trim_base(path) + self.is_valid = True + image = None + try: + mtime = file_mtime(path) + except KeyboardInterrupt: + raise + except: + self.is_valid = False + return + if attributes is not None and attributes["dateTimeFile"] >= mtime: + self._attributes = attributes + return + self._attributes = {} + self._attributes["dateTimeFile"] = mtime + self._attributes["mediaType"] = "photo" + try: + image = Image.open(path) + except KeyboardInterrupt: + raise + except: + self._video_metadata(path) - def _video_metadata(self, path, original=True): - p = VideoProbeWrapper().call('-show_format', '-show_streams', '-of', 'json', '-loglevel', '0', path) - if p == False: - self.is_valid = False - return - info = json.loads(p) - for s in info["streams"]: - if 'codec_type' in s and s['codec_type'] == 'video': - self._attributes["mediaType"] = "video" - self._attributes["size"] = (int(s["width"]), int(s["height"])) - if "duration" in s: - self._attributes["duration"] = s["duration"] - if "tags" in s and "rotate" in s["tags"]: - self._attributes["rotate"] = s["tags"]["rotate"] - if original: - self._attributes["originalSize"] = (int(s["width"]), int(s["height"])) - # we break, because a video can contain several streams - # this way we only get/use values from the first stream - break - - # use time from EXIF (rather than file creation) - try: - info['format']['tags']['creation_time'] - except KeyError: - pass - else: - # we have time modifiable via exif - # lets use this - - try: - self._attributes["dateTimeVideo"] = datetime.strptime(info['format']['tags']['creation_time'], '%Y-%m-%d %H:%M:%S') - except KeyboardInterrupt: - raise - except TypeError: - pass - - def _photo_thumbnail(self, original_path, thumb_path, size, square=False): - try: - image = Image.open(original_path) - except KeyboardInterrupt: - raise - except: - self.is_valid = False - return - - mirror = image - if self._orientation == 2: - # Vertical Mirror - mirror = image.transpose(Image.FLIP_LEFT_RIGHT) - elif self._orientation == 3: - # Rotation 180 - mirror = image.transpose(Image.ROTATE_180) - elif self._orientation == 4: - # Horizontal Mirror - mirror = image.transpose(Image.FLIP_TOP_BOTTOM) - elif self._orientation == 5: - # Horizontal Mirror + Rotation 270 - mirror = image.transpose(Image.FLIP_TOP_BOTTOM).transpose(Image.ROTATE_270) - elif self._orientation == 6: - # Rotation 270 - mirror = image.transpose(Image.ROTATE_270) - elif self._orientation == 7: - # Vertical Mirror + Rotation 270 - mirror = image.transpose(Image.FLIP_LEFT_RIGHT).transpose(Image.ROTATE_270) - elif self._orientation == 8: - # Rotation 90 - mirror = image.transpose(Image.ROTATE_90) - - image = mirror - self._thumbnail(image, original_path, thumb_path, size, square) - - def _thumbnail(self, image, original_path, thumb_path, size, square): - thumb_path = os.path.join(thumb_path, image_cache(self._path, size, square)) - info_string = "%s -> %spx" % (os.path.basename(original_path), str(size)) - if square: - info_string += ", square" - message("thumbing", info_string) - if os.path.exists(thumb_path) and file_mtime(thumb_path) >= self._attributes["dateTimeFile"]: - return - gc.collect() - try: - image = image.copy() - except KeyboardInterrupt: - raise - except: - try: - image = image.copy() # we try again to work around PIL bug - except KeyboardInterrupt: - raise - except: - message("corrupt image", os.path.basename(original_path)) - self.is_valid = False - return - if square: - if image.size[0] > image.size[1]: - left = (image.size[0] - image.size[1]) / 2 - top = 0 - right = image.size[0] - ((image.size[0] - image.size[1]) / 2) - bottom = image.size[1] - else: - left = 0 - top = (image.size[1] - image.size[0]) / 2 - right = image.size[0] - bottom = image.size[1] - ((image.size[1] - image.size[0]) / 2) - image = image.crop((left, top, right, bottom)) - gc.collect() - image.thumbnail((size, size), Image.ANTIALIAS) - try: - image.save(thumb_path, "JPEG", quality=88) - except KeyboardInterrupt: - try: - os.unlink(thumb_path) - except: - pass - raise - except: - message("save failure", os.path.basename(thumb_path)) - try: - os.unlink(thumb_path) - except: - pass - - def _photo_thumbnails(self, original_path, thumb_path): - # get number of cores on the system, and use all minus one - num_of_cores = os.sysconf('SC_NPROCESSORS_ONLN') - 1 - pool = Pool(processes=num_of_cores) - - try: - for size in Photo.thumb_sizes: - pool.apply_async(make_photo_thumbs, args = (self, original_path, thumb_path, size)) - except: - pool.terminate() - - pool.close() - pool.join() + if isinstance(image, Image.Image): + self._photo_metadata(image) + self._photo_thumbnails(path, thumb_path) + elif self._attributes["mediaType"] == "video": + self._video_thumbnails(thumb_path, path) + self._video_transcode(thumb_path, path) + else: + self.is_valid = False + return - def _video_thumbnails(self, thumb_path, original_path): - (tfd, tfn) = tempfile.mkstemp(); - p = VideoTranscodeWrapper().call( - '-i', original_path, # original file to extract thumbs from - '-f', 'image2', # extract image - '-vsync', '1', # CRF - '-vframes', '1', # extrat 1 single frame - '-an', # disable audio - '-loglevel', 'quiet', # don't display anything - '-y', # don't prompt for overwrite - tfn # temporary file to store extracted image - ) - if p == False: - message("couldn't extract video frame", os.path.basename(original_path)) - try: - os.unlink(tfn) - except: - pass - self.is_valid = False - return - try: - image = Image.open(tfn) - except KeyboardInterrupt: - try: - os.unlink(tfn) - except: - pass - raise - except: - message("couldn't open video thumbnail", tfn) - try: - os.unlink(tfn) - except: - pass - self.is_valid = False - return - mirror = image - if "rotate" in self._attributes: - if self._attributes["rotate"] == "90": - mirror = image.transpose(Image.ROTATE_270) - elif self._attributes["rotate"] == "180": - mirror = image.transpose(Image.ROTATE_180) - elif self._attributes["rotate"] == "270": - mirror = image.transpose(Image.ROTATE_90) - for size in Photo.thumb_sizes: - if size[1]: - self._thumbnail(mirror, original_path, thumb_path, size[0], size[1]) - try: - os.unlink(tfn) - except: - pass - - def _video_transcode(self, transcode_path, original_path): - transcode_path = os.path.join(transcode_path, video_cache(self._path)) - # get number of cores on the system, and use all minus one - num_of_cores = os.sysconf('SC_NPROCESSORS_ONLN') - 1 - transcode_cmd = [ - '-i', original_path, # original file to be encoded - '-c:v', 'libx264', # set h264 as videocodec - '-preset', 'slow', # set specific preset that provides a certain encoding speed to compression ratio - '-profile:v', 'baseline', # set output to specific h264 profile - '-level', '3.0', # sets highest compatibility with target devices - '-crf', '20', # set quality - '-b:v', '4M', # set videobitrate to 4Mbps - '-strict', 'experimental', # allow native aac codec below - '-c:a', 'aac', # set aac as audiocodec - '-ac', '2', # force two audiochannels - '-ab', '160k', # set audiobitrate to 160Kbps - '-maxrate', '10000000', # limits max rate, will degrade CRF if needed - '-bufsize', '10000000', # define how much the client should buffer - '-f', 'mp4', # fileformat mp4 - '-threads', str(num_of_cores), # number of cores (all minus one) - '-loglevel', 'quiet', # don't display anything - '-y' # don't prompt for overwrite - ] - filters = [] - info_string = "%s -> mp4, h264" % (os.path.basename(original_path)) - message("transcoding", info_string) - if os.path.exists(transcode_path) and file_mtime(transcode_path) >= self._attributes["dateTimeFile"]: - self._video_metadata(transcode_path, False) - return - if "originalSize" in self._attributes: - width = self._attributes["originalSize"][0] - height = self._attributes["originalSize"][1] - if width > height: - # horisontal orientation - if height > 720: - transcode_cmd.append('-vf') - transcode_cmd.append('scale=-1:720') - elif (height > width) or (width == height): - # vertical orientation, or equal sides - if width > 720: - transcode_cmd.append('-vf') - transcode_cmd.append('scale=720:-1') - if "rotate" in self._attributes: - if self._attributes["rotate"] == "90": - filters.append('transpose=1') - elif self._attributes["rotate"] == "180": - filters.append('vflip,hflip') - elif self._attributes["rotate"] == "270": - filters.append('transpose=2') - if len(filters): - transcode_cmd.append('-vf') - transcode_cmd.append(','.join(filters)) - - tmp_transcode_cmd = transcode_cmd[:] - transcode_cmd.append(transcode_path) - p = VideoTranscodeWrapper().call(*transcode_cmd) - if p == False: - # add another option, try transcoding again - # done to avoid this error; - # x264 [error]: baseline profile doesn't support 4:2:2 - message("transcoding failure, trying yuv420p", os.path.basename(original_path)) - tmp_transcode_cmd.append('-pix_fmt') - tmp_transcode_cmd.append('yuv420p') - tmp_transcode_cmd.append(transcode_path) - p = VideoTranscodeWrapper().call(*tmp_transcode_cmd) - - if p == False: - message("transcoding failure", os.path.basename(original_path)) - try: - os.unlink(transcode_path) - except: - pass - self.is_valid = False - return - self._video_metadata(transcode_path, False) - - @property - def name(self): - return os.path.basename(self._path) - def __str__(self): - return self.name - @property - def path(self): - return self._path - @property - def image_caches(self): - caches = [] - if "mediaType" in self._attributes and self._attributes["mediaType"] == "video": - for size in Photo.thumb_sizes: - if size[1]: - caches.append(image_cache(self._path, size[0], size[1])) - caches.append(video_cache(self._path)) - else: - caches = [image_cache(self._path, size[0], size[1]) for size in Photo.thumb_sizes] - return caches - @property - def date(self): - correct_date = None; - if not self.is_valid: - correct_date = datetime(1900, 1, 1) - if "dateTimeVideo" in self._attributes: - correct_date = self._attributes["dateTimeVideo"] - elif "dateTimeOriginal" in self._attributes: - correct_date = self._attributes["dateTimeOriginal"] - elif "dateTime" in self._attributes: - correct_date = self._attributes["dateTime"] - else: - correct_date = self._attributes["dateTimeFile"] - return correct_date + def _photo_metadata(self, image): + self._attributes["size"] = image.size + self._orientation = 1 + try: + info = image._getexif() + except KeyboardInterrupt: + raise + except: + return + if not info: + return + + exif = {} + for tag, value in info.items(): + decoded = TAGS.get(tag, tag) + if ((isinstance(value, tuple) or isinstance(value, list)) and + (isinstance(decoded, str) or + isinstance(decoded, unicode)) and + decoded.startswith("DateTime") and + len(value) >= 1): + value = value[0] + if isinstance(value, str) or isinstance(value, unicode): + value = value.strip().partition("\x00")[0] + if (isinstance(decoded, str) or isinstance(decoded, unicode)) and decoded.startswith("DateTime"): + try: + value = datetime.strptime(value, '%Y:%m:%d %H:%M:%S') + except KeyboardInterrupt: + raise + except: + continue + exif[decoded] = value + + if "Orientation" in exif: + self._orientation = exif["Orientation"] + if self._orientation in range(5, 9): + self._attributes["size"] = ( + self._attributes["size"][1], self._attributes["size"][0]) + if self._orientation - 1 < len( + self._photo_metadata.orientation_list): + self._attributes["orientation"] = ( + self._photo_metadata.orientation_list[ + self._orientation - 1]) + if "Make" in exif: + self._attributes["make"] = exif["Make"] + if "Model" in exif: + self._attributes["model"] = exif["Model"] + if "ApertureValue" in exif: + self._attributes["aperture"] = exif["ApertureValue"] + elif "FNumber" in exif: + self._attributes["aperture"] = exif["FNumber"] + if "FocalLength" in exif: + self._attributes["focalLength"] = exif["FocalLength"] + if "ISOSpeedRatings" in exif: + self._attributes["iso"] = exif["ISOSpeedRatings"] + if "ISO" in exif: + self._attributes["iso"] = exif["ISO"] + if "PhotographicSensitivity" in exif: + self._attributes["iso"] = exif["PhotographicSensitivity"] + if "ExposureTime" in exif: + self._attributes["exposureTime"] = exif["ExposureTime"] + if "Flash" in exif and exif["Flash"] in self._photo_metadata.flash_dictionary: + try: + self._attributes["flash"] = self._photo_metadata.flash_dictionary[exif["Flash"]] + except KeyboardInterrupt: + raise + except: + pass + if "LightSource" in exif and exif["LightSource"] in self._photo_metadata.light_source_dictionary: + try: + self._attributes["lightSource"] = self._photo_metadata.light_source_dictionary[exif["LightSource"]] + except KeyboardInterrupt: + raise + except: + pass + if "ExposureProgram" in exif and exif["ExposureProgram"] < len(self._photo_metadata.exposure_list): + self._attributes["exposureProgram"] = self._photo_metadata.exposure_list[exif["ExposureProgram"]] + if "SpectralSensitivity" in exif: + self._attributes["spectralSensitivity"] = exif["SpectralSensitivity"] + if "MeteringMode" in exif and exif["MeteringMode"] < len(self._photo_metadata.metering_list): + self._attributes["meteringMode"] = self._photo_metadata.metering_list[exif["MeteringMode"]] + if "SensingMethod" in exif and exif["SensingMethod"] < len(self._photo_metadata.sensing_method_list): + self._attributes["sensingMethod"] = self._photo_metadata.sensing_method_list[exif["SensingMethod"]] + if "SceneCaptureType" in exif and exif["SceneCaptureType"] < len(self._photo_metadata.scene_capture_type_list): + self._attributes["sceneCaptureType"] = self._photo_metadata.scene_capture_type_list[exif["SceneCaptureType"]] + if "SubjectDistanceRange" in exif and exif["SubjectDistanceRange"] < len(self._photo_metadata.subject_distance_range_list): + self._attributes["subjectDistanceRange"] = self._photo_metadata.subject_distance_range_list[exif["SubjectDistanceRange"]] + if "ExposureCompensation" in exif: + self._attributes["exposureCompensation"] = exif["ExposureCompensation"] + if "ExposureBiasValue" in exif: + self._attributes["exposureCompensation"] = exif["ExposureBiasValue"] + if "DateTimeOriginal" in exif: + try: + self._attributes["dateTimeOriginal"] = datetime.strptime(exif["DateTimeOriginal"], '%Y:%m:%d %H:%M:%S') + except KeyboardInterrupt: + raise + except TypeError: + self._attributes["dateTimeOriginal"] = exif["DateTimeOriginal"] + if "DateTime" in exif: + try: + self._attributes["dateTime"] = datetime.strptime(exif["DateTime"], '%Y:%m:%d %H:%M:%S') + except KeyboardInterrupt: + raise + except TypeError: + self._attributes["dateTime"] = exif["DateTime"] + + _photo_metadata.flash_dictionary = { + 0x0: "No Flash", + 0x1: "Fired", + 0x5: "Fired, Return not detected", + 0x7: "Fired, Return detected", + 0x8: "On, Did not fire", + 0x9: "On, Fired", + 0xd: "On, Return not detected", + 0xf: "On, Return detected", + 0x10: "Off, Did not fire", + 0x14: "Off, Did not fire, Return not detected", + 0x18: "Auto, Did not fire", + 0x19: "Auto, Fired", + 0x1d: "Auto, Fired, Return not detected", + 0x1f: "Auto, Fired, Return detected", + 0x20: "No flash function", + 0x30: "Off, No flash function", + 0x41: "Fired, Red-eye reduction", + 0x45: "Fired, Red-eye reduction, Return not detected", + 0x47: "Fired, Red-eye reduction, Return detected", + 0x49: "On, Red-eye reduction", + 0x4d: "On, Red-eye reduction, Return not detected", + 0x4f: "On, Red-eye reduction, Return detected", + 0x50: "Off, Red-eye reduction", + 0x58: "Auto, Did not fire, Red-eye reduction", + 0x59: "Auto, Fired, Red-eye reduction", + 0x5d: "Auto, Fired, Red-eye reduction, Return not detected", + 0x5f: "Auto, Fired, Red-eye reduction, Return detected" + } + _photo_metadata.light_source_dictionary = { + 0: "Unknown", + 1: "Daylight", + 2: "Fluorescent", + 3: "Tungsten (incandescent light)", + 4: "Flash", + 9: "Fine weather", + 10: "Cloudy weather", + 11: "Shade", + 12: "Daylight fluorescent (D 5700 - 7100K)", + 13: "Day white fluorescent (N 4600 - 5400K)", + 14: "Cool white fluorescent (W 3900 - 4500K)", + 15: "White fluorescent (WW 3200 - 3700K)", + 17: "Standard light A", + 18: "Standard light B", + 19: "Standard light C", + 20: "D55", + 21: "D65", + 22: "D75", + 23: "D50", + 24: "ISO studio tungsten" + } + _photo_metadata.metering_list = [ + "Unknown", + "Average", + "Center-weighted average", + "Spot", + "Multi-spot", + "Multi-segment", + "Partial" + ] + _photo_metadata.exposure_list = [ + "Not Defined", + "Manual", + "Program AE", + "Aperture-priority AE", + "Shutter speed priority AE", + "Creative (Slow speed)", + "Action (High speed)", + "Portrait", + "Landscape", + "Bulb" + ] + _photo_metadata.orientation_list = [ + "Horizontal (normal)", + "Mirror horizontal", + "Rotate 180", + "Mirror vertical", + "Mirror horizontal and rotate 270 CW", + "Rotate 90 CW", + "Mirror horizontal and rotate 90 CW", + "Rotate 270 CW" + ] + _photo_metadata.sensing_method_list = [ + "Not defined", + "One-chip color area sensor", + "Two-chip color area sensor", + "Three-chip color area sensor", + "Color sequential area sensor", + "Trilinear sensor", + "Color sequential linear sensor" + ] + _photo_metadata.scene_capture_type_list = [ + "Standard", + "Landscape", + "Portrait", + "Night scene" + ] + _photo_metadata.subject_distance_range_list = [ + "Unknown", + "Macro", + "Close view", + "Distant view" + ] + + def _video_metadata(self, path, original=True): + p = VideoProbeWrapper().call( + '-show_format', + '-show_streams', + '-of', + 'json', + '-loglevel', + '0', + path) + if p is False: + self.is_valid = False + return + info = json.loads(p) + for s in info["streams"]: + if 'codec_type' in s and s['codec_type'] == 'video': + self._attributes["mediaType"] = "video" + self._attributes["size"] = (int(s["width"]), int(s["height"])) + if "duration" in s: + self._attributes["duration"] = s["duration"] + if "tags" in s and "rotate" in s["tags"]: + self._attributes["rotate"] = s["tags"]["rotate"] + if original: + self._attributes["originalSize"] = ( + int(s["width"]), int(s["height"])) + # we break, because a video can contain several streams + # this way we only get/use values from the first stream + break + + # use time from EXIF (rather than file creation) + try: + info['format']['tags']['creation_time'] + except KeyError: + pass + else: + # we have time modifiable via exif + # lets use this + + try: + self._attributes["dateTimeVideo"] = datetime.strptime( + info['format']['tags']['creation_time'], + '%Y-%m-%d %H:%M:%S') + except KeyboardInterrupt: + raise + except TypeError: + pass + + def _photo_thumbnail(self, original_path, thumb_path, size, square=False): + try: + image = Image.open(original_path) + except KeyboardInterrupt: + raise + except: + self.is_valid = False + return + + mirror = image + if self._orientation == 2: + # Vertical Mirror + mirror = image.transpose(Image.FLIP_LEFT_RIGHT) + elif self._orientation == 3: + # Rotation 180 + mirror = image.transpose(Image.ROTATE_180) + elif self._orientation == 4: + # Horizontal Mirror + mirror = image.transpose(Image.FLIP_TOP_BOTTOM) + elif self._orientation == 5: + # Horizontal Mirror + Rotation 270 + mirror = image.transpose( + Image.FLIP_TOP_BOTTOM).transpose(Image.ROTATE_270) + elif self._orientation == 6: + # Rotation 270 + mirror = image.transpose(Image.ROTATE_270) + elif self._orientation == 7: + # Vertical Mirror + Rotation 270 + mirror = image.transpose( + Image.FLIP_LEFT_RIGHT).transpose(Image.ROTATE_270) + elif self._orientation == 8: + # Rotation 90 + mirror = image.transpose(Image.ROTATE_90) + + image = mirror + self._thumbnail(image, original_path, thumb_path, size, square) + + def _thumbnail(self, image, original_path, thumb_path, size, square): + thumb_path = os.path.join( + thumb_path, image_cache(self._path, size, square)) + info_string = "%s -> %spx" % ( + os.path.basename(original_path), + str(size)) + if square: + info_string += ", square" + message("thumbing", info_string) + if os.path.exists(thumb_path) and file_mtime( + thumb_path) >= self._attributes["dateTimeFile"]: + return + gc.collect() + try: + image = image.copy() + except KeyboardInterrupt: + raise + except: + try: + image = image.copy() # we try again to work around PIL bug + except KeyboardInterrupt: + raise + except: + message("corrupt image", os.path.basename(original_path)) + self.is_valid = False + return + if square: + if image.size[0] > image.size[1]: + left = (image.size[0] - image.size[1]) / 2 + top = 0 + right = image.size[0] - ((image.size[0] - image.size[1]) / 2) + bottom = image.size[1] + else: + left = 0 + top = (image.size[1] - image.size[0]) / 2 + right = image.size[0] + bottom = image.size[1] - ((image.size[1] - image.size[0]) / 2) + image = image.crop((left, top, right, bottom)) + gc.collect() + image.thumbnail((size, size), Image.ANTIALIAS) + try: + image.save(thumb_path, "JPEG", quality=88) + except KeyboardInterrupt: + try: + os.unlink(thumb_path) + except: + pass + raise + except: + message("save failure", os.path.basename(thumb_path)) + try: + os.unlink(thumb_path) + except: + pass + + def _photo_thumbnails(self, original_path, thumb_path): + # get number of cores on the system, and use all minus one + num_of_cores = os.sysconf('SC_NPROCESSORS_ONLN') - 1 + pool = Pool(processes=num_of_cores) + + try: + for size in Photo.thumb_sizes: + pool.apply_async(make_photo_thumbs, args=( + self, original_path, thumb_path, size)) + except: + pool.terminate() + + pool.close() + pool.join() + + def _video_thumbnails(self, thumb_path, original_path): + (tfd, tfn) = tempfile.mkstemp() + p = VideoTranscodeWrapper().call( + '-i', original_path, # original file to extract thumbs from + '-f', 'image2', # extract image + '-vsync', '1', # CRF + '-vframes', '1', # extrat 1 single frame + '-an', # disable audio + '-loglevel', 'quiet', # don't display anything + '-y', # don't prompt for overwrite + tfn # temporary file to store extracted image + ) + if p is False: + message( + "couldn't extract video frame", + os.path.basename(original_path)) + try: + os.unlink(tfn) + except: + pass + self.is_valid = False + return + try: + image = Image.open(tfn) + except KeyboardInterrupt: + try: + os.unlink(tfn) + except: + pass + raise + except: + message("couldn't open video thumbnail", tfn) + try: + os.unlink(tfn) + except: + pass + self.is_valid = False + return + mirror = image + if "rotate" in self._attributes: + if self._attributes["rotate"] == "90": + mirror = image.transpose(Image.ROTATE_270) + elif self._attributes["rotate"] == "180": + mirror = image.transpose(Image.ROTATE_180) + elif self._attributes["rotate"] == "270": + mirror = image.transpose(Image.ROTATE_90) + for size in Photo.thumb_sizes: + if size[1]: + self._thumbnail( + mirror, original_path, thumb_path, size[0], size[1]) + try: + os.unlink(tfn) + except: + pass + + def _video_transcode(self, transcode_path, original_path): + transcode_path = os.path.join(transcode_path, video_cache(self._path)) + # get number of cores on the system, and use all minus one + num_of_cores = os.sysconf('SC_NPROCESSORS_ONLN') - 1 + transcode_cmd = [ + '-i', original_path, # original file to be encoded + '-c:v', 'libx264', # set h264 as videocodec + # set specific preset that provides a certain encoding speed to + # compression ratio + '-preset', 'slow', + '-profile:v', 'baseline', # set output to specific h264 profile + '-level', '3.0', # sets highest compatibility with target devices + '-crf', '20', # set quality + '-b:v', '4M', # set videobitrate to 4Mbps + '-strict', 'experimental', # allow native aac codec below + '-c:a', 'aac', # set aac as audiocodec + '-ac', '2', # force two audiochannels + '-ab', '160k', # set audiobitrate to 160Kbps + # limits max rate, will degrade CRF if needed + '-maxrate', '10000000', + '-bufsize', '10000000', # define how much the client should buffer + '-f', 'mp4', # fileformat mp4 + '-threads', str(num_of_cores), # number of cores (all minus one) + '-loglevel', 'quiet', # don't display anything + '-y' # don't prompt for overwrite + ] + filters = [] + info_string = "%s -> mp4, h264" % (os.path.basename(original_path)) + message("transcoding", info_string) + if (os.path.exists(transcode_path) and + file_mtime( + transcode_path) >= self._attributes["dateTimeFile"]): + self._video_metadata(transcode_path, False) + return + if "originalSize" in self._attributes: + width = self._attributes["originalSize"][0] + height = self._attributes["originalSize"][1] + if width > height: + # horisontal orientation + if height > 720: + transcode_cmd.append('-vf') + transcode_cmd.append('scale=-1:720') + elif (height > width) or (width == height): + # vertical orientation, or equal sides + if width > 720: + transcode_cmd.append('-vf') + transcode_cmd.append('scale=720:-1') + if "rotate" in self._attributes: + if self._attributes["rotate"] == "90": + filters.append('transpose=1') + elif self._attributes["rotate"] == "180": + filters.append('vflip,hflip') + elif self._attributes["rotate"] == "270": + filters.append('transpose=2') + if len(filters): + transcode_cmd.append('-vf') + transcode_cmd.append(','.join(filters)) + + tmp_transcode_cmd = transcode_cmd[:] + transcode_cmd.append(transcode_path) + p = VideoTranscodeWrapper().call(*transcode_cmd) + if p is False: + # add another option, try transcoding again + # done to avoid this error; + # x264 [error]: baseline profile doesn't support 4:2:2 + message( + "transcoding failure, trying yuv420p", + os.path.basename(original_path)) + tmp_transcode_cmd.append('-pix_fmt') + tmp_transcode_cmd.append('yuv420p') + tmp_transcode_cmd.append(transcode_path) + p = VideoTranscodeWrapper().call(*tmp_transcode_cmd) + + if p is False: + message("transcoding failure", os.path.basename(original_path)) + try: + os.unlink(transcode_path) + except: + pass + self.is_valid = False + return + self._video_metadata(transcode_path, False) + + @property + def name(self): + return os.path.basename(self._path) + + def __str__(self): + return self.name + + @property + def path(self): + return self._path + + @property + def image_caches(self): + caches = [] + if ("mediaType" in self._attributes and + self._attributes["mediaType"] == "video"): + for size in Photo.thumb_sizes: + if size[1]: + caches.append(image_cache(self._path, size[0], size[1])) + caches.append(video_cache(self._path)) + else: + caches = [ + image_cache(self._path, size[0], size[1]) + for size in Photo.thumb_sizes + ] + return caches + + @property + def date(self): + correct_date = None + if not self.is_valid: + correct_date = datetime(1900, 1, 1) + if "dateTimeVideo" in self._attributes: + correct_date = self._attributes["dateTimeVideo"] + elif "dateTimeOriginal" in self._attributes: + correct_date = self._attributes["dateTimeOriginal"] + elif "dateTime" in self._attributes: + correct_date = self._attributes["dateTime"] + else: + correct_date = self._attributes["dateTimeFile"] + return correct_date + + def __cmp__(self, other): + date_compare = cmp(self.date, other.date) + + if date_compare == 0: + return cmp(self.name, other.name) + return date_compare + + @property + def attributes(self): + return self._attributes + + @staticmethod + def from_dict(dictionary, basepath): + del dictionary["date"] + path = os.path.join(basepath, dictionary["name"]) + del dictionary["name"] + for key, value in dictionary.items(): + if key.startswith("dateTime"): + try: + dictionary[key] = datetime.strptime( + dictionary[key], + "%a %b %d %H:%M:%S %Y") + except KeyboardInterrupt: + raise + except: + pass + return Photo(path, None, dictionary) + + def to_dict(self): + photo = {"name": self.name, "date": self.date} + photo.update(self.attributes) + return photo - def __cmp__(self, other): - date_compare = cmp(self.date, other.date) - - if date_compare == 0: - return cmp(self.name, other.name) - return date_compare - - @property - def attributes(self): - return self._attributes - @staticmethod - def from_dict(dictionary, basepath): - del dictionary["date"] - path = os.path.join(basepath, dictionary["name"]) - del dictionary["name"] - for key, value in dictionary.items(): - if key.startswith("dateTime"): - try: - dictionary[key] = datetime.strptime(dictionary[key], "%a %b %d %H:%M:%S %Y") - except KeyboardInterrupt: - raise - except: - pass - return Photo(path, None, dictionary) - def to_dict(self): - photo = { "name": self.name, "date": self.date } - photo.update(self.attributes) - return photo class PhotoAlbumEncoder(json.JSONEncoder): - def default(self, obj): - if isinstance(obj, datetime): - return obj.strftime("%a %b %d %H:%M:%S %Y") - if isinstance(obj, Album) or isinstance(obj, Photo): - return obj.to_dict() - return json.JSONEncoder.default(self, obj) - + def default(self, obj): + if isinstance(obj, datetime): + return obj.strftime("%a %b %d %H:%M:%S %Y") + if isinstance(obj, Album) or isinstance(obj, Photo): + return obj.to_dict() + return json.JSONEncoder.default(self, obj) diff --git a/scanner/TreeWalker.py b/scanner/TreeWalker.py index 51bc2ff..08c52ec 100644 --- a/scanner/TreeWalker.py +++ b/scanner/TreeWalker.py @@ -6,129 +6,143 @@ from PhotoAlbum import Photo, Album, PhotoAlbumEncoder from CachePath import * import json -class TreeWalker: - def __init__(self, album_path, cache_path): - self.album_path = os.path.abspath(album_path).decode(sys.getfilesystemencoding()) - self.cache_path = os.path.abspath(cache_path).decode(sys.getfilesystemencoding()) - set_cache_path_base(self.album_path) - self.all_albums = list() - self.all_photos = list() - self.walk(self.album_path) - self.big_lists() - self.remove_stale() - message("complete", "") - def walk(self, path): - next_level() - if not os.access(path, os.R_OK | os.X_OK): - message("access denied", os.path.basename(path)) - back_level() - return None - message("walking", os.path.basename(path)) - cache = os.path.join(self.cache_path, json_cache(path)) - cached = False - cached_album = None - if os.path.exists(cache): - try: - cached_album = Album.from_cache(cache) - if file_mtime(path) <= file_mtime(cache): - message("full cache", os.path.basename(path)) - cached = True - album = cached_album - for photo in album.photos: - self.all_photos.append(photo) - else: - message("partial cache", os.path.basename(path)) - except KeyboardInterrupt: - raise - except: - message("corrupt cache", os.path.basename(path)) - cached_album = None - if not cached: - album = Album(path) - for entry in os.listdir(path): - if entry[0] == '.': - continue - try: - entry = entry.decode(sys.getfilesystemencoding()) - except KeyboardInterrupt: - raise - except: - next_level() - message("unicode error", entry.decode(sys.getfilesystemencoding(), "replace")) - back_level() - continue - entry = os.path.join(path, entry) - if os.path.isdir(entry): - next_walked_album = self.walk(entry) - if next_walked_album is not None: - album.add_album(next_walked_album) - elif not cached and os.path.isfile(entry): - next_level() - cache_hit = False - if cached_album: - cached_photo = cached_album.photo_from_path(entry) - if cached_photo and file_mtime(entry) <= cached_photo.attributes["dateTimeFile"]: - cache_file = None - if "mediaType" in cached_photo.attributes: - if cached_photo.attributes["mediaType"] == "video": - # if video - cache_file = os.path.join(self.cache_path, video_cache(entry)) - else: - # if image - cache_file = os.path.join(self.cache_path, image_cache(entry, 1024, False)) - else: - # if image - cache_file = os.path.join(self.cache_path, image_cache(entry, 1024, False)) - # at this point we have full path to cache image/video - # check if it actually exists - if os.path.exists(cache_file): - message("cache hit", os.path.basename(entry)) - cache_hit = True - photo = cached_photo - - if not cache_hit: - message("metainfo", os.path.basename(entry)) - photo = Photo(entry, self.cache_path) - if photo.is_valid: - self.all_photos.append(photo) - album.add_photo(photo) - else: - message("unreadable", os.path.basename(entry)) - back_level() - if not album.empty: - message("caching", os.path.basename(path)) - album.cache(self.cache_path) - self.all_albums.append(album) - else: - message("empty", os.path.basename(path)) - back_level() - return album - def big_lists(self): - photo_list = [] - self.all_photos.sort() - for photo in self.all_photos: - photo_list.append(photo.path) - message("caching", "all photos path list") - fp = open(os.path.join(self.cache_path, "all_photos.json"), 'w') - json.dump(photo_list, fp, cls=PhotoAlbumEncoder) - fp.close() - def remove_stale(self): - message("cleanup", "building stale list") - all_cache_entries = { "all_photos.json": True, "latest_photos.json": True } - for album in self.all_albums: - all_cache_entries[album.cache_path] = True - for photo in self.all_photos: - for entry in photo.image_caches: - all_cache_entries[entry] = True - message("cleanup", "searching for stale cache entries") - for cache in os.listdir(self.cache_path): - try: - cache = cache.decode(sys.getfilesystemencoding()) - except KeyboardInterrupt: - raise - except: - pass - if cache not in all_cache_entries: - message("cleanup", os.path.basename(cache)) - os.unlink(os.path.join(self.cache_path, cache)) +class TreeWalker: + def __init__(self, album_path, cache_path): + self.album_path = os.path.abspath( + album_path).decode(sys.getfilesystemencoding()) + self.cache_path = os.path.abspath( + cache_path).decode(sys.getfilesystemencoding()) + set_cache_path_base(self.album_path) + self.all_albums = list() + self.all_photos = list() + self.walk(self.album_path) + self.big_lists() + self.remove_stale() + message("complete", "") + + def walk(self, path): + next_level() + if not os.access(path, os.R_OK | os.X_OK): + message("access denied", os.path.basename(path)) + back_level() + return None + message("walking", os.path.basename(path)) + cache = os.path.join(self.cache_path, json_cache(path)) + cached = False + cached_album = None + if os.path.exists(cache): + try: + cached_album = Album.from_cache(cache) + if file_mtime(path) <= file_mtime(cache): + message("full cache", os.path.basename(path)) + cached = True + album = cached_album + for photo in album.photos: + self.all_photos.append(photo) + else: + message("partial cache", os.path.basename(path)) + except KeyboardInterrupt: + raise + except: + message("corrupt cache", os.path.basename(path)) + cached_album = None + if not cached: + album = Album(path) + for entry in os.listdir(path): + if entry[0] == '.': + continue + try: + entry = entry.decode(sys.getfilesystemencoding()) + except KeyboardInterrupt: + raise + except: + next_level() + message("unicode error", entry.decode( + sys.getfilesystemencoding(), "replace")) + back_level() + continue + entry = os.path.join(path, entry) + if os.path.isdir(entry): + next_walked_album = self.walk(entry) + if next_walked_album is not None: + album.add_album(next_walked_album) + elif not cached and os.path.isfile(entry): + next_level() + cache_hit = False + if cached_album: + cached_photo = cached_album.photo_from_path(entry) + if (cached_photo and file_mtime( + entry) <= cached_photo.attributes["dateTimeFile"]): + cache_file = None + if "mediaType" in cached_photo.attributes: + if cached_photo.attributes["mediaType"] == "video": + # if video + cache_file = os.path.join( + self.cache_path, video_cache(entry)) + else: + # if image + cache_file = os.path.join( + self.cache_path, + image_cache(entry, 1024, False)) + else: + # if image + cache_file = os.path.join( + self.cache_path, + image_cache(entry, 1024, False)) + + # at this point we have full path to cache image/video + # check if it actually exists + if os.path.exists(cache_file): + message("cache hit", os.path.basename(entry)) + cache_hit = True + photo = cached_photo + + if not cache_hit: + message("metainfo", os.path.basename(entry)) + photo = Photo(entry, self.cache_path) + if photo.is_valid: + self.all_photos.append(photo) + album.add_photo(photo) + else: + message("unreadable", os.path.basename(entry)) + back_level() + if not album.empty: + message("caching", os.path.basename(path)) + album.cache(self.cache_path) + self.all_albums.append(album) + else: + message("empty", os.path.basename(path)) + back_level() + return album + + def big_lists(self): + photo_list = [] + self.all_photos.sort() + for photo in self.all_photos: + photo_list.append(photo.path) + message("caching", "all photos path list") + fp = open(os.path.join(self.cache_path, "all_photos.json"), 'w') + json.dump(photo_list, fp, cls=PhotoAlbumEncoder) + fp.close() + + def remove_stale(self): + message("cleanup", "building stale list") + all_cache_entries = {"all_photos.json": True, + "latest_photos.json": True} + for album in self.all_albums: + all_cache_entries[album.cache_path] = True + for photo in self.all_photos: + for entry in photo.image_caches: + all_cache_entries[entry] = True + message("cleanup", "searching for stale cache entries") + for cache in os.listdir(self.cache_path): + try: + cache = cache.decode(sys.getfilesystemencoding()) + except KeyboardInterrupt: + raise + except: + pass + if cache not in all_cache_entries: + message("cleanup", os.path.basename(cache)) + os.unlink(os.path.join(self.cache_path, cache)) diff --git a/scanner/VideoToolWrapper.py b/scanner/VideoToolWrapper.py index ef215a3..a1d69c8 100644 --- a/scanner/VideoToolWrapper.py +++ b/scanner/VideoToolWrapper.py @@ -2,46 +2,49 @@ from CachePath import message import os import subprocess -class VideoToolWrapper(object): - def call(self, *args): - path = args[-1] - for tool in self.wrappers: - try: - if self.check_output: - p = subprocess.check_output((tool,) + args) - else: - p = subprocess.call((tool,) + args) - if p > 0: - return False - else: - return "SUCCESS" - except KeyboardInterrupt: - if self.cleanup: - self.remove(path) - raise - except OSError: - continue - except: - if self.cleanup: - self.remove(path) - continue - return p - return False - def remove(self, path): - try: - os.unlink(path) - except: - pass +class VideoToolWrapper(object): + def call(self, *args): + path = args[-1] + for tool in self.wrappers: + try: + if self.check_output: + p = subprocess.check_output((tool,) + args) + else: + p = subprocess.call((tool,) + args) + if p > 0: + return False + else: + return "SUCCESS" + except KeyboardInterrupt: + if self.cleanup: + self.remove(path) + raise + except OSError: + continue + except: + if self.cleanup: + self.remove(path) + continue + return p + return False + + def remove(self, path): + try: + os.unlink(path) + except: + pass + class VideoTranscodeWrapper(VideoToolWrapper): - def __init__(self): - self.wrappers = ['avconv', 'ffmpeg'] - self.check_output = False - self.cleanup = True + def __init__(self): + self.wrappers = ['avconv', 'ffmpeg'] + self.check_output = False + self.cleanup = True + class VideoProbeWrapper(VideoToolWrapper): - def __init__(self): - self.wrappers = ['avprobe', 'ffprobe'] - self.check_output = True - self.cleanup = False + def __init__(self): + self.wrappers = ['avprobe', 'ffprobe'] + self.check_output = True + self.cleanup = False diff --git a/scanner/floatapp/__init__.py b/scanner/floatapp/__init__.py index 6e04b18..d624526 100644 --- a/scanner/floatapp/__init__.py +++ b/scanner/floatapp/__init__.py @@ -3,7 +3,9 @@ from flask_login import LoginManager import os.path app = Flask(__name__) -app.config.from_pyfile(os.path.join(os.path.dirname(os.path.abspath(__file__)), "app.cfg")) +app.config.from_pyfile( + os.path.join(os.path.dirname(os.path.abspath(__file__)), "app.cfg")) + login_manager = LoginManager() import login login_manager.setup_app(app) diff --git a/scanner/floatapp/endpoints.py b/scanner/floatapp/endpoints.py index 01fee88..6b80727 100644 --- a/scanner/floatapp/endpoints.py +++ b/scanner/floatapp/endpoints.py @@ -1,118 +1,160 @@ -from floatapp import app -from floatapp.login import admin_required, login_required, is_authenticated, query_is_photo_user, query_is_admin_user, photo_user, admin_user -from floatapp.jsonp import jsonp -from process import send_process -from TreeWalker import TreeWalker -from flask import Response, abort, json, request, jsonify -from flask_login import login_user, current_user -from random import shuffle import os from mimetypes import guess_type +from random import shuffle + +from flask import Response, abort, json, jsonify, request +from flask_login import current_user, login_user + +from floatapp import app +from floatapp.jsonp import jsonp +from floatapp.login import (admin_required, admin_user, is_authenticated, + login_required, photo_user, query_is_admin_user, + query_is_photo_user) +from process import send_process +from TreeWalker import TreeWalker cwd = os.path.dirname(os.path.abspath(__file__)) + @app.route("/scan") @admin_required def scan_photos(): - global cwd - response = send_process([ "stdbuf", "-oL", os.path.abspath(os.path.join(cwd, "../main.py")), - os.path.abspath(app.config["ALBUM_PATH"]), os.path.abspath(app.config["CACHE_PATH"]) ], - os.path.join(cwd, "scanner.pid")) - response.headers.add("X-Accel-Buffering", "no") - response.cache_control.no_cache = True - return response + global cwd + response = send_process([ + "stdbuf", + "-oL", + os.path.abspath(os.path.join(cwd, "../venv/bin/python")), + os.path.abspath(os.path.join(cwd, "../main.py")), + os.path.abspath(app.config["ALBUM_PATH"]), + os.path.abspath(app.config["CACHE_PATH"]) + ], + os.path.join(cwd, "scanner.pid")) + response.headers.add("X-Accel-Buffering", "no") + response.cache_control.no_cache = True + return response + @app.route("/auth") def login(): - success = False - if current_user.is_authenticated(): - success = True - elif query_is_photo_user(request.form) or query_is_photo_user(request.args): - success = login_user(photo_user, remember=True) - elif query_is_admin_user(request.form) or query_is_admin_user(request.args): - success = login_user(admin_user, remember=True) - if not success: - abort(403) - return "" + success = False + if current_user.is_authenticated: + success = True + elif (query_is_photo_user(request.form) or + query_is_photo_user(request.args)): + success = login_user(photo_user, remember=True) + elif (query_is_admin_user(request.form) or + query_is_admin_user(request.args)): + success = login_user(admin_user, remember=True) + if not success: + abort(403) + return "" + def cache_base(path): - path = path.replace('/', '-').replace(' ', '_').replace('(', '').replace('&', '').replace(',', '').replace(')', '').replace('#', '').replace('[', '').replace(']', '').replace('"', '').replace("'", '').replace('_-_', '-').lower() - while path.find("--") != -1: - path = path.replace("--", "-") - while path.find("__") != -1: - path = path.replace("__", "_") - if len(path) == 0: - path = "root" - return path + path = path.replace( + '/', '-').replace( + ' ', '_').replace( + '(', '').replace( + '&', '').replace( + ',', '').replace( + ')', '').replace( + '#', '').replace( + '[', '').replace( + ']', '').replace( + '"', '').replace( + "'", '').replace( + '_-_', '-').lower() + + while path.find("--") != -1: + path = path.replace("--", "-") + + while path.find("__") != -1: + path = path.replace("__", "_") + + if len(path) == 0: + path = "root" + + return path + + +auth_list = [] + -auth_list = [ ] def read_auth_list(): - global auth_list, cwd - f = open(os.path.join(cwd, "auth.txt"), "r") - paths = [ ] - for path in f: - path = path.strip() - paths.append(path) - paths.append(cache_base(path)) - f.close() - auth_list = paths + global auth_list, cwd + f = open(os.path.join(cwd, "auth.txt"), "r") + paths = [] + for path in f: + path = path.strip() + paths.append(path) + paths.append(cache_base(path)) + f.close() + auth_list = paths + # TODO: Make this run via inotify read_auth_list() + def check_permissions(path): - if not is_authenticated(): - for auth_path in auth_list: - if path.startswith(auth_path): - abort(403) + if not is_authenticated(): + for auth_path in auth_list: + if path.startswith(auth_path): + abort(403) + @app.route("/albums/") def albums(path): - check_permissions(path) - return accel_redirect(app.config["ALBUM_ACCEL"], app.config["ALBUM_PATH"], path) + check_permissions(path) + return accel_redirect( + app.config["ALBUM_ACCEL"], app.config["ALBUM_PATH"], path) + @app.route("/cache/") def cache(path): - check_permissions(path) - return accel_redirect(app.config["CACHE_ACCEL"], app.config["CACHE_PATH"], path) + check_permissions(path) + return accel_redirect( + app.config["CACHE_ACCEL"], app.config["CACHE_PATH"], path) + def accel_redirect(internal, real, relative_name): - real_path = os.path.join(real, relative_name) - internal_path = os.path.join(internal, relative_name) - if not os.path.isfile(real_path): - abort(404) - mimetype = None - types = guess_type(real_path) - if len(types) != 0: - mimetype = types[0] - response = Response(mimetype=mimetype) - response.headers.add("X-Accel-Redirect", internal_path) - response.cache_control.public = True - if mimetype == "application/json": - response.cache_control.max_age = 3600 - else: - response.cache_control.max_age = 29030400 - return response + real_path = os.path.join(real, relative_name) + internal_path = os.path.join(internal, relative_name) + if not os.path.isfile(real_path): + abort(404) + mimetype = None + types = guess_type(real_path) + if len(types) != 0: + mimetype = types[0] + response = Response(mimetype=mimetype) + response.headers.add("X-Accel-Redirect", internal_path) + response.cache_control.public = True + if mimetype == "application/json": + response.cache_control.max_age = 3600 + else: + response.cache_control.max_age = 29030400 + return response + @app.route("/photos") @jsonp def photos(): - f = open(os.path.join(app.config["CACHE_PATH"], "all_photos.json"), "r") - photos = json.load(f) - f.close() - if not is_authenticated(): - def allowed(photo): - for auth_path in auth_list: - if photo.startswith(auth_path): - return False - return True - photos = [photo for photo in photos if allowed(photo)] - count = int(request.args.get("count", len(photos))) - random = request.args.get("random") == "true" - if random: - shuffle(photos) - else: - photos.reverse() - response = jsonify(photos=photos[0:count]) - response.cache_control.no_cache = True - return response + f = open(os.path.join(app.config["CACHE_PATH"], "all_photos.json"), "r") + photos = json.load(f) + f.close() + if not is_authenticated(): + def allowed(photo): + for auth_path in auth_list: + if photo.startswith(auth_path): + return False + return True + photos = [photo for photo in photos if allowed(photo)] + count = int(request.args.get("count", len(photos))) + random = request.args.get("random") == "true" + if random: + shuffle(photos) + else: + photos.reverse() + response = jsonify(photos=photos[0:count]) + response.cache_control.no_cache = True + return response diff --git a/scanner/floatapp/jsonp.py b/scanner/floatapp/jsonp.py index cac8ad1..36563ca 100644 --- a/scanner/floatapp/jsonp.py +++ b/scanner/floatapp/jsonp.py @@ -5,14 +5,16 @@ import re jsonp_validator = re.compile("^[a-zA-Z0-9_\-.]{1,128}$") + def jsonp(f): - """Wraps JSONified output for JSONP""" - @wraps(f) - def decorated_function(*args, **kwargs): - callback = request.args.get('callback', False) - if callback and jsonp_validator.match(callback): - content = str(callback) + '(' + str(f(*args,**kwargs).data) + ')' - return current_app.response_class(content, mimetype='application/javascript') - else: - return f(*args, **kwargs) - return decorated_function + """Wraps JSONified output for JSONP""" + @wraps(f) + def decorated_function(*args, **kwargs): + callback = request.args.get('callback', False) + if callback and jsonp_validator.match(callback): + content = str(callback) + '(' + str(f(*args, **kwargs).data) + ')' + return current_app.response_class( + content, mimetype='application/javascript') + else: + return f(*args, **kwargs) + return decorated_function diff --git a/scanner/floatapp/login.py b/scanner/floatapp/login.py index d68132b..5b8ab2e 100644 --- a/scanner/floatapp/login.py +++ b/scanner/floatapp/login.py @@ -3,51 +3,66 @@ from flask import request, abort from flask_login import current_user, UserMixin from functools import wraps + class User(UserMixin): - def __init__(self, id, admin=False): - self.admin = admin - self.id = id + def __init__(self, id, admin=False): + self.admin = admin + self.id = id photo_user = User("user") admin_user = User("admin", True) + @login_manager.user_loader def load_user(id): - if id == "user": - return photo_user - elif id == "admin": - return admin_user - return None + if id == "user": + return photo_user + elif id == "admin": + return admin_user + return None + @login_manager.unauthorized_handler def unauthorized(): - return abort(403) + return abort(403) + def login_required(fn): - @wraps(fn) - def decorated_view(*args, **kwargs): - if query_is_admin_user(request.args) or query_is_photo_user(request.args) or current_user.is_authenticated(): - return fn(*args, **kwargs) - return app.login_manager.unauthorized() - return decorated_view + @wraps(fn) + def decorated_view(*args, **kwargs): + if (query_is_admin_user(request.args) or + query_is_photo_user(request.args) or + current_user.is_authenticated): + return fn(*args, **kwargs) + return app.login_manager.unauthorized() + return decorated_view + def admin_required(fn): - @wraps(fn) - def decorated_view(*args, **kwargs): - if query_is_admin_user(request.args) or (current_user.is_authenticated() and current_user.admin): - return fn(*args, **kwargs) - return app.login_manager.unauthorized() - return decorated_view + @wraps(fn) + def decorated_view(*args, **kwargs): + if (query_is_admin_user(request.args) or + (current_user.is_authenticated and current_user.admin)): + return fn(*args, **kwargs) + return app.login_manager.unauthorized() + return decorated_view + def query_is_photo_user(query): - username = query.get("username", None) - password = query.get("password", None) - return username == app.config["PHOTO_USERNAME"] and password == app.config["PHOTO_PASSWORD"] + username = query.get("username", None) + password = query.get("password", None) + return username == (app.config["PHOTO_USERNAME"] and + password == app.config["PHOTO_PASSWORD"]) + def query_is_admin_user(query): - username = query.get("username", None) - password = query.get("password", None) - return username == app.config["ADMIN_USERNAME"] and password == app.config["ADMIN_PASSWORD"] + username = query.get("username", None) + password = query.get("password", None) + return username == (app.config["ADMIN_USERNAME"] and + password == app.config["ADMIN_PASSWORD"]) + def is_authenticated(): - return query_is_admin_user(request.args) or query_is_photo_user(request.args) or current_user.is_authenticated() + return (query_is_admin_user(request.args) or + query_is_photo_user(request.args) or + current_user.is_authenticated) diff --git a/scanner/floatapp/process.py b/scanner/floatapp/process.py index 1bd8db2..8f29b7b 100644 --- a/scanner/floatapp/process.py +++ b/scanner/floatapp/process.py @@ -3,50 +3,61 @@ import subprocess import os import sys + class ProcessWrapper(object): - def __init__(self, process, done): - self.process = process - self.done = done - def close(self): - self.done() - if self.process.returncode is not None: - return - self.process.stdout.close() - self.process.terminate() - self.process.wait() - def __iter__(self): - return self - def __del__(self): - self.close() - def next(self): - try: - data = self.process.stdout.readline() - except: - self.close() - raise StopIteration() - if data: - return data - self.close() - raise StopIteration() + def __init__(self, process, done): + self.process = process + self.done = done + + def close(self): + self.done() + if self.process.returncode is not None: + return + self.process.stdout.close() + self.process.terminate() + self.process.wait() + + def __iter__(self): + return self + + def __del__(self): + self.close() + + def next(self): + try: + data = self.process.stdout.readline() + except: + self.close() + raise StopIteration() + if data: + return data + self.close() + raise StopIteration() + def send_process(args, pid_file): - def setup_proc(): - f = open(pid_file, "w") - f.write(str(os.getpid())) - f.close() - os.close(0) - os.dup2(1, 2) - def tear_down_proc(): - try: - os.unlink(pid_file) - except: - pass - if os.path.exists(pid_file): - f = open(pid_file, "r") - pid = f.read() - f.close() - if os.path.exists("/proc/%s/status" % pid): - return Response("Scanner is already running.\n", mimetype="text/plain") - process = subprocess.Popen(args, close_fds=True, stdout=subprocess.PIPE, preexec_fn=setup_proc) - response = ProcessWrapper(process, tear_down_proc) - return Response(response, direct_passthrough=True, mimetype="text/plain") + def setup_proc(): + f = open(pid_file, "w") + f.write(str(os.getpid())) + f.close() + os.close(0) + os.dup2(1, 2) + + def tear_down_proc(): + try: + os.unlink(pid_file) + except: + pass + + if os.path.exists(pid_file): + f = open(pid_file, "r") + pid = f.read() + f.close() + if os.path.exists("/proc/%s/status" % pid): + return Response( + "Scanner is already running.\n", mimetype="text/plain") + + process = subprocess.Popen( + args, close_fds=True, stdout=subprocess.PIPE, preexec_fn=setup_proc) + response = ProcessWrapper(process, tear_down_proc) + return Response(response, direct_passthrough=True, mimetype="text/plain") diff --git a/scanner/main.py b/scanner/main.py index bf9fc02..c235751 100755 --- a/scanner/main.py +++ b/scanner/main.py @@ -5,19 +5,21 @@ from CachePath import message import sys import os -def main(): - reload(sys) - sys.setdefaultencoding("UTF-8") - if len(sys.argv) != 3: - print "usage: %s ALBUM_PATH CACHE_PATH" % sys.argv[0] - return - try: - os.umask(022) - TreeWalker(sys.argv[1], sys.argv[2]) - except KeyboardInterrupt: - message("keyboard", "CTRL+C pressed, quitting.") - sys.exit(-97) - +def main(): + reload(sys) + sys.setdefaultencoding("UTF-8") + + if len(sys.argv) != 3: + print "usage: %s ALBUM_PATH CACHE_PATH" % sys.argv[0] + return + + try: + os.umask(022) + TreeWalker(sys.argv[1], sys.argv[2]) + except KeyboardInterrupt: + message("keyboard", "CTRL+C pressed, quitting.") + sys.exit(-97) + if __name__ == "__main__": - main() + main()