As part of my Web gallery solution, I wrote some logic that creates a YYYY/MM directory creator that places symlinks to my original files within the new directory structure. The links are added to the relevant directory based on exif metadata if possible, and file timestamp as a last resort.
Check out the source files at my gitlab space:
Here is my main sample invocation. I used this a lot while testing.
./generate.py -i /mnt/bgstack15/Backups/Images/Photos/camera/2018/ -o /mnt/public/www/gallery/my2018 -n -d2 -x 'October' -s --nr
The help text is generated with ArgumentParser.
$ ./generate.py --helpusage: generate.py [-h] [-d [{0,1,2,3,4,5,6,7,8,9,10}]] [-v] [-n | -a] [-z | --nz] [-c | -s | -m] [-r | --nr] -i
INDIR -o OUTDIR [-x EXCLUDE]
Make symlink forest for images
optional arguments:
-h, --help show this help message and exit
-d [{0,1,2,3,4,5,6,7,8,9,10}], --debug [{0,1,2,3,4,5,6,7,8,9,10}]
Set debug level
-v, --version show program's version number and exit
-n, --dryrun Make no changes (default)
-a, --apply Actually make changes
-z, -0, --zeropad Zero pad month directories (default)
--nz, --nozeropad, --no-zeropad
Do not zero pad
-c, --copy Copy files instead of symlinks. Not recommended.
-s, --symlink Make symlinks (default)
-m, --move Move files. Not recommended.
-r, --relative Make relative symlinks if possible.
--nr, --norelative, --no-relative
Make absolute symlinks (default)
-i INDIR, --indir INDIR
-o OUTDIR, --outdir OUTDIR
-x EXCLUDE, --exclude EXCLUDE
Exclude pathname matches. Can be used multiple times.
For posterity and in case my gitlab ever dies, here are their contents at the time of this writing.
generate.py
#!/usr/bin/env python3
# Startdate: 2021-01-29 15:35
# Dependencies:
# devuan-req: python3-exifread
# Reference:
# https://stackoverflow.com/questions/1192978/python-get-relative-path-of-all-files-and-subfolders-in-a-directory
# https://stackoverflow.com/questions/3207219/how-do-i-list-all-files-of-a-directory
# https://stackoverflow.com/questions/600268/mkdir-p-functionality-in-python
from sys import stderr
from exifread import process_file
from os import walk, path, stat, symlink
from pathlib import Path
from datetime import date
from shutil import copy2, move
try:
if debuglevel is None:
debuglevel = 0
except:
debuglevel = 0
# Functions
def eprint(*args, **kwargs):
print(*args, file=stderr, **kwargs)
def list_files(indir,relative=False,debuglevel=debuglevel, excludes=None):
x = 0
f = []
if debuglevel >= 9:
eprint(f"Listing all files underneath {indir}")
if len(excludes) > 0:
eprint("While excluding path matches:",excludes)
for (dirpath, dirnames, filenames) in walk(indir):
for file in filenames:
x += 1
fullpath=path.join(dirpath,file)
relpath=path.join(path.relpath(dirpath,indir),file)
use = True
use_pattern = ""
for e in excludes:
# simple match, no regex
if e in relpath:
use = False
use_pattern = e
break # short-circuit
if use:
if debuglevel >= 9: eprint(x,fullpath,relpath)
f.append(relpath if relative else fullpath)
else:
if debuglevel >= 9: eprint(x,fullpath,f"ignored per {use_pattern}")
return f
def limit(listf=None,filters=["image","video"]):
# available types are "image", "video"
newlist = []
for f in listf:
_, ext = path.splitext(f)
ext = ext[1:].lower()
if ext in ["jpg","jpeg","png","svg","tif","tiff","gif"] and "image" in filters:
newlist.append(f)
elif ext in ["mp4","webm","avi"] and "video" in filters:
newlist.append(f)
return newlist
def get_file_YM(tf,debuglevel=debuglevel,zero_pad=False):
Y = 0 ; M = 0
#eprint(f"Debuglevel {debuglevel}")
try:
# read exif data
with open(tf,'rb') as f:
tags = process_file(f, details=False, stop_tag="Image DateTime")
if "Image DateTime" in tags:
#if debuglevel >= 5: eprint(tf,tags["Image DateTime"])
Y = str(tags["Image DateTime"]).split(' ')[0].split(':')
if Y == "['']" or Y == ['']:
Y = ['0','0']
M = Y[1] ; Y = Y[0];
# Any other image timestamps could be used here
#eprint(tags)
except KeyboardInterrupt:
return -1, -1
except:
if debuglevel >= 1: eprint(f"Unable to extract any exif data for {tf}")
if int(Y) == 0 or int(M) == 0:
# need to just use timestamp
try:
ts = date.fromtimestamp(stat(tf).st_mtime)
except KeyboardInterrupt:
return -1, -1
except:
# no timestamp available from filesystem?
if debuglevel >= 1: eprint(f"No timestamp available for {tf}")
if int(Y) == 0: Y = ts.year
if int(M) == 0: M = ts.month
if zero_pad:
M = str(M).zfill(2)
else:
M = str(int(M))
if debuglevel >= 3:
eprint(f"{tf} {Y}/{M}")
return Y, M
def make_YM_forest(outdir, flist, action = "symlink", dryrun=True, debuglevel=debuglevel, zero_pad=False, relative_symlinks = False):
"""
For each file in flist, [action] it to outdir, sorting into YYYY/MM subdirs based on exif metadata or timestamp.
Action may be one of ['symlink', 'copy', 'move'].
If relative_symlink = True, then make relative symlinks if possible.
"""
result = 0
stop = False
# validate input
if action not in ['symlink', 'copy', 'move']:
eprint("make_YM_forest action may be one of ['symlink', 'copy', 'move'].")
return -1
# Learn all directories to make
# and also cache the files with Y/M value
fdict = {}
destdirs = []
for f in flist:
try:
Y, M = get_file_YM(f,debuglevel=debuglevel,zero_pad=zero_pad)
if Y == -1 and M == -1:
#eprint("Stopping due to keyboard variant 2.")
stop = True
break
except KeyboardInterrupt:
#eprint("Stopping due to keyboard variant 1.")
stop = True
break
if zero_pad:
M = str(M).zfill(2)
else:
M = str(int(M))
YM = f"{Y}/{M}"
if YM not in destdirs: destdirs.append(YM)
fdict[f] = YM
# finish the for f in flist
# short-circuit if keyboard action happened in one of the nested functions
if stop:
return -1
# Make directories
for d in destdirs:
dd = path.join(outdir,d)
if debuglevel >= 3:
eprint(f"Make dir: {dd}")
if not dryrun:
Path(dd).mkdir(parents=True, exist_ok=True)
# Make symlinks
for f in fdict:
ff = f
basename = path.basename(ff)
destfile = path.join(outdir,fdict[f],basename)
if action == "symlink" and relative_symlinks:
ff = path.relpath(ff,destfile)
if debuglevel >= 2:
if action == "copy":
eprint(f"cp -p {ff} {destfile}")
elif action == "symlink":
eprint(f"ln -s {ff} {destfile}")
elif action == "move":
eprint(f"mv {ff} {destfile}")
if not dryrun:
#if False:
if action == "copy":
copy2(ff,destfile)
elif action == "symlink":
symlink(ff,destfile)
elif action == "move":
move(ff,destfile)
return result
genlib.py
#!/usr/bin/env python3
# Startdate: 2021-01-29 15:35
# Dependencies:
# devuan-req: python3-exifread
# Reference:
# https://stackoverflow.com/questions/1192978/python-get-relative-path-of-all-files-and-subfolders-in-a-directory
# https://stackoverflow.com/questions/3207219/how-do-i-list-all-files-of-a-directory
# https://stackoverflow.com/questions/600268/mkdir-p-functionality-in-python
from sys import stderr
from exifread import process_file
from os import walk, path, stat, symlink
from pathlib import Path
from datetime import date
from shutil import copy2, move
try:
if debuglevel is None:
debuglevel = 0
except:
debuglevel = 0
# Functions
def eprint(*args, **kwargs):
print(*args, file=stderr, **kwargs)
def list_files(indir,relative=False,debuglevel=debuglevel, excludes=None):
x = 0
f = []
if debuglevel >= 9:
eprint(f"Listing all files underneath {indir}")
if len(excludes) > 0:
eprint("While excluding path matches:",excludes)
for (dirpath, dirnames, filenames) in walk(indir):
for file in filenames:
x += 1
fullpath=path.join(dirpath,file)
relpath=path.join(path.relpath(dirpath,indir),file)
use = True
use_pattern = ""
for e in excludes:
# simple match, no regex
if e in relpath:
use = False
use_pattern = e
break # short-circuit
if use:
if debuglevel >= 9: eprint(x,fullpath,relpath)
f.append(relpath if relative else fullpath)
else:
if debuglevel >= 9: eprint(x,fullpath,f"ignored per {use_pattern}")
return f
def limit(listf=None,filters=["image","video"]):
# available types are "image", "video"
newlist = []
for f in listf:
_, ext = path.splitext(f)
ext = ext[1:].lower()
if ext in ["jpg","jpeg","png","svg","tif","tiff","gif"] and "image" in filters:
newlist.append(f)
elif ext in ["mp4","webm","avi"] and "video" in filters:
newlist.append(f)
return newlist
def get_file_YM(tf,debuglevel=debuglevel,zero_pad=False):
Y = 0 ; M = 0
#eprint(f"Debuglevel {debuglevel}")
try:
# read exif data
with open(tf,'rb') as f:
tags = process_file(f, details=False, stop_tag="Image DateTime")
if "Image DateTime" in tags:
#if debuglevel >= 5: eprint(tf,tags["Image DateTime"])
Y = str(tags["Image DateTime"]).split(' ')[0].split(':')
if Y == "['']" or Y == ['']:
Y = ['0','0']
M = Y[1] ; Y = Y[0];
# Any other image timestamps could be used here
#eprint(tags)
except KeyboardInterrupt:
return -1, -1
except:
if debuglevel >= 1: eprint(f"Unable to extract any exif data for {tf}")
if int(Y) == 0 or int(M) == 0:
# need to just use timestamp
try:
ts = date.fromtimestamp(stat(tf).st_mtime)
except KeyboardInterrupt:
return -1, -1
except:
# no timestamp available from filesystem?
if debuglevel >= 1: eprint(f"No timestamp available for {tf}")
if int(Y) == 0: Y = ts.year
if int(M) == 0: M = ts.month
if zero_pad:
M = str(M).zfill(2)
else:
M = str(int(M))
if debuglevel >= 3:
eprint(f"{tf} {Y}/{M}")
return Y, M
def make_YM_forest(outdir, flist, action = "symlink", dryrun=True, debuglevel=debuglevel, zero_pad=False, relative_symlinks = False):
"""
For each file in flist, [action] it to outdir, sorting into YYYY/MM subdirs based on exif metadata or timestamp.
Action may be one of ['symlink', 'copy', 'move'].
If relative_symlink = True, then make relative symlinks if possible.
"""
result = 0
stop = False
# validate input
if action not in ['symlink', 'copy', 'move']:
eprint("make_YM_forest action may be one of ['symlink', 'copy', 'move'].")
return -1
# Learn all directories to make
# and also cache the files with Y/M value
fdict = {}
destdirs = []
for f in flist:
try:
Y, M = get_file_YM(f,debuglevel=debuglevel,zero_pad=zero_pad)
if Y == -1 and M == -1:
#eprint("Stopping due to keyboard variant 2.")
stop = True
break
except KeyboardInterrupt:
#eprint("Stopping due to keyboard variant 1.")
stop = True
break
if zero_pad:
M = str(M).zfill(2)
else:
M = str(int(M))
YM = f"{Y}/{M}"
if YM not in destdirs: destdirs.append(YM)
fdict[f] = YM
# finish the for f in flist
# short-circuit if keyboard action happened in one of the nested functions
if stop:
return -1
# Make directories
for d in destdirs:
dd = path.join(outdir,d)
if debuglevel >= 3:
eprint(f"Make dir: {dd}")
if not dryrun:
Path(dd).mkdir(parents=True, exist_ok=True)
# Make symlinks
for f in fdict:
ff = f
basename = path.basename(ff)
destfile = path.join(outdir,fdict[f],basename)
if action == "symlink" and relative_symlinks:
ff = path.relpath(ff,destfile)
if debuglevel >= 2:
if action == "copy":
eprint(f"cp -p {ff} {destfile}")
elif action == "symlink":
eprint(f"ln -s {ff} {destfile}")
elif action == "move":
eprint(f"mv {ff} {destfile}")
if not dryrun:
#if False:
if action == "copy":
copy2(ff,destfile)
elif action == "symlink":
symlink(ff,destfile)
elif action == "move":
move(ff,destfile)
return result
