Cleanup

generator/engraving/__init__.py

@@ -10,7 +10,7 @@ import shutil
 from PIL import Image, ImageChops
 
 from generator.engraving.fonts import Fonts
-from generator.engraving.cache import renderer_fingerprint, restore_pages, score_fingerprint, store_pages
+from generator.engraving.cache import renderer_fingerprint, renderer_namespace, restore_pages, score_fingerprint, store_pages
 from generator.engraving.layout import layout_score
 from generator.engraving.raster import rasterize_page
 from generator.engraving.types import EngravingError, Style
@@ -35,12 +35,11 @@ def check_page(path: Path) -> None:
     if bounds[0] < 4 or bounds[1] < 4 or bounds[2] > PAGE_WIDTH - 4 or bounds[3] > PAGE_HEIGHT - 4:
       raise EngravingError(f'{path.name}: notation reaches the page edge; refusing potentially clipped output')
 
-def engrave_scores(paths: list[Path], root: Path, fonts: Path, cache_dir: Path | None, progress: bool, worker: int) -> tuple[int, int]:
+def engrave_scores(paths: list[Path], root: Path, fonts: Path, cache_dir: Path | None, progress: bool, worker: int, renderer: str) -> tuple[int, int, int, int]:
   family = Fonts(fonts)
   style = Style()
   fingerprint = family.fingerprint(style)
-  renderer = renderer_fingerprint(family, style) if cache_dir else ''
-  reused = 0
+  reused, missing, rejected = 0, 0, 0
   for index, path in enumerate(paths, 1):
     document = json.loads(path.read_text(encoding='utf-8'))
     try:
@@ -49,8 +48,12 @@ def engrave_scores(paths: list[Path], root: Path, fonts: Path, cache_dir: Path |
       if destination.exists():
         shutil.rmtree(destination)
       entry = cache_dir / score_fingerprint(document, renderer) if cache_dir else None
+      present = entry is not None and (entry.exists() or entry.is_symlink())
       pages = restore_pages(entry, destination) if entry else None
       if pages is None:
+        if entry:
+          rejected += int(present)
+          missing += int(not present)
         if destination.exists():
           shutil.rmtree(destination)
         destination.mkdir()
@@ -73,10 +76,10 @@ def engrave_scores(paths: list[Path], root: Path, fonts: Path, cache_dir: Path |
         bundle['playback'] = document['playback']
       path.write_text(json.dumps(bundle, indent=2, ensure_ascii=False) + '\n', encoding='utf-8')
       if progress and (index % 50 == 0 or index == len(paths)):
-        print(f'Engraving worker {worker}: {index}/{len(paths)} scores, {reused} reused', flush=True)
+        print(f'Engraving worker {worker}: {index}/{len(paths)} scores, {reused} reused, {missing} missing, {rejected} rejected', flush=True)
     except ValueError as error:
       raise EngravingError(f'{path.relative_to(root)}: {error}') from error
-  return len(paths), reused
+  return len(paths), reused, missing, rejected
 
 def engrave_tree(root: Path, fonts: Path, *, cache_dir: Path | None = None, jobs: int = 1, progress: bool = False) -> None:
   if type(jobs) is not int or not 1 <= jobs <= 16:
@@ -86,12 +89,21 @@ def engrave_tree(root: Path, fonts: Path, *, cache_dir: Path | None = None, jobs
   paths = sorted(root.rglob('*.json'))
   if not paths:
     return
+  renderer = renderer_fingerprint(Fonts(fonts), Style()) if cache_dir else ''
+  if cache_dir:
+    cache_dir = renderer_namespace(cache_dir.resolve(), renderer)
+    if progress:
+      state = 'warm namespace' if cache_dir.is_dir() else 'cold namespace'
+      print(f'Engraving cache enabled: {cache_dir} ({state}; renderer {renderer})', flush=True)
+  elif progress:
+    print('Engraving cache disabled', flush=True)
   jobs = min(jobs, len(paths))
   if jobs == 1:
-    results = [engrave_scores(paths, root, fonts, cache_dir, progress, 1)]
+    results = [engrave_scores(paths, root, fonts, cache_dir, progress, 1, renderer)]
   else:
     with ProcessPoolExecutor(max_workers=jobs, mp_context=multiprocessing.get_context('spawn')) as pool:
-      futures = [pool.submit(engrave_scores, paths[index::jobs], root, fonts, cache_dir, progress, index + 1) for index in range(jobs)]
+      futures = [pool.submit(engrave_scores, paths[index::jobs], root, fonts, cache_dir, progress, index + 1, renderer) for index in range(jobs)]
       results = [future.result() for future in futures]
   if progress:
-    print(f'Engraving complete: {sum(total for total, _ in results)} scores, {sum(reused for _, reused in results)} reused', flush=True)
+    total, reused, missing, rejected = (sum(row[index] for row in results) for index in range(4))
+    print(f'Engraving complete: {total} scores, {reused} reused, {missing} missing, {rejected} rejected', flush=True)

generator/engraving/cache.py

@@ -5,20 +5,27 @@ import importlib.metadata
 import json
 from pathlib import Path
 import shutil
-import sys
+import platform
 import tempfile
 
 from PIL import features
 
-def renderer_fingerprint(fonts, style) -> str:
+def renderer_identity(fonts, style) -> dict:
   root = Path(__file__).resolve().parents[1]
   versions = {name: importlib.metadata.version(name) for name in ('Pillow', 'fonttools', 'aggdraw')}
   versions.update({name: features.version(name) for name in features.get_supported()})
-  digest = hashlib.sha256(json.dumps([fonts.fingerprint(style), sys.version, versions], sort_keys=True).encode())
-  for path in sorted([*root.joinpath('engraving').glob('*.py'), root / 'model.py', root / 'harmony.py', root / 'requirements.txt']):
-    digest.update(path.relative_to(root).as_posix().encode())
-    digest.update(path.read_bytes())
-  return digest.hexdigest()
+  sources = [*root.joinpath('engraving').glob('*.py'), root / 'model.py', root / 'harmony.py', root / 'requirements.txt']
+  return {'fonts_style': fonts.fingerprint(style), 'python': {'implementation': platform.python_implementation(), 'version': platform.python_version()}, 'dependencies': versions,
+    'sources': {path.relative_to(root).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() for path in sorted(sources)}}
+
+def renderer_fingerprint(fonts, style) -> str:
+  return hashlib.sha256(json.dumps(renderer_identity(fonts, style), sort_keys=True).encode()).hexdigest()
+
+def renderer_namespace(cache: Path, renderer: str) -> Path:
+  namespace = cache / 'v2' / renderer
+  if namespace.parent.is_symlink() or namespace.is_symlink():
+    raise ValueError('engraving cache namespaces must not be symlinks')
+  return namespace
 
 def score_fingerprint(document: dict, renderer: str) -> str:
   music = {key: value for key, value in document.items() if key != 'playback'}