Cleanup

generator/__main__.py

@@ -11,7 +11,8 @@ from generator.practice.generate import generate_practice_scores
 from generator.score import count_json_files, validate_generated_tree
 from generator.summary import validate_expected_performance_summary, validate_practice_parts
 from generator.parts import source_paths
-from generator.engraving import engrave_tree
+from generator.engraving import Fonts, Style, engrave_tree
+from generator.engraving.cache import renderer_fingerprint
 from generator.playback import compile_tree
 from generator.catalog import build_catalog
 
@@ -26,14 +27,18 @@ def copy_runtime_fonts(static_assets_dir: Path, output_dir: Path) -> None:
 
 def parse_args() -> argparse.Namespace:
   parser = argparse.ArgumentParser(description="Generate Product Vamp package score assets.")
-  parser.add_argument("--data-dir", type=Path, required=True)
-  parser.add_argument("--output-dir", type=Path, required=True)
+  parser.add_argument("--data-dir", type=Path)
+  parser.add_argument("--output-dir", type=Path)
+  parser.add_argument("--renderer-fingerprint", action="store_true", help="Print the engraving cache identity without generating assets.")
   parser.add_argument("--static-assets-dir", type=Path, default=Path("package/assets"))
   parser.add_argument("--check", action="store_true")
   parser.add_argument("--engraving-cache", type=Path)
   parser.add_argument("--jobs", type=int, default=1, choices=range(1, 17), metavar="1-16")
   parser.add_argument("--progress", action="store_true")
-  return parser.parse_args()
+  args = parser.parse_args()
+  if not args.renderer_fingerprint and (args.data_dir is None or args.output_dir is None):
+    parser.error('--data-dir and --output-dir are required for generation')
+  return args
 
 def generate_assets(args: argparse.Namespace) -> None:
   data_dir = args.data_dir.resolve()
@@ -92,7 +97,11 @@ def generate_assets(args: argparse.Namespace) -> None:
       print('Generator check passed.')
 
 def main() -> int:
-  generate_assets(parse_args())
+  args = parse_args()
+  if args.renderer_fingerprint:
+    print(renderer_fingerprint(Fonts(args.static_assets_dir / 'fonts' / 'petaluma'), Style()))
+  else:
+    generate_assets(args)
   return 0
 
 if __name__ == "__main__":

generator/engraving/__init__.py

@@ -10,7 +10,7 @@ import shutil
 from PIL import Image, ImageChops
 
 from generator.engraving.fonts import Fonts
-from generator.engraving.cache import renderer_fingerprint, restore_pages, score_fingerprint, store_pages
+from generator.engraving.cache import renderer_fingerprint, renderer_namespace, restore_pages, score_fingerprint, store_pages
 from generator.engraving.layout import layout_score
 from generator.engraving.raster import rasterize_page
 from generator.engraving.types import EngravingError, Style
@@ -35,12 +35,11 @@ def check_page(path: Path) -> None:
     if bounds[0] < 4 or bounds[1] < 4 or bounds[2] > PAGE_WIDTH - 4 or bounds[3] > PAGE_HEIGHT - 4:
       raise EngravingError(f'{path.name}: notation reaches the page edge; refusing potentially clipped output')
 
-def engrave_scores(paths: list[Path], root: Path, fonts: Path, cache_dir: Path | None, progress: bool, worker: int) -> tuple[int, int]:
+def engrave_scores(paths: list[Path], root: Path, fonts: Path, cache_dir: Path | None, progress: bool, worker: int, renderer: str) -> tuple[int, int, int, int]:
   family = Fonts(fonts)
   style = Style()
   fingerprint = family.fingerprint(style)
-  renderer = renderer_fingerprint(family, style) if cache_dir else ''
-  reused = 0
+  reused, missing, rejected = 0, 0, 0
   for index, path in enumerate(paths, 1):
     document = json.loads(path.read_text(encoding='utf-8'))
     try:
@@ -49,8 +48,12 @@ def engrave_scores(paths: list[Path], root: Path, fonts: Path, cache_dir: Path |
       if destination.exists():
         shutil.rmtree(destination)
       entry = cache_dir / score_fingerprint(document, renderer) if cache_dir else None
+      present = entry is not None and (entry.exists() or entry.is_symlink())
       pages = restore_pages(entry, destination) if entry else None
       if pages is None:
+        if entry:
+          rejected += int(present)
+          missing += int(not present)
         if destination.exists():
           shutil.rmtree(destination)
         destination.mkdir()
@@ -73,10 +76,10 @@ def engrave_scores(paths: list[Path], root: Path, fonts: Path, cache_dir: Path |
         bundle['playback'] = document['playback']
       path.write_text(json.dumps(bundle, indent=2, ensure_ascii=False) + '\n', encoding='utf-8')
       if progress and (index % 50 == 0 or index == len(paths)):
-        print(f'Engraving worker {worker}: {index}/{len(paths)} scores, {reused} reused', flush=True)
+        print(f'Engraving worker {worker}: {index}/{len(paths)} scores, {reused} reused, {missing} missing, {rejected} rejected', flush=True)
     except ValueError as error:
       raise EngravingError(f'{path.relative_to(root)}: {error}') from error
-  return len(paths), reused
+  return len(paths), reused, missing, rejected
 
 def engrave_tree(root: Path, fonts: Path, *, cache_dir: Path | None = None, jobs: int = 1, progress: bool = False) -> None:
   if type(jobs) is not int or not 1 <= jobs <= 16:
@@ -86,12 +89,21 @@ def engrave_tree(root: Path, fonts: Path, *, cache_dir: Path | None = None, jobs
   paths = sorted(root.rglob('*.json'))
   if not paths:
     return
+  renderer = renderer_fingerprint(Fonts(fonts), Style()) if cache_dir else ''
+  if cache_dir:
+    cache_dir = renderer_namespace(cache_dir.resolve(), renderer)
+    if progress:
+      state = 'warm namespace' if cache_dir.is_dir() else 'cold namespace'
+      print(f'Engraving cache enabled: {cache_dir} ({state}; renderer {renderer})', flush=True)
+  elif progress:
+    print('Engraving cache disabled', flush=True)
   jobs = min(jobs, len(paths))
   if jobs == 1:
-    results = [engrave_scores(paths, root, fonts, cache_dir, progress, 1)]
+    results = [engrave_scores(paths, root, fonts, cache_dir, progress, 1, renderer)]
   else:
     with ProcessPoolExecutor(max_workers=jobs, mp_context=multiprocessing.get_context('spawn')) as pool:
-      futures = [pool.submit(engrave_scores, paths[index::jobs], root, fonts, cache_dir, progress, index + 1) for index in range(jobs)]
+      futures = [pool.submit(engrave_scores, paths[index::jobs], root, fonts, cache_dir, progress, index + 1, renderer) for index in range(jobs)]
       results = [future.result() for future in futures]
   if progress:
-    print(f'Engraving complete: {sum(total for total, _ in results)} scores, {sum(reused for _, reused in results)} reused', flush=True)
+    total, reused, missing, rejected = (sum(row[index] for row in results) for index in range(4))
+    print(f'Engraving complete: {total} scores, {reused} reused, {missing} missing, {rejected} rejected', flush=True)

generator/engraving/cache.py

@@ -5,20 +5,27 @@ import importlib.metadata
 import json
 from pathlib import Path
 import shutil
-import sys
+import platform
 import tempfile
 
 from PIL import features
 
-def renderer_fingerprint(fonts, style) -> str:
+def renderer_identity(fonts, style) -> dict:
   root = Path(__file__).resolve().parents[1]
   versions = {name: importlib.metadata.version(name) for name in ('Pillow', 'fonttools', 'aggdraw')}
   versions.update({name: features.version(name) for name in features.get_supported()})
-  digest = hashlib.sha256(json.dumps([fonts.fingerprint(style), sys.version, versions], sort_keys=True).encode())
-  for path in sorted([*root.joinpath('engraving').glob('*.py'), root / 'model.py', root / 'harmony.py', root / 'requirements.txt']):
-    digest.update(path.relative_to(root).as_posix().encode())
-    digest.update(path.read_bytes())
-  return digest.hexdigest()
+  sources = [*root.joinpath('engraving').glob('*.py'), root / 'model.py', root / 'harmony.py', root / 'requirements.txt']
+  return {'fonts_style': fonts.fingerprint(style), 'python': {'implementation': platform.python_implementation(), 'version': platform.python_version()}, 'dependencies': versions,
+    'sources': {path.relative_to(root).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() for path in sorted(sources)}}
+
+def renderer_fingerprint(fonts, style) -> str:
+  return hashlib.sha256(json.dumps(renderer_identity(fonts, style), sort_keys=True).encode()).hexdigest()
+
+def renderer_namespace(cache: Path, renderer: str) -> Path:
+  namespace = cache / 'v2' / renderer
+  if namespace.parent.is_symlink() or namespace.is_symlink():
+    raise ValueError('engraving cache namespaces must not be symlinks')
+  return namespace
 
 def score_fingerprint(document: dict, renderer: str) -> str:
   music = {key: value for key, value in document.items() if key != 'playback'}

generator/score.py

@@ -10,6 +10,9 @@ from generator.model import SCHEMA_VERSION, validate_document
 
 def load_json(path: Path) -> dict:
   document = json.loads(path.read_text(encoding='utf-8'))
+  return validate_json(document, path)
+
+def validate_json(document: dict, path: Path) -> dict:
   if 'music' in document:
     from generator.engraving import ENGINE_NAME, ENGINE_VERSION, PAGE_HEIGHT, PAGE_WIDTH, check_page
     presentation = document.get('presentation', {})
@@ -122,8 +125,8 @@ def count_json_files(root: Path) -> int:
 def validate_generated_tree(root: Path) -> None:
   audio_cache = {}
   for path in root.rglob('*.json'):
-    load_json(path)
     document = json.loads(path.read_text(encoding='utf-8'))
+    validate_json(document, path)
     if 'playback' in document:
       from generator.playback import validate_assets
       validate_assets(document['playback'], root, audio_cache)