diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 6f8e1fd..e57d62e 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -102,8 +102,21 @@ jobs: run: | python3 -m pip install --no-cache-dir -r build/build_tools/docker/format-requirements.txt python3 modulus.py format --check + - name: Restore engraving cache + id: engraving-cache + uses: actions/cache/restore@v4 + with: + path: build/build_common/engraving-cache + key: product-vamp-pages-v1-${{ runner.os }}-${{ hashFiles('generator/engraving/*.py', 'generator/model.py', 'generator/harmony.py', 'generator/requirements.txt', 'package/assets/fonts/petaluma/*') }}-${{ hashFiles('data/**/*.xml', 'data/*.xml') }} + restore-keys: product-vamp-pages-v1-${{ runner.os }}-${{ hashFiles('generator/engraving/*.py', 'generator/model.py', 'generator/harmony.py', 'generator/requirements.txt', 'package/assets/fonts/petaluma/*') }}- - name: Build native release run: python3 modulus.py build --target x86_64_gcc12_release + - name: Save engraving cache + if: steps.engraving-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: build/build_common/engraving-cache + key: ${{ steps.engraving-cache.outputs.cache-primary-key }} - name: Test native release run: python3 modulus.py test --target x86_64_gcc12_release - name: Test asset generation diff --git a/README.md b/README.md index 553dc7c..ae4d589 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,8 @@ python3 -m generator --data-dir data --output-dir /tmp/product-vamp-generated -- The `--check` path compares generation invariants against `data/expected/performance_summary.json`. The pinned build-only dependencies are in `generator/requirements.txt`. Configure/build automatically refresh the project build image when this file changes; direct builds must install it in their Python environment. +CMake builds engrave with two Python workers and share a disposable page cache under `build/build_common/engraving-cache`. Native and ARM packaging reuse unchanged pages; CI also retains this cache between runs. Music, playback, and catalogue validation still run for every generation. Set `VAMP_ENGRAVING_JOBS` (1-16) to adjust concurrency, or set `VAMP_ENGRAVING_CACHE_DIR` to an empty CMake value to disable caching. Direct generator runs can opt in with `--jobs 2 --engraving-cache build/build_common/engraving-cache --progress`. + The build-time pipeline is Soundslice/MuseScore -> `.xml` -> musical model -> automatic arranger/transposer -> schema-v3 JSON, engraved PNG pages, and generated-tone audio assets. The Pi displays pages without interpreting notation. Sources in `data/songs/` produce acoustic, electric, and vocal parts in the authored musical key, with no song-key selection. Numbered practice sources `data/001.xml` through `data/366.xml` produce twelve tonic variants each relative to the MusicXML key. Their current C-major placeholders are included; identical titles or contents never merge exercises. Independent voices and exact pitch/rhythm remain in the semantic JSON. Practice preserves the authored voicings, shifting the entire exercise by an octave only when needed for guitar. Practice and generated guitar voicings use standard tuning, frets 0-20 inclusive, and a maximum four-fret span excluding open strings. Write all music in reading order: repeat bars, numbered endings, navigation jumps, and pass-specific ties are rejected. The seven-section performance arrangements remain fully written out. Chord labels use `M`, `m`, `aug`, `dim`, `sus`, and `mM7`, including explicit major triads such as `CM`. Recognized Soundslice text-only harmonies are normalized into structured chords; parenthesized chords mark optional turnarounds omitted from the final arrangement ending. diff --git a/TECHNICAL_GUIDE.md b/TECHNICAL_GUIDE.md index 7cf33d3..903a166 100644 --- a/TECHNICAL_GUIDE.md +++ b/TECHNICAL_GUIDE.md @@ -136,6 +136,8 @@ At runtime the shared on-demand score loader serves both display and audio. A ba `generator/engraving/` is a custom Python engraver, not a wrapper around MuseScore or another notation engine. `layout_score(document, style, fonts)` produces immutable layout pages with exact source intervals, notation anchors and drawing primitives. `rasterize_page(page, fonts)` draws one page at four-times resolution and downsamples it. `engrave_tree(root, fonts)` retains the asset-pipeline entry point. `generator/musicxml_writer.py` remains a round-trip/export utility and is not involved in engraving. +Engraving supports a bounded process pool (`VAMP_ENGRAVING_JOBS`, default 2) and a build-only content-addressed page cache (`VAMP_ENGRAVING_CACHE_DIR`, default `build/build_common/engraving-cache`). Keys include the complete musical document, engraving implementation, fonts/metadata, style, Python and rendering dependency versions. Playback is deliberately excluded from page keys and is always rebuilt independently. Cache hits are copied into the current staged output and checked for checksums, PNG decoding, dimensions, blank pages and clipped ink. Missing or corrupt entries are regenerated. All semantic and asset validation, database construction and transactional publication remain in place. The cache is not installed and can be deleted at any time when builds are idle. CI saves it after the native build and reuses it for ARM packaging; generation prints stage timing and per-worker progress instead of remaining silent during engraving. + `notation.py` interprets staff state, exact pitch positions and shared accidental state; `notes.py` and `attributes.py` construct symbols; `spans.py` handles beams, ties, tuplets, octave shifts and lyric continuations; `layout.py` handles spacing and pagination. `fonts.py` reads bundled OTF outlines and SMuFL metadata, caching metrics and outlines for one generation run. `raster.py` uses FontTools, aggdraw and Pillow only as low-level font and drawing libraries. All dependencies are pinned in `generator/requirements.txt` and remain build-only. Standard glyph names and their upstream notice are vendored alongside the fonts; no system music fonts are consulted. The former backend-specific four-voice, duration, unequal-tone and tuplet restrictions are removed. Voices retain separate stems and collision offsets; chord tones retain their own written durations and per-tone ties. Pitched, percussion and explicitly fingered TAB material use their own staff positions. Unknown exact microtonal alterations use a numeric rational semitone annotation relative to the natural pitch, not a rounded accidental. An explicit unavailable SMuFL glyph is an error. The single-part/single-staff importer contract and exclusions for dynamics, accents, expressive ornaments and navigation remain unchanged. diff --git a/cmake/zero.cmake b/cmake/zero.cmake index bc1522d..a25e861 100644 --- a/cmake/zero.cmake +++ b/cmake/zero.cmake @@ -122,6 +122,12 @@ file(GLOB_RECURSE VAMP_GENERATOR_INPUTS CONFIGURE_DEPENDS ) set(VAMP_GENERATED_ASSETS_DIR ${CMAKE_CURRENT_BINARY_DIR}/generated-assets) set(VAMP_GENERATED_ASSETS_STAMP ${CMAKE_CURRENT_BINARY_DIR}/generated-assets.stamp) +set(VAMP_ENGRAVING_JOBS 2 CACHE STRING "Maximum parallel Python engraving workers (1-16)") +set(VAMP_ENGRAVING_CACHE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/build/build_common/engraving-cache" CACHE PATH "Shared build-only page cache; empty disables caching") +set(VAMP_ENGRAVING_CACHE_ARGS) +if(VAMP_ENGRAVING_CACHE_DIR) + list(APPEND VAMP_ENGRAVING_CACHE_ARGS --engraving-cache "${VAMP_ENGRAVING_CACHE_DIR}") +endif() string(JOIN "\n" VAMP_GENERATOR_INPUT_MANIFEST ${VAMP_GENERATOR_INPUTS}) file(CONFIGURE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/generated-assets.inputs CONTENT "@VAMP_GENERATOR_INPUT_MANIFEST@\n" @ONLY) @@ -129,14 +135,19 @@ add_custom_command( OUTPUT ${VAMP_GENERATED_ASSETS_STAMP} BYPRODUCTS ${VAMP_GENERATED_ASSETS_DIR}/catalogue.db COMMAND ${Python3_EXECUTABLE} + -u -m generator --data-dir ${CMAKE_CURRENT_SOURCE_DIR}/data --output-dir ${VAMP_GENERATED_ASSETS_DIR} --static-assets-dir ${CMAKE_CURRENT_SOURCE_DIR}/package/assets --check + --jobs ${VAMP_ENGRAVING_JOBS} + --progress + ${VAMP_ENGRAVING_CACHE_ARGS} COMMAND ${CMAKE_COMMAND} -E touch ${VAMP_GENERATED_ASSETS_STAMP} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} DEPENDS ${VAMP_GENERATOR_INPUTS} ${CMAKE_CURRENT_BINARY_DIR}/generated-assets.inputs + USES_TERMINAL VERBATIM ) add_custom_target(vamp-generated-assets ALL DEPENDS ${VAMP_GENERATED_ASSETS_STAMP}) diff --git a/generator/__main__.py b/generator/__main__.py index 7ab6add..beff518 100644 --- a/generator/__main__.py +++ b/generator/__main__.py @@ -4,6 +4,7 @@ import argparse import shutil import tempfile from pathlib import Path +from time import monotonic from generator.arrange.generate import generate_performance_scores from generator.practice.generate import generate_practice_scores @@ -29,12 +30,27 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--output-dir", type=Path, required=True) parser.add_argument("--static-assets-dir", type=Path, default=Path("package/assets")) parser.add_argument("--check", action="store_true") + parser.add_argument("--engraving-cache", type=Path) + parser.add_argument("--jobs", type=int, default=1, choices=range(1, 17), metavar="1-16") + parser.add_argument("--progress", action="store_true") return parser.parse_args() def generate_assets(args: argparse.Namespace) -> None: data_dir = args.data_dir.resolve() output_dir = args.output_dir.resolve() static_assets_dir = args.static_assets_dir.resolve() + cache_dir = getattr(args, 'engraving_cache', None) + cache_dir = cache_dir.resolve() if cache_dir else None + if cache_dir and (cache_dir.is_relative_to(output_dir) or output_dir.is_relative_to(cache_dir)): + raise ValueError('engraving cache and generated assets must not overlap') + progress = getattr(args, 'progress', False) + started = monotonic() + + def report(message): + if progress: + print(f'Assets [{monotonic() - started:.0f}s]: {message}', flush=True) + + report('validating MusicXML sources') for source in sorted(data_dir.rglob('*.xml')): load_musicxml(source) output_dir.parent.mkdir(parents=True, exist_ok=True) @@ -42,15 +58,20 @@ def generate_assets(args: argparse.Namespace) -> None: staged = Path(temporary) / 'assets' staged.mkdir(parents=True, exist_ok=True) copy_runtime_fonts(static_assets_dir, staged) + report('generating performance parts and practice variants') performance_count = generate_performance_scores(data_dir / 'songs', staged / 'songs') practice_count = generate_practice_scores(data_dir, staged / 'practice') validate_generated_tree(staged) if args.check: validate_expected_performance_summary(data_dir / 'expected' / 'performance_summary.json', staged / 'songs') validate_practice_variants(staged / 'practice') + report('compiling playback') compile_tree(staged) - engrave_tree(staged, static_assets_dir / 'fonts' / 'petaluma') + report('engraving pages') + engrave_tree(staged, static_assets_dir / 'fonts' / 'petaluma', cache_dir=cache_dir, jobs=getattr(args, 'jobs', 1), progress=progress) + report('validating score, page, and audio assets') validate_generated_tree(staged) + report('building catalogue database') build_catalogue(staged) backup = Path(temporary) / 'previous' if output_dir.exists(): @@ -61,6 +82,7 @@ def generate_assets(args: argparse.Namespace) -> None: if backup.exists(): backup.rename(output_dir) raise + report('publishing complete') print(f'Generated Product Vamp assets: {performance_count} performance scores, ' f'{practice_count} practice score variants, {count_json_files(output_dir)} total JSON files.') if args.check: diff --git a/generator/engraving/__init__.py b/generator/engraving/__init__.py index 19fafb6..400a0ff 100644 --- a/generator/engraving/__init__.py +++ b/generator/engraving/__init__.py @@ -1,16 +1,20 @@ from __future__ import annotations +from concurrent.futures import ProcessPoolExecutor import hashlib import json +import multiprocessing from pathlib import Path import shutil from PIL import Image, ImageChops from generator.engraving.fonts import Fonts +from generator.engraving.cache import renderer_fingerprint, restore_pages, score_fingerprint, store_pages from generator.engraving.layout import layout_score from generator.engraving.raster import rasterize_page from generator.engraving.types import EngravingError, Style +from generator.model import validate_document ENGINE_NAME = 'Vamp Python' ENGINE_VERSION = '1.0.0' @@ -31,31 +35,63 @@ def check_page(path: Path) -> None: if bounds[0] < 4 or bounds[1] < 4 or bounds[2] > PAGE_WIDTH - 4 or bounds[3] > PAGE_HEIGHT - 4: raise EngravingError(f'{path.name}: notation reaches the page edge; refusing potentially clipped output') -def engrave_tree(root: Path, fonts: Path) -> None: - paths = sorted(root.rglob('*.json')) - if not paths: - return +def engrave_scores(paths: list[Path], root: Path, fonts: Path, cache_dir: Path | None, progress: bool, worker: int) -> tuple[int, int]: family = Fonts(fonts) style = Style() - for path in paths: + fingerprint = family.fingerprint(style) + renderer = renderer_fingerprint(family, style) if cache_dir else '' + reused = 0 + for index, path in enumerate(paths, 1): document = json.loads(path.read_text(encoding='utf-8')) try: - layout = layout_score(document, style, family) + validate_document(document) destination = path.with_suffix('.pages') if destination.exists(): shutil.rmtree(destination) - destination.mkdir() - entries = [] - for page in layout.pages: - name = f'{page.number:04}.png' - target = destination / name - rasterize_page(page, family).save(target, format='PNG') - check_page(target) - entries.append({'path': destination.name + '/' + name, 'sha256': hashlib.sha256(target.read_bytes()).hexdigest()}) + entry = cache_dir / score_fingerprint(document, renderer) if cache_dir else None + pages = restore_pages(entry, destination) if entry else None + if pages is None: + if destination.exists(): + shutil.rmtree(destination) + destination.mkdir() + layout = layout_score(document, style, family) + pages = [] + for page in layout.pages: + name = f'{page.number:04}.png' + target = destination / name + rasterize_page(page, family).save(target, format='PNG') + check_page(target) + pages.append({'path': name, 'sha256': hashlib.sha256(target.read_bytes()).hexdigest()}) + if entry: + store_pages(entry, destination, pages) + else: + reused += 1 + entries = [{**page, 'path': destination.name + '/' + page['path']} for page in pages] music = {k: v for k, v in document.items() if k not in ('schema_version', 'playback')} - bundle = {'schema_version': 3, 'music': music, 'presentation': {'engine': ENGINE_NAME, 'engine_version': ENGINE_VERSION, 'style_sha256': layout.fingerprint, 'width': PAGE_WIDTH, 'height': PAGE_HEIGHT, 'pages': entries}} + bundle = {'schema_version': 3, 'music': music, 'presentation': {'engine': ENGINE_NAME, 'engine_version': ENGINE_VERSION, 'style_sha256': fingerprint, 'width': PAGE_WIDTH, 'height': PAGE_HEIGHT, 'pages': entries}} if 'playback' in document: bundle['playback'] = document['playback'] path.write_text(json.dumps(bundle, indent=2, ensure_ascii=False) + '\n', encoding='utf-8') + if progress and (index % 50 == 0 or index == len(paths)): + print(f'Engraving worker {worker}: {index}/{len(paths)} scores, {reused} reused', flush=True) except ValueError as error: raise EngravingError(f'{path.relative_to(root)}: {error}') from error + return len(paths), reused + +def engrave_tree(root: Path, fonts: Path, *, cache_dir: Path | None = None, jobs: int = 1, progress: bool = False) -> None: + if type(jobs) is not int or not 1 <= jobs <= 16: + raise ValueError('engraving jobs must be between 1 and 16') + if cache_dir and (cache_dir.resolve().is_relative_to(root.resolve()) or root.resolve().is_relative_to(cache_dir.resolve())): + raise ValueError('engraving cache and generated assets must not overlap') + paths = sorted(root.rglob('*.json')) + if not paths: + return + jobs = min(jobs, len(paths)) + if jobs == 1: + results = [engrave_scores(paths, root, fonts, cache_dir, progress, 1)] + else: + with ProcessPoolExecutor(max_workers=jobs, mp_context=multiprocessing.get_context('spawn')) as pool: + futures = [pool.submit(engrave_scores, paths[index::jobs], root, fonts, cache_dir, progress, index + 1) for index in range(jobs)] + results = [future.result() for future in futures] + if progress: + print(f'Engraving complete: {sum(total for total, _ in results)} scores, {sum(reused for _, reused in results)} reused', flush=True) diff --git a/generator/engraving/cache.py b/generator/engraving/cache.py new file mode 100644 index 0000000..08b7e38 --- /dev/null +++ b/generator/engraving/cache.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import hashlib +import importlib.metadata +import json +from pathlib import Path +import shutil +import sys +import tempfile + +from PIL import features + +def renderer_fingerprint(fonts, style) -> str: + root = Path(__file__).resolve().parents[1] + versions = {name: importlib.metadata.version(name) for name in ('Pillow', 'fonttools', 'aggdraw')} + versions.update({name: features.version(name) for name in features.get_supported()}) + digest = hashlib.sha256(json.dumps([fonts.fingerprint(style), sys.version, versions], sort_keys=True).encode()) + for path in sorted([*root.joinpath('engraving').glob('*.py'), root / 'model.py', root / 'harmony.py', root / 'requirements.txt']): + digest.update(path.relative_to(root).as_posix().encode()) + digest.update(path.read_bytes()) + return digest.hexdigest() + +def score_fingerprint(document: dict, renderer: str) -> str: + music = {key: value for key, value in document.items() if key != 'playback'} + return hashlib.sha256(json.dumps([renderer, music], sort_keys=True, ensure_ascii=False).encode()).hexdigest() + +def restore_pages(entry: Path, destination: Path) -> list[dict] | None: + from generator.engraving import check_page + try: + if entry.is_symlink(): + return None + manifest = entry / 'pages.json' + if manifest.is_symlink(): + return None + pages = json.loads(manifest.read_text(encoding='utf-8')) + if not isinstance(pages, list) or not 0 < len(pages) <= 10000: + return None + destination.mkdir(parents=True, exist_ok=True) + for index, page in enumerate(pages, 1): + if not isinstance(page, dict) or set(page) != {'path', 'sha256'} or page['path'] != f'{index:04}.png': + return None + path = entry / page['path'] + if path.is_symlink(): + return None + target = destination / page['path'] + shutil.copyfile(path, target) + if hashlib.sha256(target.read_bytes()).hexdigest() != page['sha256']: + return None + check_page(target) + return pages + except (OSError, ValueError, TypeError): + return None + +def store_pages(entry: Path, destination: Path, pages: list[dict]) -> None: + entry.parent.mkdir(parents=True, exist_ok=True) + if entry.is_symlink() or entry.is_file(): + entry.unlink(missing_ok=True) + entry.mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(prefix='.engraving-', dir=entry.parent) as temporary: + staged = Path(temporary) + for page in pages: + shutil.copyfile(destination / page['path'], staged / page['path']) + (staged / 'pages.json').write_text(json.dumps(pages), encoding='utf-8') + # Identical keys have identical pages; publish complete files, then the manifest. + for name in [page['path'] for page in pages] + ['pages.json']: + target = entry / name + if target.is_dir() and not target.is_symlink(): + shutil.rmtree(target, ignore_errors=True) + (staged / name).replace(target) diff --git a/tests/test_engraving_cache.py b/tests/test_engraving_cache.py new file mode 100644 index 0000000..e555fa5 --- /dev/null +++ b/tests/test_engraving_cache.py @@ -0,0 +1,151 @@ +import argparse +import copy +import hashlib +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from generator.__main__ import generate_assets +from generator.engraving import EngravingError, Fonts, Style, engrave_tree, layout_score +from generator.engraving.cache import renderer_fingerprint, restore_pages, score_fingerprint +from tests.test_musicxml import parse, xml_note, xml_score + +class EngravingCacheTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.cache = self.root / 'cache' + self.fonts = Path('package/assets/fonts/petaluma') + self.document = parse(xml_score(xml_note())) + def write_scores(self, root, documents): + root.mkdir(parents=True, exist_ok=True) + for index, document in enumerate(documents): + (root / f'{index:03}.json').write_text(json.dumps(document), encoding='utf-8') + def files(self, root): + return {p.relative_to(root): p.read_bytes() for p in root.rglob('*') if p.is_file()} + def seed_cache(self): + root = self.root / 'seed' + self.write_scores(root, [self.document]) + engrave_tree(root, self.fonts, cache_dir=self.cache) + return next(self.cache.iterdir()) + def test_cache_hit_preserves_bundle_bytes_across_output_directories(self): + cold, warm = self.root / 'cold', self.root / 'warm' + self.write_scores(cold, [self.document]) + engrave_tree(cold, self.fonts, cache_dir=self.cache) + self.write_scores(warm, [self.document]) + with patch('generator.engraving.layout_score', side_effect=AssertionError('cache miss')): + with patch('generator.engraving.rasterize_page', side_effect=AssertionError('cache miss')): + engrave_tree(warm, self.fonts, cache_dir=self.cache) + self.assertEqual(self.files(cold), self.files(warm)) + def test_playback_is_not_restored_from_page_cache(self): + self.seed_cache() + changed = copy.deepcopy(self.document) + changed['playback'] = {'new-playback': 'independent of page rendering'} + root = self.root / 'warm' + self.write_scores(root, [changed]) + with patch('generator.engraving.layout_score', side_effect=AssertionError('cache miss')): + engrave_tree(root, self.fonts, cache_dir=self.cache) + self.assertEqual(json.loads((root / '000.json').read_text())['playback'], changed['playback']) + def test_changed_music_and_renderer_invalidate_pages(self): + self.seed_cache() + changed = parse(xml_score(xml_note('D')).replace('Fixture', 'Changed')) + self.assertNotEqual(score_fingerprint(self.document, 'renderer'), score_fingerprint(changed, 'renderer')) + self.assertNotEqual(score_fingerprint(self.document, 'renderer'), score_fingerprint(self.document, 'new-renderer')) + root = self.root / 'changed' + self.write_scores(root, [changed]) + with patch('generator.engraving.layout_score', wraps=layout_score) as render: + engrave_tree(root, self.fonts, cache_dir=self.cache) + render.assert_called_once() + self.assertEqual(len(list(self.cache.iterdir())), 2) + def test_renderer_fingerprint_tracks_fonts_style_code_and_dependencies(self): + fonts, style = Fonts(self.fonts), Style() + expected = renderer_fingerprint(fonts, style) + self.assertEqual(expected, renderer_fingerprint(fonts, style)) + self.assertNotEqual(expected, renderer_fingerprint(fonts, Style(margin_x=29))) + with patch.object(fonts, 'fingerprint', return_value='changed-fonts'): + self.assertNotEqual(expected, renderer_fingerprint(fonts, style)) + with patch('generator.engraving.cache.importlib.metadata.version', return_value='new-version'): + self.assertNotEqual(expected, renderer_fingerprint(fonts, style)) + read_bytes = Path.read_bytes + with patch.object(Path, 'read_bytes', lambda path: read_bytes(path) + (b'\n' if path.name == 'raster.py' else b'')): + self.assertNotEqual(expected, renderer_fingerprint(fonts, style)) + def test_corrupt_or_missing_pages_are_regenerated_and_repaired(self): + entry = self.seed_cache() + page = entry / '0001.png' + original = page.read_bytes() + for corruption in ('missing', 'checksum', 'undecodable', 'manifest', 'page_directory', 'manifest_directory'): + with self.subTest(corruption=corruption): + if corruption == 'missing': + page.unlink() + elif corruption == 'checksum': + page.write_bytes(original + b'changed') + elif corruption == 'undecodable': + page.write_bytes(original[:24]) + (entry / 'pages.json').write_text(json.dumps([{'path': page.name, 'sha256': hashlib.sha256(page.read_bytes()).hexdigest()}])) + elif corruption == 'manifest': + (entry / 'pages.json').write_text('invalid json') + else: + target = page if corruption == 'page_directory' else entry / 'pages.json' + target.unlink() + target.mkdir() + root = self.root / corruption + self.write_scores(root, [self.document]) + with patch('generator.engraving.layout_score', wraps=layout_score) as render: + engrave_tree(root, self.fonts, cache_dir=self.cache) + render.assert_called_once() + self.assertEqual(page.read_bytes(), original) + self.assertIsNotNone(restore_pages(entry, self.root / 'restored')) + def test_cache_manifest_cannot_escape_entry_or_inject_bundle_fields(self): + entry = self.seed_cache() + original = json.loads((entry / 'pages.json').read_text()) + for invalid in ([], {}, [{'path': '../outside.png', 'sha256': original[0]['sha256']}], [{**original[0], 'extra': True}]): + with self.subTest(manifest=invalid): + (entry / 'pages.json').write_text(json.dumps(invalid)) + self.assertIsNone(restore_pages(entry, self.root / 'output')) + (entry / 'pages.json').write_text(json.dumps(original)) + page = entry / '0001.png' + external = self.root / 'outside.png' + page.rename(external) + page.symlink_to(external) + self.assertIsNone(restore_pages(entry, self.root / 'output')) + alias = self.root / 'alias' + alias.symlink_to(entry, target_is_directory=True) + self.assertIsNone(restore_pages(alias, self.root / 'output')) + def test_parallel_output_and_shared_cache_are_deterministic(self): + documents = [self.document, parse(xml_score(xml_note('D')))] * 4 + serial, parallel, warm = (self.root / name for name in ('serial', 'parallel', 'warm')) + self.write_scores(serial, documents) + engrave_tree(serial, self.fonts) + for root in (parallel, warm): + self.write_scores(root, documents) + engrave_tree(root, self.fonts, cache_dir=self.cache, jobs=2) + self.assertEqual(self.files(serial), self.files(root)) + self.assertEqual(len(list(self.cache.iterdir())), 2) + self.assertFalse(list(self.cache.glob('.engraving-*'))) + def test_invalid_worker_count_and_overlapping_cache_are_rejected(self): + root = self.root / 'assets' + self.write_scores(root, [self.document]) + for jobs in (0, -1, 17, True): + with self.assertRaisesRegex(ValueError, 'between 1 and 16'): + engrave_tree(root, self.fonts, jobs=jobs) + for cache in (root, root / 'cache', self.root): + with self.assertRaisesRegex(ValueError, 'must not overlap'): + engrave_tree(root, self.fonts, cache_dir=cache) + def test_parallel_generation_preserves_assets_on_worker_failure(self): + data, output = self.root / 'data', self.root / 'output' + (data / 'songs').mkdir(parents=True) + source = data / '001.xml' + source.write_text(xml_score(xml_note())) + args = argparse.Namespace(data_dir=data, output_dir=output, static_assets_dir=Path('package/assets'), check=False, engraving_cache=self.cache, jobs=2) + generate_assets(args) + original = self.files(output) + generate_assets(args) + self.assertEqual(original, self.files(output)) + source.write_text(xml_score(xml_note(extra='other'))) + with self.assertRaisesRegex(EngravingError, 'unavailableGlyph'): + generate_assets(args) + self.assertEqual(original, self.files(output)) + self.assertFalse(list(output.parent.glob('.vamp-assets-*')))