Cached Generation

.gitea/workflows/ci.yml

@@ -102,8 +102,21 @@ jobs:
         run: |
           python3 -m pip install --no-cache-dir -r build/build_tools/docker/format-requirements.txt
           python3 modulus.py format --check
+      - name: Restore engraving cache
+        id: engraving-cache
+        uses: actions/cache/restore@v4
+        with:
+          path: build/build_common/engraving-cache
+          key: product-vamp-pages-v1-${{ runner.os }}-${{ hashFiles('generator/engraving/*.py', 'generator/model.py', 'generator/harmony.py', 'generator/requirements.txt', 'package/assets/fonts/petaluma/*') }}-${{ hashFiles('data/**/*.xml', 'data/*.xml') }}
+          restore-keys: product-vamp-pages-v1-${{ runner.os }}-${{ hashFiles('generator/engraving/*.py', 'generator/model.py', 'generator/harmony.py', 'generator/requirements.txt', 'package/assets/fonts/petaluma/*') }}-
       - name: Build native release
         run: python3 modulus.py build --target x86_64_gcc12_release
+      - name: Save engraving cache
+        if: steps.engraving-cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v4
+        with:
+          path: build/build_common/engraving-cache
+          key: ${{ steps.engraving-cache.outputs.cache-primary-key }}
       - name: Test native release
         run: python3 modulus.py test --target x86_64_gcc12_release
       - name: Test asset generation

README.md

@@ -91,6 +91,8 @@ python3 -m generator --data-dir data --output-dir /tmp/product-vamp-generated --
 The `--check` path compares generation invariants against `data/expected/performance_summary.json`.
 The pinned build-only dependencies are in `generator/requirements.txt`. Configure/build automatically refresh the project build image when this file changes; direct builds must install it in their Python environment.
 
+CMake builds engrave with two Python workers and share a disposable page cache under `build/build_common/engraving-cache`. Native and ARM packaging reuse unchanged pages; CI also retains this cache between runs. Music, playback, and catalogue validation still run for every generation. Set `VAMP_ENGRAVING_JOBS` (1-16) to adjust concurrency, or set `VAMP_ENGRAVING_CACHE_DIR` to an empty CMake value to disable caching. Direct generator runs can opt in with `--jobs 2 --engraving-cache build/build_common/engraving-cache --progress`.
+
 The build-time pipeline is Soundslice/MuseScore -> `.xml` -> musical model -> automatic arranger/transposer -> schema-v3 JSON, engraved PNG pages, and generated-tone audio assets. The Pi displays pages without interpreting notation. Sources in `data/songs/` produce acoustic, electric, and vocal parts in the authored musical key, with no song-key selection. Numbered practice sources `data/001.xml` through `data/366.xml` produce twelve tonic variants each relative to the MusicXML key. Their current C-major placeholders are included; identical titles or contents never merge exercises. Independent voices and exact pitch/rhythm remain in the semantic JSON. Practice preserves the authored voicings, shifting the entire exercise by an octave only when needed for guitar. Practice and generated guitar voicings use standard tuning, frets 0-20 inclusive, and a maximum four-fret span excluding open strings.
 
 Write all music in reading order: repeat bars, numbered endings, navigation jumps, and pass-specific ties are rejected. The seven-section performance arrangements remain fully written out. Chord labels use `M`, `m`, `aug`, `dim`, `sus`, and `mM7`, including explicit major triads such as `CM`. Recognized Soundslice text-only harmonies are normalized into structured chords; parenthesized chords mark optional turnarounds omitted from the final arrangement ending.

TECHNICAL_GUIDE.md

@@ -136,6 +136,8 @@ At runtime the shared on-demand score loader serves both display and audio. A ba
 
 `generator/engraving/` is a custom Python engraver, not a wrapper around MuseScore or another notation engine. `layout_score(document, style, fonts)` produces immutable layout pages with exact source intervals, notation anchors and drawing primitives. `rasterize_page(page, fonts)` draws one page at four-times resolution and downsamples it. `engrave_tree(root, fonts)` retains the asset-pipeline entry point. `generator/musicxml_writer.py` remains a round-trip/export utility and is not involved in engraving.
 
+Engraving supports a bounded process pool (`VAMP_ENGRAVING_JOBS`, default 2) and a build-only content-addressed page cache (`VAMP_ENGRAVING_CACHE_DIR`, default `build/build_common/engraving-cache`). Keys include the complete musical document, engraving implementation, fonts/metadata, style, Python and rendering dependency versions. Playback is deliberately excluded from page keys and is always rebuilt independently. Cache hits are copied into the current staged output and checked for checksums, PNG decoding, dimensions, blank pages and clipped ink. Missing or corrupt entries are regenerated. All semantic and asset validation, database construction and transactional publication remain in place. The cache is not installed and can be deleted at any time when builds are idle. CI saves it after the native build and reuses it for ARM packaging; generation prints stage timing and per-worker progress instead of remaining silent during engraving.
+
 `notation.py` interprets staff state, exact pitch positions and shared accidental state; `notes.py` and `attributes.py` construct symbols; `spans.py` handles beams, ties, tuplets, octave shifts and lyric continuations; `layout.py` handles spacing and pagination. `fonts.py` reads bundled OTF outlines and SMuFL metadata, caching metrics and outlines for one generation run. `raster.py` uses FontTools, aggdraw and Pillow only as low-level font and drawing libraries. All dependencies are pinned in `generator/requirements.txt` and remain build-only. Standard glyph names and their upstream notice are vendored alongside the fonts; no system music fonts are consulted.
 
 The former backend-specific four-voice, duration, unequal-tone and tuplet restrictions are removed. Voices retain separate stems and collision offsets; chord tones retain their own written durations and per-tone ties. Pitched, percussion and explicitly fingered TAB material use their own staff positions. Unknown exact microtonal alterations use a numeric rational semitone annotation relative to the natural pitch, not a rounded accidental. An explicit unavailable SMuFL glyph is an error. The single-part/single-staff importer contract and exclusions for dynamics, accents, expressive ornaments and navigation remain unchanged.

cmake/zero.cmake

@@ -122,6 +122,12 @@ file(GLOB_RECURSE VAMP_GENERATOR_INPUTS CONFIGURE_DEPENDS
 )
 set(VAMP_GENERATED_ASSETS_DIR ${CMAKE_CURRENT_BINARY_DIR}/generated-assets)
 set(VAMP_GENERATED_ASSETS_STAMP ${CMAKE_CURRENT_BINARY_DIR}/generated-assets.stamp)
+set(VAMP_ENGRAVING_JOBS 2 CACHE STRING "Maximum parallel Python engraving workers (1-16)")
+set(VAMP_ENGRAVING_CACHE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/build/build_common/engraving-cache" CACHE PATH "Shared build-only page cache; empty disables caching")
+set(VAMP_ENGRAVING_CACHE_ARGS)
+if(VAMP_ENGRAVING_CACHE_DIR)
+  list(APPEND VAMP_ENGRAVING_CACHE_ARGS --engraving-cache "${VAMP_ENGRAVING_CACHE_DIR}")
+endif()
 string(JOIN "\n" VAMP_GENERATOR_INPUT_MANIFEST ${VAMP_GENERATOR_INPUTS})
 file(CONFIGURE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/generated-assets.inputs
   CONTENT "@VAMP_GENERATOR_INPUT_MANIFEST@\n" @ONLY)
@@ -129,14 +135,19 @@ add_custom_command(
   OUTPUT ${VAMP_GENERATED_ASSETS_STAMP}
   BYPRODUCTS ${VAMP_GENERATED_ASSETS_DIR}/catalogue.db
   COMMAND ${Python3_EXECUTABLE}
+    -u
     -m generator
     --data-dir ${CMAKE_CURRENT_SOURCE_DIR}/data
     --output-dir ${VAMP_GENERATED_ASSETS_DIR}
     --static-assets-dir ${CMAKE_CURRENT_SOURCE_DIR}/package/assets
     --check
+    --jobs ${VAMP_ENGRAVING_JOBS}
+    --progress
+    ${VAMP_ENGRAVING_CACHE_ARGS}
   COMMAND ${CMAKE_COMMAND} -E touch ${VAMP_GENERATED_ASSETS_STAMP}
   WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
   DEPENDS ${VAMP_GENERATOR_INPUTS} ${CMAKE_CURRENT_BINARY_DIR}/generated-assets.inputs
+  USES_TERMINAL
   VERBATIM
 )
 add_custom_target(vamp-generated-assets ALL DEPENDS ${VAMP_GENERATED_ASSETS_STAMP})