diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index edf6fda..f65464e 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -97,25 +97,28 @@ jobs: key: product-vamp-conan-${{ runner.os }}-${{ hashFiles('conanfile.py', 'modulus.json', 'conan/profiles/*') }} restore-keys: product-vamp-conan-${{ runner.os }}- - name: Configure native release - run: python3 modulus.py configure --target x86_64_gcc12_release + run: python3 modulus.py configure --target x86_64_gcc12_release --update - name: Check code style run: | python3 -m pip install --no-cache-dir -r build/build_tools/docker/format-requirements.txt python3 modulus.py format --check + - name: Identify engraving renderer + id: renderer + run: echo "fingerprint=$(python3 -m generator --renderer-fingerprint)" >> "$GITHUB_OUTPUT" - name: Restore engraving cache id: engraving-cache uses: actions/cache/restore@v4 with: - path: build/build_common/engraving-cache - key: product-vamp-pages-v1-${{ runner.os }}-${{ hashFiles('generator/engraving/*.py', 'generator/model.py', 'generator/harmony.py', 'generator/requirements.txt', 'package/assets/fonts/petaluma/*') }}-${{ hashFiles('generator/**/*.py', 'generator/*.py', 'data/**/*.xml', 'data/*.xml') }} - restore-keys: product-vamp-pages-v1-${{ runner.os }}-${{ hashFiles('generator/engraving/*.py', 'generator/model.py', 'generator/harmony.py', 'generator/requirements.txt', 'package/assets/fonts/petaluma/*') }}- + path: build/build_common/engraving-cache/v2/${{ steps.renderer.outputs.fingerprint }} + key: product-vamp-pages-v2-${{ steps.renderer.outputs.fingerprint }}-${{ hashFiles('generator/**/*.py', 'generator/*.py', 'data/**/*.xml', 'data/*.xml') }} + restore-keys: product-vamp-pages-v2-${{ steps.renderer.outputs.fingerprint }}- - name: Build native release run: python3 modulus.py build --target x86_64_gcc12_release - name: Save engraving cache if: steps.engraving-cache.outputs.cache-hit != 'true' uses: actions/cache/save@v4 with: - path: build/build_common/engraving-cache + path: build/build_common/engraving-cache/v2/${{ steps.renderer.outputs.fingerprint }} key: ${{ steps.engraving-cache.outputs.cache-primary-key }} - name: Test native release run: python3 modulus.py test --target x86_64_gcc12_release diff --git a/README.md b/README.md index 8d06bb9..f2437ad 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,9 @@ python3 modulus.py build --target x86_64_gcc12_release `python3 modulus.py run` uses `package/config/vamp.development.json` for a headless readiness check against native build assets. `vamp-zero --once` checks the packaged database and referenced page/audio assets, checks runtime writes and the server bind, then exits without opening audio or display hardware. Missing or incompatible catalogs require rebuilding or reinstalling; they are never repaired at runtime. `--check-config` only validates configuration; `--health-check` probes the running service's `/api/health` endpoint without modifying files. -Configure/build can read public packages anonymously from the khodo Conan registry. +Configure/build read public packages anonymously from the khodo Conan registry, even when the project cache contains an expired login. `configure --update` refreshes both the exported Modulus tooling and Conan dependencies. `--no-remote` uses only locally cached packages. Publishing is the only operation that needs credentials. CMake checks the required Sonus and Luma APIs during configuration and reports stale dependencies early. + +`modulus.py` is an unchanged copy of Modulus's canonical bootstrap. The project-specific build image lives in `package/Dockerfile.build`; `tooling.docker` declares its additional inputs and permits Docker socket access only for build/package commands that assemble the Zero image. Refresh clangd compile databases, including the Pico firmware database, with: diff --git a/TECHNICAL_GUIDE.md b/TECHNICAL_GUIDE.md index 77bc51c..e950698 100644 --- a/TECHNICAL_GUIDE.md +++ b/TECHNICAL_GUIDE.md @@ -29,7 +29,7 @@ The API is plain HTTP on the private AP: - `POST /api/remote/previous` - `POST /api/remote/select` -Legacy `/api/page/next` and `/api/page/previous` routes are still accepted for simple manual checks. +Page commands use `/api/remote/next` and `/api/remote/previous`; the old `/api/page/*` routes are removed. The AP setup pins NetworkManager to 2.4 GHz channel 6 with WPA2/RSN and CCMP ciphers so the Pico W can join using `CYW43_AUTH_WPA2_AES_PSK`. @@ -148,7 +148,7 @@ When grace groups steal from both ends of one donor, their adjustments accumulat At runtime the shared on-demand score loader serves both display and audio. A background worker validates and loads only the selected score's samples, converts absolute beat positions to rounded sample frames at the menu BPM, and prepares a Sonus sequence. Pitch does not change with BPM, and rounding absolute positions avoids cumulative timing drift. The I/O loop publishes the prepared sequence and polls completion without doing disk I/O or musical interpretation. Sonus mixes preallocated voices from immutable PCM and retires them on control threads; the callback never synthesizes tones or loads files. Limits are one million events, 512 distinct samples, 64 MiB decoded samples per selection, 64 simultaneous voices including releases, and 24 hours at the selected BPM. Exceeding them reports an error rather than dropping notes. -`audio.playback_enabled` defaults to false and replaces the old inert `track_enabled` setting. The old config name is accepted for migration, but conflicting values fail. Playback suppresses the metronome without changing its stored toggle. Authored tempo marks remain notation; menu BPM is constant throughout a session. Selecting a practice or performance part starts a new session. Manual page changes never restart or seek. Natural completion, including trailing silence/release, returns to the corresponding part menu with its selection preserved. Leaving the score cancels playback; stale results cannot close a newer score. Errors remain visible rather than being mistaken for completion. +`audio.playback_enabled` defaults to false. The obsolete `track_enabled` setting is rejected with a diagnostic naming its replacement. Playback suppresses the metronome without changing its stored toggle. Authored tempo marks remain notation; menu BPM is constant throughout a session. Selecting a practice or performance part starts a new session. Manual page changes never restart or seek. Natural completion, including trailing silence/release, returns to the corresponding part menu with its selection preserved. Leaving the score cancels playback; stale results cannot close a newer score. Errors remain visible rather than being mistaken for completion. `--once` additionally validates referenced audio without opening display/audio hardware. Runtime packages contain WAVs alongside pages and the SQLite catalog; the source XML and Python dependencies remain build-only. @@ -174,7 +174,7 @@ Pages are 976x484 pixels with 12-pixel staff spacing, 28-pixel horizontal margin Generation validates sources and complete staged output before replacing the previous asset directory. Replacement is always complete; deleted sources cannot leave stale JSON, pages, or audio. CMake tracks the input list as well as file contents, so additions and removals regenerate the catalog. Runtime packages contain `catalog.db` with embedded semantic JSON, PNG pages, generated PCM WAVs, and the OTF UI text font, not MusicXML, MuseScore, or Python dependencies. Python test discovery runs model/import/export tests; the normal asset build additionally qualifies and audits every catalog score. -CI page-cache snapshot keys include every generator Python module and XML source, while retaining a renderer/font-compatible restore prefix. Individual cached pages remain validated by their semantic-content and renderer fingerprints. +The page cache uses `v2///` namespaces. `python3 -m generator --renderer-fingerprint` reports the same identity used by rendering and CI: source bytes, fonts, style, Python implementation/version, and rendering-library versions. Python compiler/build timestamps are excluded. CI keys append all generator and XML inputs and restore older snapshots only within the same renderer namespace; only that namespace is saved after a successful build. A renderer or environment change intentionally starts cold. Progress reports the resolved namespace and reused, missing, and rejected counts. Old flat cache entries are not migrated and can be removed when builds are idle. Diagnostics identify the file, part, measure, and unsupported element or invalid relationship. Examples include `part P1, measure 4: unsupported musical element `, `tie stop has no contiguous matching start`, and `optional-chord marker must identify exactly one harmony`. diff --git a/cmake/zero.cmake b/cmake/zero.cmake index b2394fa..52729d4 100644 --- a/cmake/zero.cmake +++ b/cmake/zero.cmake @@ -12,6 +12,28 @@ find_package(ligo CONFIG REQUIRED) find_package(noto CONFIG REQUIRED) find_package(luma CONFIG REQUIRED) find_package(sonus CONFIG REQUIRED) +include(CheckCXXSourceCompiles) +include(CMakePushCheckState) +cmake_push_check_state(RESET) +set(CMAKE_TRY_COMPILE_CONFIGURATION "${CMAKE_BUILD_TYPE}") +set(CMAKE_REQUIRED_LIBRARIES sonus::sonus luma::luma) +unset(VAMP_DEPENDENCY_APIS_AVAILABLE CACHE) +check_cxx_source_compiles(" + #include + #include + #include + static_assert(requires(sonus::engine& engine, sonus::prepared_sequence sequence) { + engine.play_sequence(std::move(sequence)); + }); + static_assert(requires(luma::application& application, std::string path) { + application.load_texture_uncached(path); + }); + int main() { return 0; } +" VAMP_DEPENDENCY_APIS_AVAILABLE) +cmake_pop_check_state() +if(NOT VAMP_DEPENDENCY_APIS_AVAILABLE) + message(FATAL_ERROR "Product Vamp requires current Sonus sequence and Luma uncached-texture APIs. Run ./modulus.py configure --update after publishing the updated dependencies, or create their Conan packages locally before configuring --no-remote. See CMake's configure log for the compiler diagnostic.") +endif() include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/font.cmake) execute_process( diff --git a/conanfile.py b/conanfile.py index 4428aac..7a1442a 100644 --- a/conanfile.py +++ b/conanfile.py @@ -19,8 +19,11 @@ class product_vamp_conan(conan.ConanFile): "with_tests": False, } python_requires = "modulus/latest" - exports = "modulus.json" + exports = "modulus.json", "!build", "!.git", "!test_package/build" exports_sources = ( + "!build", + "!.git", + "!test_package/build", ".gitea/workflows/ci.yml", "CMakeLists.txt", "modulus.py", @@ -43,6 +46,7 @@ class product_vamp_conan(conan.ConanFile): "include/vamp_zero/*.hpp", "package/assets/*", "package/assets/**/*", + "package/Dockerfile.build", "package/config/*.json", "package/manifest.json.in", "package/pico/*", diff --git a/generator/__main__.py b/generator/__main__.py index bb657d4..72d60a9 100644 --- a/generator/__main__.py +++ b/generator/__main__.py @@ -11,7 +11,8 @@ from generator.practice.generate import generate_practice_scores from generator.score import count_json_files, validate_generated_tree from generator.summary import validate_expected_performance_summary, validate_practice_parts from generator.parts import source_paths -from generator.engraving import engrave_tree +from generator.engraving import Fonts, Style, engrave_tree +from generator.engraving.cache import renderer_fingerprint from generator.playback import compile_tree from generator.catalog import build_catalog @@ -26,14 +27,18 @@ def copy_runtime_fonts(static_assets_dir: Path, output_dir: Path) -> None: def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Generate Product Vamp package score assets.") - parser.add_argument("--data-dir", type=Path, required=True) - parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--data-dir", type=Path) + parser.add_argument("--output-dir", type=Path) + parser.add_argument("--renderer-fingerprint", action="store_true", help="Print the engraving cache identity without generating assets.") parser.add_argument("--static-assets-dir", type=Path, default=Path("package/assets")) parser.add_argument("--check", action="store_true") parser.add_argument("--engraving-cache", type=Path) parser.add_argument("--jobs", type=int, default=1, choices=range(1, 17), metavar="1-16") parser.add_argument("--progress", action="store_true") - return parser.parse_args() + args = parser.parse_args() + if not args.renderer_fingerprint and (args.data_dir is None or args.output_dir is None): + parser.error('--data-dir and --output-dir are required for generation') + return args def generate_assets(args: argparse.Namespace) -> None: data_dir = args.data_dir.resolve() @@ -92,7 +97,11 @@ def generate_assets(args: argparse.Namespace) -> None: print('Generator check passed.') def main() -> int: - generate_assets(parse_args()) + args = parse_args() + if args.renderer_fingerprint: + print(renderer_fingerprint(Fonts(args.static_assets_dir / 'fonts' / 'petaluma'), Style())) + else: + generate_assets(args) return 0 if __name__ == "__main__": diff --git a/generator/engraving/__init__.py b/generator/engraving/__init__.py index 400a0ff..cc6d479 100644 --- a/generator/engraving/__init__.py +++ b/generator/engraving/__init__.py @@ -10,7 +10,7 @@ import shutil from PIL import Image, ImageChops from generator.engraving.fonts import Fonts -from generator.engraving.cache import renderer_fingerprint, restore_pages, score_fingerprint, store_pages +from generator.engraving.cache import renderer_fingerprint, renderer_namespace, restore_pages, score_fingerprint, store_pages from generator.engraving.layout import layout_score from generator.engraving.raster import rasterize_page from generator.engraving.types import EngravingError, Style @@ -35,12 +35,11 @@ def check_page(path: Path) -> None: if bounds[0] < 4 or bounds[1] < 4 or bounds[2] > PAGE_WIDTH - 4 or bounds[3] > PAGE_HEIGHT - 4: raise EngravingError(f'{path.name}: notation reaches the page edge; refusing potentially clipped output') -def engrave_scores(paths: list[Path], root: Path, fonts: Path, cache_dir: Path | None, progress: bool, worker: int) -> tuple[int, int]: +def engrave_scores(paths: list[Path], root: Path, fonts: Path, cache_dir: Path | None, progress: bool, worker: int, renderer: str) -> tuple[int, int, int, int]: family = Fonts(fonts) style = Style() fingerprint = family.fingerprint(style) - renderer = renderer_fingerprint(family, style) if cache_dir else '' - reused = 0 + reused, missing, rejected = 0, 0, 0 for index, path in enumerate(paths, 1): document = json.loads(path.read_text(encoding='utf-8')) try: @@ -49,8 +48,12 @@ def engrave_scores(paths: list[Path], root: Path, fonts: Path, cache_dir: Path | if destination.exists(): shutil.rmtree(destination) entry = cache_dir / score_fingerprint(document, renderer) if cache_dir else None + present = entry is not None and (entry.exists() or entry.is_symlink()) pages = restore_pages(entry, destination) if entry else None if pages is None: + if entry: + rejected += int(present) + missing += int(not present) if destination.exists(): shutil.rmtree(destination) destination.mkdir() @@ -73,10 +76,10 @@ def engrave_scores(paths: list[Path], root: Path, fonts: Path, cache_dir: Path | bundle['playback'] = document['playback'] path.write_text(json.dumps(bundle, indent=2, ensure_ascii=False) + '\n', encoding='utf-8') if progress and (index % 50 == 0 or index == len(paths)): - print(f'Engraving worker {worker}: {index}/{len(paths)} scores, {reused} reused', flush=True) + print(f'Engraving worker {worker}: {index}/{len(paths)} scores, {reused} reused, {missing} missing, {rejected} rejected', flush=True) except ValueError as error: raise EngravingError(f'{path.relative_to(root)}: {error}') from error - return len(paths), reused + return len(paths), reused, missing, rejected def engrave_tree(root: Path, fonts: Path, *, cache_dir: Path | None = None, jobs: int = 1, progress: bool = False) -> None: if type(jobs) is not int or not 1 <= jobs <= 16: @@ -86,12 +89,21 @@ def engrave_tree(root: Path, fonts: Path, *, cache_dir: Path | None = None, jobs paths = sorted(root.rglob('*.json')) if not paths: return + renderer = renderer_fingerprint(Fonts(fonts), Style()) if cache_dir else '' + if cache_dir: + cache_dir = renderer_namespace(cache_dir.resolve(), renderer) + if progress: + state = 'warm namespace' if cache_dir.is_dir() else 'cold namespace' + print(f'Engraving cache enabled: {cache_dir} ({state}; renderer {renderer})', flush=True) + elif progress: + print('Engraving cache disabled', flush=True) jobs = min(jobs, len(paths)) if jobs == 1: - results = [engrave_scores(paths, root, fonts, cache_dir, progress, 1)] + results = [engrave_scores(paths, root, fonts, cache_dir, progress, 1, renderer)] else: with ProcessPoolExecutor(max_workers=jobs, mp_context=multiprocessing.get_context('spawn')) as pool: - futures = [pool.submit(engrave_scores, paths[index::jobs], root, fonts, cache_dir, progress, index + 1) for index in range(jobs)] + futures = [pool.submit(engrave_scores, paths[index::jobs], root, fonts, cache_dir, progress, index + 1, renderer) for index in range(jobs)] results = [future.result() for future in futures] if progress: - print(f'Engraving complete: {sum(total for total, _ in results)} scores, {sum(reused for _, reused in results)} reused', flush=True) + total, reused, missing, rejected = (sum(row[index] for row in results) for index in range(4)) + print(f'Engraving complete: {total} scores, {reused} reused, {missing} missing, {rejected} rejected', flush=True) diff --git a/generator/engraving/cache.py b/generator/engraving/cache.py index 08b7e38..6e12916 100644 --- a/generator/engraving/cache.py +++ b/generator/engraving/cache.py @@ -5,20 +5,27 @@ import importlib.metadata import json from pathlib import Path import shutil -import sys +import platform import tempfile from PIL import features -def renderer_fingerprint(fonts, style) -> str: +def renderer_identity(fonts, style) -> dict: root = Path(__file__).resolve().parents[1] versions = {name: importlib.metadata.version(name) for name in ('Pillow', 'fonttools', 'aggdraw')} versions.update({name: features.version(name) for name in features.get_supported()}) - digest = hashlib.sha256(json.dumps([fonts.fingerprint(style), sys.version, versions], sort_keys=True).encode()) - for path in sorted([*root.joinpath('engraving').glob('*.py'), root / 'model.py', root / 'harmony.py', root / 'requirements.txt']): - digest.update(path.relative_to(root).as_posix().encode()) - digest.update(path.read_bytes()) - return digest.hexdigest() + sources = [*root.joinpath('engraving').glob('*.py'), root / 'model.py', root / 'harmony.py', root / 'requirements.txt'] + return {'fonts_style': fonts.fingerprint(style), 'python': {'implementation': platform.python_implementation(), 'version': platform.python_version()}, 'dependencies': versions, + 'sources': {path.relative_to(root).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() for path in sorted(sources)}} + +def renderer_fingerprint(fonts, style) -> str: + return hashlib.sha256(json.dumps(renderer_identity(fonts, style), sort_keys=True).encode()).hexdigest() + +def renderer_namespace(cache: Path, renderer: str) -> Path: + namespace = cache / 'v2' / renderer + if namespace.parent.is_symlink() or namespace.is_symlink(): + raise ValueError('engraving cache namespaces must not be symlinks') + return namespace def score_fingerprint(document: dict, renderer: str) -> str: music = {key: value for key, value in document.items() if key != 'playback'} diff --git a/generator/score.py b/generator/score.py index ce8aa2c..273fc08 100644 --- a/generator/score.py +++ b/generator/score.py @@ -10,6 +10,9 @@ from generator.model import SCHEMA_VERSION, validate_document def load_json(path: Path) -> dict: document = json.loads(path.read_text(encoding='utf-8')) + return validate_json(document, path) + +def validate_json(document: dict, path: Path) -> dict: if 'music' in document: from generator.engraving import ENGINE_NAME, ENGINE_VERSION, PAGE_HEIGHT, PAGE_WIDTH, check_page presentation = document.get('presentation', {}) @@ -122,8 +125,8 @@ def count_json_files(root: Path) -> int: def validate_generated_tree(root: Path) -> None: audio_cache = {} for path in root.rglob('*.json'): - load_json(path) document = json.loads(path.read_text(encoding='utf-8')) + validate_json(document, path) if 'playback' in document: from generator.playback import validate_assets validate_assets(document['playback'], root, audio_cache) diff --git a/include/vamp_protocol/page_control.hpp b/include/vamp_protocol/page_control.hpp index ddac21d..69d07f9 100644 --- a/include/vamp_protocol/page_control.hpp +++ b/include/vamp_protocol/page_control.hpp @@ -7,8 +7,6 @@ namespace vamp_protocol { constexpr std::char_t root_path[] = "/"; constexpr std::char_t status_path[] = "/api/status"; -constexpr std::char_t next_page_path[] = "/api/page/next"; -constexpr std::char_t previous_page_path[] = "/api/page/previous"; constexpr std::char_t remote_next_path[] = "/api/remote/next"; constexpr std::char_t remote_previous_path[] = "/api/remote/previous"; constexpr std::char_t remote_select_path[] = "/api/remote/select"; @@ -22,7 +20,6 @@ enum class page_command { }; auto page_command_path(vamp_protocol::page_command command) -> std::char_t const *; -auto page_command_legacy_path(vamp_protocol::page_command command) -> std::char_t const *; auto page_command_name(vamp_protocol::page_command command) -> std::char_t const *; auto page_command_display_label(vamp_protocol::page_command command) -> std::char_t const *; diff --git a/modulus.json b/modulus.json index 7711c37..5a66c0e 100644 --- a/modulus.json +++ b/modulus.json @@ -44,7 +44,10 @@ }, "default_target": "x86_64_gcc12_debug", "docker": { - "image": "product-vamp-modulus-build:dev" + "image": "product-vamp-modulus-build:dev", + "dockerfile": "package/Dockerfile.build", + "inputs": ["generator/requirements.txt"], + "socket_commands": ["build", "package"] }, "remote": { "name": "khodo", diff --git a/modulus.py b/modulus.py index 7b23db9..856cec7 100755 --- a/modulus.py +++ b/modulus.py @@ -1,18 +1,15 @@ #!/usr/bin/env python3 +# Canonical consumer bootstrap. Copy this file unchanged to a project's modulus.py. import json -import hashlib import os import pathlib import shlex import subprocess import sys -import time PACKAGE_REF = "modulus/latest" CONAN_VERSION = "2.26.2" FETCH_IMAGE = "modulus-fetch:dev" -DEFAULT_BUILD_BASE_IMAGE = "debian:trixie-slim" -PICO_SDK_VERSION = "2.3.0" COMMANDS = ( "configure", "build", @@ -29,6 +26,7 @@ RUNTIME_DIR = COMMON_DIR / "modulus" CONAN_HOME_DIR = COMMON_DIR / "conan_cache" DEFAULT_REMOTE_NAME = "khodo" DEFAULT_REMOTE_URL = "https://git.thyme-labs.com/api/packages/khodo/conan" +DEFAULT_DOCKER_MARKER = "/opt/modulus-build-image" def repo_root(): root = pathlib.Path(__file__).resolve().parent @@ -63,7 +61,6 @@ def runtime_ready(root): (root / TOOLS_DIR / "configure.py").is_file() and (root / TOOLS_DIR / "lsp.py").is_file() and (root / TOOLS_DIR / "clangd.py").is_file() - and (root / TOOLS_DIR / "run.py").is_file() and (root / RUNTIME_DIR / "modulus" / "cli.py").is_file() and (root / RUNTIME_DIR / "modulus" / "export.py").is_file() and (root / TOOLS_DIR / "format.py").is_file() @@ -71,18 +68,19 @@ def runtime_ready(root): and (root / RUNTIME_DIR / "modulus" / "format_layout.py").is_file() ) -def optional_credentials(): - user = os.environ.get("CONAN_USER") - token = os.environ.get("CONAN_TOKEN") - if bool(user) != bool(token): - raise SystemExit("CONAN_USER and CONAN_TOKEN must either both be set or both be unset") - return user, token - def env_flag(name): return os.environ.get(name, "").lower() in {"1", "true", "yes", "on"} +def docker_marker(root): + config = read_json(root / "modulus.json") + marker = config.get( + "tooling", + {}, + ).get("docker", {}).get("marker", DEFAULT_DOCKER_MARKER) + return pathlib.Path(str(marker)) + def direct_mode(root): - return env_flag("MODULUS_DIRECT") or project_docker_marker(root).exists() + return env_flag("MODULUS_DIRECT") or docker_marker(root).exists() def run(command, input_text=None, capture=False, display_command=None, env=None, check=True): visible = display_command if display_command is not None else command @@ -104,20 +102,8 @@ def run(command, input_text=None, capture=False, display_command=None, env=None, raise SystemExit(completed.returncode) return completed -def run_with_retries(command, attempts=3, input_text=None, display_command=None, env=None): - completed = None - for attempt in range(1, attempts + 1): - completed = run(command, input_text=input_text, display_command=display_command, env=env, check=False) - if completed.returncode == 0: - return completed - if attempt != attempts: - delay = attempt * 20 - print(f"command failed with exit code {completed.returncode}; retrying in {delay}s") - time.sleep(delay) - raise SystemExit(completed.returncode if completed is not None else 1) - def fetch_dockerfile(): - return f"""FROM {DEFAULT_BUILD_BASE_IMAGE} + return f"""FROM debian:bookworm-slim ENV DEBIAN_FRONTEND=noninteractive @@ -137,112 +123,23 @@ RUN python -m pip install --no-cache-dir conan=={CONAN_VERSION} WORKDIR /workspace """ -def build_requirements_digest(): - return hashlib.sha256((repo_root() / "generator/requirements.txt").read_bytes()).hexdigest() - -def product_build_dockerfile(): - return f"""FROM {DEFAULT_BUILD_BASE_IMAGE} - -ENV DEBIAN_FRONTEND=noninteractive - -RUN dpkg --add-architecture arm64 - -RUN apt-get update \\ - && apt-get install -y --no-install-recommends \\ - bash \\ - build-essential \\ - ca-certificates \\ - clangd-19 \\ - cmake \\ - curl \\ - docker-buildx \\ - docker-cli \\ - docker.io \\ - g++-12 \\ - g++-12-aarch64-linux-gnu \\ - gcc-12 \\ - gcc-12-aarch64-linux-gnu \\ - gcc-arm-none-eabi \\ - git \\ - libdrm-dev \\ - libdrm-dev:arm64 \\ - libegl-dev \\ - libegl-dev:arm64 \\ - libfribidi0 \\ - libgbm-dev \\ - libgbm-dev:arm64 \\ - libgles-dev \\ - libgles-dev:arm64 \\ - libglvnd-dev \\ - libglvnd-dev:arm64 \\ - libnewlib-arm-none-eabi \\ - libstdc++-arm-none-eabi-newlib \\ - ninja-build \\ - pkg-config \\ - python3 \\ - python3-venv \\ - && ln -sf /usr/bin/clangd-19 /usr/local/bin/clangd \\ - && rm -rf /var/lib/apt/lists/* - -ENV CC=/usr/bin/gcc-12 -ENV CXX=/usr/bin/g++-12 -ENV VAMP_PICO_SDK_VERSION={PICO_SDK_VERSION} -ENV PICO_SDK_PATH=/opt/pico-sdk - -RUN git clone \\ - --depth 1 \\ - --branch "$VAMP_PICO_SDK_VERSION" \\ - --recurse-submodules \\ - --shallow-submodules \\ - https://github.com/raspberrypi/pico-sdk.git \\ - "$PICO_SDK_PATH" - -RUN python3 -m venv /opt/modulus-python - -ENV PATH=/opt/modulus-python/bin:$PATH - -COPY generator/requirements.txt /tmp/vamp-build-requirements.txt -ARG MODULUS_TOOLS=build/build_tools -COPY ${{MODULUS_TOOLS}}/docker/format-requirements.txt /tmp/modulus-format-requirements.txt - -RUN python -m pip install --no-cache-dir \\ - conan=={CONAN_VERSION} -r /tmp/vamp-build-requirements.txt -r /tmp/modulus-format-requirements.txt - -LABEL org.khodo.product-vamp.build-requirements="{build_requirements_digest()}" - -WORKDIR /workspace - -RUN mkdir -p /opt && touch /opt/modulus-build-image -""" - -def sync_product_build_dockerfile(root): - dockerfile = root / TOOLS_DIR / "docker" / "Dockerfile" - if not dockerfile.parent.is_dir(): +def ensure_fetch_image(offline=False): + inspected = subprocess.run( + [ + "docker", + "image", + "inspect", + FETCH_IMAGE, + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + if inspected.returncode == 0 and (offline or not docker_platform_args()): return - desired = product_build_dockerfile() - try: - current = dockerfile.read_text(encoding="utf-8") - except OSError: - current = None - if current != desired: - dockerfile.write_text(desired, encoding="utf-8") - -def ensure_fetch_image(): - if not docker_platform_args(): - inspected = subprocess.run( - [ - "docker", - "image", - "inspect", - FETCH_IMAGE, - ], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - check=False, - ) - if inspected.returncode == 0: - return - run_with_retries( + if offline: + raise SystemExit('offline bootstrap requires the existing Modulus fetch image or MODULUS_DIRECT=1 with Conan installed') + run( [ "docker", "build", @@ -257,6 +154,10 @@ def ensure_fetch_image(): def in_container(): return pathlib.Path("/.dockerenv").exists() +def docker_platform_args(): + platform = os.environ.get("MODULUS_DOCKER_PLATFORM", "").strip() + return ["--platform", platform] if platform else [] + def docker_workspace_args(root): if in_container(): container = os.environ.get("HOSTNAME", "").strip() @@ -271,43 +172,26 @@ def docker_workspace_args(root): f"{root}:{root}", ] -def docker_socket_args(): - socket = pathlib.Path("/var/run/docker.sock") - if not socket.is_socket(): - return [] - args = [ - "--volume", - "/var/run/docker.sock:/var/run/docker.sock", - ] - try: - args.extend(["--group-add", str(socket.stat().st_gid)]) - except OSError: - pass - return args - -def docker_platform_args(): - platform = os.environ.get("MODULUS_DOCKER_PLATFORM", "").strip() - if not platform: - return [] - return [ - "--platform", - platform, - ] - def fetch_env(root): env = os.environ.copy() + for name in list(env): + if name in {"CONAN_USER", "CONAN_TOKEN"} or name.startswith(("CONAN_LOGIN_USERNAME", "CONAN_PASSWORD")): + del env[name] env["CONAN_HOME"] = str(root / CONAN_HOME_DIR) + env["CONAN_NON_INTERACTIVE"] = "1" return env -def run_fetch(root, command, capture=False, display_command=None, check=True): +def run_fetch(root, command, capture=False, display_command=None, check=True, offline=False): if direct_mode(root): return run(command, capture=capture, display_command=display_command, env=fetch_env(root), check=check) - ensure_fetch_image() + ensure_fetch_image(offline=offline) docker_command = [ "docker", "run", "--rm", "--interactive", + *(["--pull=never", "--network=none"] if offline else []), + *docker_platform_args(), "--user", f"{os.getuid()}:{os.getgid()}", "--env", @@ -315,13 +199,8 @@ def run_fetch(root, command, capture=False, display_command=None, check=True): "--env", "CONAN_HOME", "--env", - "CONAN_USER", - "--env", - "CONAN_TOKEN", - "--env", - "DOCKER_HOST", + "CONAN_NON_INTERACTIVE", *docker_workspace_args(root), - *docker_socket_args(), "--workdir", str(root), FETCH_IMAGE, @@ -330,7 +209,6 @@ def run_fetch(root, command, capture=False, display_command=None, check=True): return run(docker_command, capture=capture, display_command=display_command, env=fetch_env(root), check=check) def configure_remote(root, remote_name, remote_url): - user, token = optional_credentials() run_fetch( root, [ @@ -342,29 +220,7 @@ def configure_remote(root, remote_name, remote_url): remote_url, ], ) - if not user and not token: - return - run_fetch( - root, - [ - "conan", - "remote", - "login", - remote_name, - user, - "-p", - token, - ], - display_command=[ - "conan", - "remote", - "login", - remote_name, - user, - "-p", - "", - ], - ) + run_fetch(root, ["conan", "remote", "logout", remote_name]) def package_refs(data): refs = [] @@ -401,46 +257,26 @@ def package_refs(data): refs.append(f"{PACKAGE_REF}#{recipe_revision}:{package_id}") return sorted(refs) -def remove_cached_modulus(root): - run_fetch( - root, - [ - "conan", - "remove", - PACKAGE_REF + ":*", - "-c", - ], - check=False, - ) - -def download_modulus(root, remote_name): - remove_cached_modulus(root) - run_fetch( - root, - [ - "conan", - "download", - PACKAGE_REF + ":*", - "-r", - remote_name, - ], - ) +def find_modulus(root, remote_name=None): completed = run_fetch( root, [ "conan", "list", - PACKAGE_REF + ":*", - "-c", + PACKAGE_REF + "#latest:*#latest", + *(["-r", remote_name] if remote_name else ["-c"]), "-f", "json", ], capture=True, + offline=remote_name is None, ) refs = package_refs(json.loads(completed.stdout)) if not refs: - raise SystemExit(f"no downloaded {PACKAGE_REF} package found in the Conan cache") + raise SystemExit(f"no {PACKAGE_REF} package available in {remote_name or 'the local Conan cache'}") package_ref = refs[-1] + if remote_name: + run_fetch(root, ["conan", "download", package_ref, "-r", remote_name]) completed = run_fetch( root, [ @@ -450,40 +286,25 @@ def download_modulus(root, remote_name): package_ref, ], capture=True, + offline=remote_name is None, ) package_root = pathlib.Path(completed.stdout.strip()) if not (package_root / "modulus" / "export.py").is_file() or not (package_root / "tools").is_dir(): raise SystemExit(f"downloaded {PACKAGE_REF} package does not contain Modulus export files: {package_root}") return package_root -def materialize_runtime(root): +def materialize_runtime(root, no_remote=False): remote_name, remote_url = remote_config(root) - configure_remote(root, remote_name, remote_url) - package_root = download_modulus(root, remote_name) + if not no_remote: + configure_remote(root, remote_name, remote_url) + package_root = find_modulus(root, None if no_remote else remote_name) sys.path.insert(0, str(package_root)) import modulus.export modulus.export.sync_project(root, package_root) -def ensure_runtime(root): - if not runtime_ready(root): - materialize_runtime(root) - sync_product_build_dockerfile(root) - -def project_docker_marker(root): - config = read_json(root / "modulus.json") - tooling = config.get("tooling", {}) - if not isinstance(tooling, dict): - tooling = {} - docker = tooling.get("docker", {}) - if not isinstance(docker, dict): - docker = {} - return pathlib.Path(str(docker.get("marker") or "/opt/modulus-build-image")) - -def run_configure(argv): - return delegate("configure", argv) - -def run_package(argv): - return delegate("package", argv) +def ensure_runtime(root, update=False, no_remote=False): + if update or not runtime_ready(root): + materialize_runtime(root, no_remote=no_remote) def delegate(command, argv): root = repo_root() @@ -492,19 +313,9 @@ def delegate(command, argv): print(f"Modulus {command} tooling is missing; run ./modulus.py configure first", file=sys.stderr) return 1 else: - ensure_runtime(root) + ensure_runtime(root, update=command == "configure" and "--update" in argv, no_remote="--no-remote" in argv) sys.path.insert(0, str(root / RUNTIME_DIR)) import modulus.cli - original_docker_run = modulus.cli.docker_run - - def docker_run_with_host_socket(context, command, docker_args=None, env_overrides=None): - merged_docker_args = [ - *docker_socket_args(), - *(docker_args or []), - ] - return original_docker_run(context, command, docker_args=merged_docker_args, env_overrides=env_overrides) - - modulus.cli.docker_run = docker_run_with_host_socket previous_argv = sys.argv previous_cwd = pathlib.Path.cwd() sys.argv = [ @@ -515,7 +326,6 @@ def delegate(command, argv): try: return modulus.cli.main(command) finally: - modulus.cli.docker_run = original_docker_run os.chdir(previous_cwd) sys.argv = previous_argv @@ -526,10 +336,6 @@ def main(argv): command = argv[0] if command not in COMMANDS: raise SystemExit(f"unknown Modulus command: {command}") - if command == "configure": - return run_configure(argv[1:]) - if command == "package": - return run_package(argv[1:]) return delegate(command, argv[1:]) if __name__ == "__main__": diff --git a/package/Dockerfile.build b/package/Dockerfile.build new file mode 100644 index 0000000..401db0e --- /dev/null +++ b/package/Dockerfile.build @@ -0,0 +1,69 @@ +FROM debian:trixie-slim + +ENV DEBIAN_FRONTEND=noninteractive + +RUN dpkg --add-architecture arm64 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + bash \ + build-essential \ + ca-certificates \ + clangd-19 \ + cmake \ + curl \ + docker-buildx \ + docker-cli \ + docker.io \ + g++-12 \ + g++-12-aarch64-linux-gnu \ + gcc-12 \ + gcc-12-aarch64-linux-gnu \ + gcc-arm-none-eabi \ + git \ + libdrm-dev \ + libdrm-dev:arm64 \ + libegl-dev \ + libegl-dev:arm64 \ + libfribidi0 \ + libgbm-dev \ + libgbm-dev:arm64 \ + libgles-dev \ + libgles-dev:arm64 \ + libglvnd-dev \ + libglvnd-dev:arm64 \ + libnewlib-arm-none-eabi \ + libstdc++-arm-none-eabi-newlib \ + ninja-build \ + pkg-config \ + python3 \ + python3-venv \ + && ln -sf /usr/bin/clangd-19 /usr/local/bin/clangd \ + && rm -rf /var/lib/apt/lists/* + +ENV CC=/usr/bin/gcc-12 +ENV CXX=/usr/bin/g++-12 +ENV VAMP_PICO_SDK_VERSION=2.3.0 +ENV PICO_SDK_PATH=/opt/pico-sdk + +RUN git clone \ + --depth 1 \ + --branch "$VAMP_PICO_SDK_VERSION" \ + --recurse-submodules \ + --shallow-submodules \ + https://github.com/raspberrypi/pico-sdk.git \ + "$PICO_SDK_PATH" + +RUN python3 -m venv /opt/modulus-python + +ENV PATH=/opt/modulus-python/bin:$PATH + +COPY generator/requirements.txt /tmp/vamp-build-requirements.txt +ARG MODULUS_TOOLS=build/build_tools +COPY ${MODULUS_TOOLS}/docker/format-requirements.txt /tmp/modulus-format-requirements.txt + +RUN python -m pip install --no-cache-dir \ + conan==2.26.2 -r /tmp/vamp-build-requirements.txt -r /tmp/modulus-format-requirements.txt +WORKDIR /workspace + +RUN mkdir -p /opt && touch /opt/modulus-build-image diff --git a/src/vamp_protocol/page_control.cpp b/src/vamp_protocol/page_control.cpp index 0460196..91891e6 100644 --- a/src/vamp_protocol/page_control.cpp +++ b/src/vamp_protocol/page_control.cpp @@ -13,18 +13,6 @@ auto vamp_protocol::page_command_path(vamp_protocol::page_command command) -> st return vamp_protocol::remote_next_path; } -auto vamp_protocol::page_command_legacy_path(vamp_protocol::page_command command) -> std::char_t const * { - switch (command) { - case vamp_protocol::page_command::PREVIOUS: - return vamp_protocol::previous_page_path; - case vamp_protocol::page_command::NEXT: - return vamp_protocol::next_page_path; - case vamp_protocol::page_command::SELECT: - return vamp_protocol::remote_select_path; - } - return vamp_protocol::next_page_path; -} - auto vamp_protocol::page_command_name(vamp_protocol::page_command command) -> std::char_t const * { switch (command) { case vamp_protocol::page_command::PREVIOUS: diff --git a/src/vamp_zero/config.cpp b/src/vamp_zero/config.cpp index 8cb5f77..57e2e7d 100644 --- a/src/vamp_zero/config.cpp +++ b/src/vamp_zero/config.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -6,9 +5,7 @@ #include #include -#include #include -#include #include #include @@ -22,93 +19,49 @@ constexpr auto json_options = modulus::json_config::options{ .not_found_code = static_cast(vamp_zero::error_code::NOT_FOUND), }; +constexpr auto field_options = modulus::json_config::options{ + .category = "product-vamp", + .invalid_argument_code = static_cast(vamp_zero::error_code::INVALID_ARGUMENT), + .not_found_code = static_cast(vamp_zero::error_code::NOT_FOUND), +}; + auto invalid(std::string message) -> modulus::error { return vamp_zero::make_error(vamp_zero::error_code::INVALID_ARGUMENT, std::move(message)); } -auto require_object(boost::json::value const &value, std::string_view name) -> std::expected { - if (!value.is_object()) { - return std::unexpected(invalid(std::string{name} + " must be an object")); - } - return &value.as_object(); -} - auto optional_object(boost::json::object const &object, std::string_view field) -> std::expected { auto found = object.find(field); if (found == object.end()) { return nullptr; } - return require_object(found->value(), field); + return modulus::json_config::require_object(found->value(), field, field_options); } auto optional_string(boost::json::object const &object, std::string_view field, std::string fallback) -> std::expected { - auto found = object.find(field); - if (found == object.end()) { - return fallback; - } - if (!found->value().is_string()) { - return std::unexpected(invalid(std::string{field} + " must be a string")); - } - auto value = std::string{found->value().as_string()}; - if (value.empty()) { + auto value = modulus::json_config::optional_string_field(object, field, std::move(fallback), field_options); + if (value && value->empty() && object.contains(field)) { return std::unexpected(invalid(std::string{field} + " must not be empty")); } return value; } -auto optional_bool(boost::json::object const &object, std::string_view field, std::bool_t fallback) -> std::expected { - auto found = object.find(field); - if (found == object.end()) { - return fallback; - } - if (!found->value().is_bool()) { - return std::unexpected(invalid(std::string{field} + " must be a boolean")); - } - return found->value().as_bool(); -} - auto optional_uint32(boost::json::object const &object, std::string_view field, std::uint32_t fallback, std::uint32_t min_value, std::uint32_t max_value) -> std::expected { - auto found = object.find(field); - if (found == object.end()) { - return fallback; + auto value = modulus::json_config::optional_uint64_field(object, field, fallback, max_value, field_options); + if (!value) { + return std::unexpected(value.error()); } - auto value = std::uint64_t{}; - if (found->value().is_uint64()) { - value = found->value().as_uint64(); - } else if (found->value().is_int64()) { - auto signed_value = found->value().as_int64(); - if (signed_value < 0) { - return std::unexpected(invalid(std::string{field} + " must not be negative")); - } - value = static_cast(signed_value); - } else { - return std::unexpected(invalid(std::string{field} + " must be an integer")); - } - if (value < min_value || value > max_value) { + if (*value < min_value) { return std::unexpected(invalid(std::string{field} + " is out of range")); } - return static_cast(value); + return static_cast(*value); } auto optional_float(boost::json::object const &object, std::string_view field, std::float32_t fallback, std::float32_t min_value, std::float32_t max_value) -> std::expected { - auto found = object.find(field); - if (found == object.end()) { - return fallback; + auto value = modulus::json_config::optional_float64_field(object, field, fallback, min_value, max_value, field_options); + if (!value) { + return std::unexpected(value.error()); } - auto value = std::float64_t{}; - if (found->value().is_double()) { - value = found->value().as_double(); - } else if (found->value().is_uint64()) { - value = static_cast(found->value().as_uint64()); - } else if (found->value().is_int64()) { - value = static_cast(found->value().as_int64()); - } else { - return std::unexpected(invalid(std::string{field} + " must be a number")); - } - if (!std::isfinite(value) || value < min_value || value > max_value) { - return std::unexpected(invalid(std::string{field} + " is out of range")); - } - return static_cast(value); + return static_cast(*value); } auto parse_runtime(boost::json::object const &root, vamp_zero::runtime_config fallback) -> std::expected { @@ -119,8 +72,8 @@ auto parse_runtime(boost::json::object const &root, vamp_zero::runtime_config fa if (*object == nullptr) { return fallback; } - auto headless = optional_bool(**object, "headless", fallback.headless); - auto exit_after_startup = optional_bool(**object, "exit_after_startup", fallback.exit_after_startup); + auto headless = modulus::json_config::optional_bool_field(**object, "headless", fallback.headless, field_options); + auto exit_after_startup = modulus::json_config::optional_bool_field(**object, "exit_after_startup", fallback.exit_after_startup, field_options); auto assets_path = optional_string(**object, "assets_path", fallback.assets_path); auto heartbeat_path = optional_string(**object, "heartbeat_path", fallback.heartbeat_path); if (!headless) { @@ -218,21 +171,12 @@ auto parse_audio(boost::json::object const &root, vamp_zero::audio_config fallba if (*object == nullptr) { return fallback; } - auto enabled = optional_bool(**object, "enabled", fallback.enabled); - auto playback_enabled = optional_bool(**object, "playback_enabled", fallback.playback_enabled); - auto legacy_track = optional_bool(**object, "track_enabled", fallback.playback_enabled); - if (!legacy_track) { - return std::unexpected(legacy_track.error()); - } if ((*object)->if_contains("track_enabled")) { - if ((*object)->if_contains("playback_enabled") && playback_enabled && *playback_enabled != *legacy_track) { - return std::unexpected(invalid("audio.playback_enabled contradicts legacy track_enabled")); - } - if (!(*object)->if_contains("playback_enabled")) { - playback_enabled = legacy_track; - } + return std::unexpected(invalid("audio.track_enabled is no longer supported; use audio.playback_enabled")); } - auto metronome_enabled = optional_bool(**object, "metronome_enabled", fallback.metronome_enabled); + auto enabled = modulus::json_config::optional_bool_field(**object, "enabled", fallback.enabled, field_options); + auto playback_enabled = modulus::json_config::optional_bool_field(**object, "playback_enabled", fallback.playback_enabled, field_options); + auto metronome_enabled = modulus::json_config::optional_bool_field(**object, "metronome_enabled", fallback.metronome_enabled, field_options); auto metronome_bpm = optional_uint32(**object, "metronome_bpm", fallback.metronome_bpm, 1U, 400U); auto metronome_min_bpm = optional_uint32(**object, "metronome_min_bpm", fallback.metronome_min_bpm, 1U, 400U); auto metronome_max_bpm = optional_uint32(**object, "metronome_max_bpm", fallback.metronome_max_bpm, 1U, 400U); @@ -293,7 +237,7 @@ auto parse_audio(boost::json::object const &root, vamp_zero::audio_config fallba } auto vamp_zero::parse_config(boost::json::value const &value) -> std::expected { - auto root = require_object(value, "Product Vamp config root"); + auto root = modulus::json_config::require_object(value, "Product Vamp config root", field_options); if (!root) { return std::unexpected(root.error()); } diff --git a/src/vamp_zero/portal.cpp b/src/vamp_zero/portal.cpp index ac907a7..d9168a8 100644 --- a/src/vamp_zero/portal.cpp +++ b/src/vamp_zero/portal.cpp @@ -196,13 +196,5 @@ auto vamp_zero::make_router(vamp_zero::control_state &state) -> ligo::service_ro auto updated = state.apply(vamp_protocol::page_command::SELECT); co_return remote_response(updated); }); - router.add_route("POST", vamp_protocol::next_page_path, [&state](ligo::request_context, boost::json::value) -> boost::asio::awaitable> { - auto updated = state.apply(vamp_protocol::page_command::NEXT); - co_return vamp_zero::portal_status_json(state.config(), updated); - }); - router.add_route("POST", vamp_protocol::previous_page_path, [&state](ligo::request_context, boost::json::value) -> boost::asio::awaitable> { - auto updated = state.apply(vamp_protocol::page_command::PREVIOUS); - co_return vamp_zero::portal_status_json(state.config(), updated); - }); return router; } diff --git a/tests/test_build_bootstrap.py b/tests/test_build_bootstrap.py index 152aac6..4bc5f53 100644 --- a/tests/test_build_bootstrap.py +++ b/tests/test_build_bootstrap.py @@ -1,4 +1,4 @@ -import hashlib +import json import importlib.util from pathlib import Path import sys @@ -7,15 +7,15 @@ import unittest from unittest.mock import Mock, patch class BuildBootstrapTests(unittest.TestCase): - def test_requirements_fingerprint_invalidates_cached_build_image(self): + def test_build_recipe_is_declared_and_editor_commands_do_not_bootstrap(self): root = Path(__file__).resolve().parents[1] spec = importlib.util.spec_from_file_location('vamp_bootstrap', root / 'modulus.py') bootstrap = importlib.util.module_from_spec(spec) spec.loader.exec_module(bootstrap) - digest = hashlib.sha256((root / 'generator/requirements.txt').read_bytes()).hexdigest() - self.assertEqual(bootstrap.build_requirements_digest(), digest) - self.assertIn(f'build-requirements="{digest}"', bootstrap.product_build_dockerfile()) - self.assertIn('COPY generator/requirements.txt /tmp/vamp-build-requirements.txt', bootstrap.product_build_dockerfile()) + docker = json.loads((root / 'modulus.json').read_text())['tooling']['docker'] + self.assertEqual(docker['socket_commands'], ['build', 'package']) + self.assertIn('generator/requirements.txt', docker['inputs']) + self.assertIn('COPY generator/requirements.txt /tmp/vamp-build-requirements.txt', (root / docker['dockerfile']).read_text()) package = types.ModuleType('modulus') cli = types.ModuleType('modulus.cli') package.cli = cli diff --git a/tests/test_engraving_cache.py b/tests/test_engraving_cache.py index 324c108..e325437 100644 --- a/tests/test_engraving_cache.py +++ b/tests/test_engraving_cache.py @@ -1,6 +1,8 @@ import argparse +import contextlib import copy import hashlib +import io import json from pathlib import Path import tempfile @@ -9,14 +11,16 @@ from unittest.mock import patch from generator.__main__ import generate_assets from generator.engraving import EngravingError, Fonts, Style, engrave_tree, layout_score -from generator.engraving.cache import renderer_fingerprint, restore_pages, score_fingerprint +from generator.engraving.cache import renderer_fingerprint, renderer_namespace, restore_pages, score_fingerprint +from generator.score import validate_generated_tree from tests.test_musicxml import parse, xml_note, xml_score class EngravingCacheTests(unittest.TestCase): def test_ci_snapshot_tracks_all_generator_inputs_and_keeps_fallback(self): workflow = Path('.gitea/workflows/ci.yml').read_text() self.assertIn("hashFiles('generator/**/*.py', 'generator/*.py', 'data/**/*.xml', 'data/*.xml')", workflow) - self.assertIn('restore-keys: product-vamp-pages-v1-${{ runner.os }}-', workflow) + self.assertIn('python3 -m generator --renderer-fingerprint', workflow) + self.assertIn('restore-keys: product-vamp-pages-v2-${{ steps.renderer.outputs.fingerprint }}-', workflow) self.assertIn("if: steps.engraving-cache.outputs.cache-hit != 'true'", workflow) self.assertIn('key: ${{ steps.engraving-cache.outputs.cache-primary-key }}', workflow) def setUp(self): @@ -36,7 +40,7 @@ class EngravingCacheTests(unittest.TestCase): root = self.root / 'seed' self.write_scores(root, [self.document]) engrave_tree(root, self.fonts, cache_dir=self.cache) - return next(self.cache.iterdir()) + return next(self.cache.rglob('pages.json')).parent def test_cache_hit_preserves_bundle_bytes_across_output_directories(self): cold, warm = self.root / 'cold', self.root / 'warm' self.write_scores(cold, [self.document]) @@ -65,7 +69,7 @@ class EngravingCacheTests(unittest.TestCase): with patch('generator.engraving.layout_score', wraps=layout_score) as render: engrave_tree(root, self.fonts, cache_dir=self.cache) render.assert_called_once() - self.assertEqual(len(list(self.cache.iterdir())), 2) + self.assertEqual(len(list(self.cache.rglob('pages.json'))), 2) def test_renderer_fingerprint_tracks_fonts_style_code_and_dependencies(self): fonts, style = Fonts(self.fonts), Style() expected = renderer_fingerprint(fonts, style) @@ -75,6 +79,8 @@ class EngravingCacheTests(unittest.TestCase): self.assertNotEqual(expected, renderer_fingerprint(fonts, style)) with patch('generator.engraving.cache.importlib.metadata.version', return_value='new-version'): self.assertNotEqual(expected, renderer_fingerprint(fonts, style)) + with patch('generator.engraving.cache.platform.python_version', return_value='0.0.0'): + self.assertNotEqual(expected, renderer_fingerprint(fonts, style)) read_bytes = Path.read_bytes with patch.object(Path, 'read_bytes', lambda path: read_bytes(path) + (b'\n' if path.name == 'raster.py' else b'')): self.assertNotEqual(expected, renderer_fingerprint(fonts, style)) @@ -129,8 +135,46 @@ class EngravingCacheTests(unittest.TestCase): self.write_scores(root, documents) engrave_tree(root, self.fonts, cache_dir=self.cache, jobs=2) self.assertEqual(self.files(serial), self.files(root)) - self.assertEqual(len(list(self.cache.iterdir())), 2) - self.assertFalse(list(self.cache.glob('.engraving-*'))) + self.assertEqual(len(list(self.cache.rglob('pages.json'))), 2) + self.assertFalse(list(self.cache.rglob('.engraving-*'))) + def test_environment_change_gets_new_namespace_and_clear_progress(self): + old = self.seed_cache() + root = self.root / 'new-environment' + self.write_scores(root, [self.document]) + progress = io.StringIO() + with patch('generator.engraving.cache.platform.python_version', return_value='0.0.0'), contextlib.redirect_stdout(progress): + engrave_tree(root, self.fonts, cache_dir=self.cache, progress=True) + self.assertIn('cold namespace', progress.getvalue()) + self.assertIn('0 reused, 1 missing, 0 rejected', progress.getvalue()) + self.assertTrue((old / 'pages.json').is_file()) + self.assertEqual(len(list((self.cache / 'v2').iterdir())), 2) + self.write_scores(root, [self.document]) + progress = io.StringIO() + with patch('generator.engraving.cache.platform.python_version', return_value='0.0.0'), contextlib.redirect_stdout(progress), patch('generator.engraving.layout_score', side_effect=AssertionError('cache miss')): + engrave_tree(root, self.fonts, cache_dir=self.cache, progress=True) + self.assertIn('1 reused, 0 missing, 0 rejected', progress.getvalue()) + def test_namespace_symlink_is_rejected(self): + self.cache.mkdir() + (self.cache / 'v2').symlink_to(self.root, target_is_directory=True) + with self.assertRaisesRegex(ValueError, 'must not be symlinks'): + renderer_namespace(self.cache, 'renderer') + def test_only_changed_score_is_rendered_and_validation_reads_json_once(self): + root = self.root / 'scores' + self.write_scores(root, [self.document, self.document]) + engrave_tree(root, self.fonts, cache_dir=self.cache) + changed = parse(xml_score(xml_note('D'))) + self.write_scores(root, [self.document, changed]) + with patch('generator.engraving.layout_score', wraps=layout_score) as render: + engrave_tree(root, self.fonts, cache_dir=self.cache) + render.assert_called_once() + read_text = Path.read_text + reads = [] + def read(path, *args, **kwargs): + reads.append(path) + return read_text(path, *args, **kwargs) + with patch.object(Path, 'read_text', read): + validate_generated_tree(root) + self.assertCountEqual(reads, sorted(root.glob('*.json'))) def test_invalid_worker_count_and_overlapping_cache_are_rejected(self): root = self.root / 'assets' self.write_scores(root, [self.document]) diff --git a/tests/test_playback.cpp b/tests/test_playback.cpp index 67d6ceb..6403c75 100644 --- a/tests/test_playback.cpp +++ b/tests/test_playback.cpp @@ -120,10 +120,10 @@ BOOST_AUTO_TEST_CASE(playback_worker_discards_canceled_preparation_and_reports_e BOOST_CHECK(!audio.tick(state, loaded, VAMP_TEST_PLAYBACK_ASSETS_DIR)); } -BOOST_AUTO_TEST_CASE(playback_legacy_config_and_invalid_assets) { +BOOST_AUTO_TEST_CASE(playback_rejects_legacy_config_and_invalid_assets) { auto legacy = vamp_zero::parse_config(boost::json::object{{"audio", {{"track_enabled", true}}}}); - BOOST_REQUIRE(legacy); - BOOST_TEST(legacy->audio.playback_enabled); + BOOST_REQUIRE(!legacy); + BOOST_TEST(std::string{legacy.error().message()}.find("use audio.playback_enabled") != std::string::npos); BOOST_CHECK(!vamp_zero::parse_config(boost::json::object{{"audio", {{"track_enabled", true}, {"playback_enabled", false}}}})); auto score = playback_score(); BOOST_CHECK(!vamp_zero::prepare_playback(score, "/missing-vamp-assets", 120U));