Cleanup
ci / checks (push) Failing after 1h6m47s

This commit is contained in:
khodowany
2026-09-24 19:44:05 -05:00
parent 1be98e14aa
commit 3f47c0c5f3
19 changed files with 309 additions and 404 deletions
+8 -5
View File
@@ -97,25 +97,28 @@ jobs:
key: product-vamp-conan-${{ runner.os }}-${{ hashFiles('conanfile.py', 'modulus.json', 'conan/profiles/*') }} key: product-vamp-conan-${{ runner.os }}-${{ hashFiles('conanfile.py', 'modulus.json', 'conan/profiles/*') }}
restore-keys: product-vamp-conan-${{ runner.os }}- restore-keys: product-vamp-conan-${{ runner.os }}-
- name: Configure native release - name: Configure native release
run: python3 modulus.py configure --target x86_64_gcc12_release run: python3 modulus.py configure --target x86_64_gcc12_release --update
- name: Check code style - name: Check code style
run: | run: |
python3 -m pip install --no-cache-dir -r build/build_tools/docker/format-requirements.txt python3 -m pip install --no-cache-dir -r build/build_tools/docker/format-requirements.txt
python3 modulus.py format --check python3 modulus.py format --check
- name: Identify engraving renderer
id: renderer
run: echo "fingerprint=$(python3 -m generator --renderer-fingerprint)" >> "$GITHUB_OUTPUT"
- name: Restore engraving cache - name: Restore engraving cache
id: engraving-cache id: engraving-cache
uses: actions/cache/restore@v4 uses: actions/cache/restore@v4
with: with:
path: build/build_common/engraving-cache path: build/build_common/engraving-cache/v2/${{ steps.renderer.outputs.fingerprint }}
key: product-vamp-pages-v1-${{ runner.os }}-${{ hashFiles('generator/engraving/*.py', 'generator/model.py', 'generator/harmony.py', 'generator/requirements.txt', 'package/assets/fonts/petaluma/*') }}-${{ hashFiles('generator/**/*.py', 'generator/*.py', 'data/**/*.xml', 'data/*.xml') }} key: product-vamp-pages-v2-${{ steps.renderer.outputs.fingerprint }}-${{ hashFiles('generator/**/*.py', 'generator/*.py', 'data/**/*.xml', 'data/*.xml') }}
restore-keys: product-vamp-pages-v1-${{ runner.os }}-${{ hashFiles('generator/engraving/*.py', 'generator/model.py', 'generator/harmony.py', 'generator/requirements.txt', 'package/assets/fonts/petaluma/*') }}- restore-keys: product-vamp-pages-v2-${{ steps.renderer.outputs.fingerprint }}-
- name: Build native release - name: Build native release
run: python3 modulus.py build --target x86_64_gcc12_release run: python3 modulus.py build --target x86_64_gcc12_release
- name: Save engraving cache - name: Save engraving cache
if: steps.engraving-cache.outputs.cache-hit != 'true' if: steps.engraving-cache.outputs.cache-hit != 'true'
uses: actions/cache/save@v4 uses: actions/cache/save@v4
with: with:
path: build/build_common/engraving-cache path: build/build_common/engraving-cache/v2/${{ steps.renderer.outputs.fingerprint }}
key: ${{ steps.engraving-cache.outputs.cache-primary-key }} key: ${{ steps.engraving-cache.outputs.cache-primary-key }}
- name: Test native release - name: Test native release
run: python3 modulus.py test --target x86_64_gcc12_release run: python3 modulus.py test --target x86_64_gcc12_release
+3 -1
View File
@@ -33,7 +33,9 @@ python3 modulus.py build --target x86_64_gcc12_release
`python3 modulus.py run` uses `package/config/vamp.development.json` for a headless readiness check against native build assets. `vamp-zero --once` checks the packaged database and referenced page/audio assets, checks runtime writes and the server bind, then exits without opening audio or display hardware. Missing or incompatible catalogs require rebuilding or reinstalling; they are never repaired at runtime. `--check-config` only validates configuration; `--health-check` probes the running service's `/api/health` endpoint without modifying files. `python3 modulus.py run` uses `package/config/vamp.development.json` for a headless readiness check against native build assets. `vamp-zero --once` checks the packaged database and referenced page/audio assets, checks runtime writes and the server bind, then exits without opening audio or display hardware. Missing or incompatible catalogs require rebuilding or reinstalling; they are never repaired at runtime. `--check-config` only validates configuration; `--health-check` probes the running service's `/api/health` endpoint without modifying files.
Configure/build can read public packages anonymously from the khodo Conan registry. Configure/build read public packages anonymously from the khodo Conan registry, even when the project cache contains an expired login. `configure --update` refreshes both the exported Modulus tooling and Conan dependencies. `--no-remote` uses only locally cached packages. Publishing is the only operation that needs credentials. CMake checks the required Sonus and Luma APIs during configuration and reports stale dependencies early.
`modulus.py` is an unchanged copy of Modulus's canonical bootstrap. The project-specific build image lives in `package/Dockerfile.build`; `tooling.docker` declares its additional inputs and permits Docker socket access only for build/package commands that assemble the Zero image.
Refresh clangd compile databases, including the Pico firmware database, with: Refresh clangd compile databases, including the Pico firmware database, with:
+3 -3
View File
@@ -29,7 +29,7 @@ The API is plain HTTP on the private AP:
- `POST /api/remote/previous` - `POST /api/remote/previous`
- `POST /api/remote/select` - `POST /api/remote/select`
Legacy `/api/page/next` and `/api/page/previous` routes are still accepted for simple manual checks. Page commands use `/api/remote/next` and `/api/remote/previous`; the old `/api/page/*` routes are removed.
The AP setup pins NetworkManager to 2.4 GHz channel 6 with WPA2/RSN and CCMP ciphers so the Pico W can join using `CYW43_AUTH_WPA2_AES_PSK`. The AP setup pins NetworkManager to 2.4 GHz channel 6 with WPA2/RSN and CCMP ciphers so the Pico W can join using `CYW43_AUTH_WPA2_AES_PSK`.
@@ -148,7 +148,7 @@ When grace groups steal from both ends of one donor, their adjustments accumulat
At runtime the shared on-demand score loader serves both display and audio. A background worker validates and loads only the selected score's samples, converts absolute beat positions to rounded sample frames at the menu BPM, and prepares a Sonus sequence. Pitch does not change with BPM, and rounding absolute positions avoids cumulative timing drift. The I/O loop publishes the prepared sequence and polls completion without doing disk I/O or musical interpretation. Sonus mixes preallocated voices from immutable PCM and retires them on control threads; the callback never synthesizes tones or loads files. Limits are one million events, 512 distinct samples, 64 MiB decoded samples per selection, 64 simultaneous voices including releases, and 24 hours at the selected BPM. Exceeding them reports an error rather than dropping notes. At runtime the shared on-demand score loader serves both display and audio. A background worker validates and loads only the selected score's samples, converts absolute beat positions to rounded sample frames at the menu BPM, and prepares a Sonus sequence. Pitch does not change with BPM, and rounding absolute positions avoids cumulative timing drift. The I/O loop publishes the prepared sequence and polls completion without doing disk I/O or musical interpretation. Sonus mixes preallocated voices from immutable PCM and retires them on control threads; the callback never synthesizes tones or loads files. Limits are one million events, 512 distinct samples, 64 MiB decoded samples per selection, 64 simultaneous voices including releases, and 24 hours at the selected BPM. Exceeding them reports an error rather than dropping notes.
`audio.playback_enabled` defaults to false and replaces the old inert `track_enabled` setting. The old config name is accepted for migration, but conflicting values fail. Playback suppresses the metronome without changing its stored toggle. Authored tempo marks remain notation; menu BPM is constant throughout a session. Selecting a practice or performance part starts a new session. Manual page changes never restart or seek. Natural completion, including trailing silence/release, returns to the corresponding part menu with its selection preserved. Leaving the score cancels playback; stale results cannot close a newer score. Errors remain visible rather than being mistaken for completion. `audio.playback_enabled` defaults to false. The obsolete `track_enabled` setting is rejected with a diagnostic naming its replacement. Playback suppresses the metronome without changing its stored toggle. Authored tempo marks remain notation; menu BPM is constant throughout a session. Selecting a practice or performance part starts a new session. Manual page changes never restart or seek. Natural completion, including trailing silence/release, returns to the corresponding part menu with its selection preserved. Leaving the score cancels playback; stale results cannot close a newer score. Errors remain visible rather than being mistaken for completion.
`--once` additionally validates referenced audio without opening display/audio hardware. Runtime packages contain WAVs alongside pages and the SQLite catalog; the source XML and Python dependencies remain build-only. `--once` additionally validates referenced audio without opening display/audio hardware. Runtime packages contain WAVs alongside pages and the SQLite catalog; the source XML and Python dependencies remain build-only.
@@ -174,7 +174,7 @@ Pages are 976x484 pixels with 12-pixel staff spacing, 28-pixel horizontal margin
Generation validates sources and complete staged output before replacing the previous asset directory. Replacement is always complete; deleted sources cannot leave stale JSON, pages, or audio. CMake tracks the input list as well as file contents, so additions and removals regenerate the catalog. Runtime packages contain `catalog.db` with embedded semantic JSON, PNG pages, generated PCM WAVs, and the OTF UI text font, not MusicXML, MuseScore, or Python dependencies. Python test discovery runs model/import/export tests; the normal asset build additionally qualifies and audits every catalog score. Generation validates sources and complete staged output before replacing the previous asset directory. Replacement is always complete; deleted sources cannot leave stale JSON, pages, or audio. CMake tracks the input list as well as file contents, so additions and removals regenerate the catalog. Runtime packages contain `catalog.db` with embedded semantic JSON, PNG pages, generated PCM WAVs, and the OTF UI text font, not MusicXML, MuseScore, or Python dependencies. Python test discovery runs model/import/export tests; the normal asset build additionally qualifies and audits every catalog score.
CI page-cache snapshot keys include every generator Python module and XML source, while retaining a renderer/font-compatible restore prefix. Individual cached pages remain validated by their semantic-content and renderer fingerprints. The page cache uses `v2/<renderer fingerprint>/<score fingerprint>/` namespaces. `python3 -m generator --renderer-fingerprint` reports the same identity used by rendering and CI: source bytes, fonts, style, Python implementation/version, and rendering-library versions. Python compiler/build timestamps are excluded. CI keys append all generator and XML inputs and restore older snapshots only within the same renderer namespace; only that namespace is saved after a successful build. A renderer or environment change intentionally starts cold. Progress reports the resolved namespace and reused, missing, and rejected counts. Old flat cache entries are not migrated and can be removed when builds are idle.
Diagnostics identify the file, part, measure, and unsupported element or invalid relationship. Examples include `part P1, measure 4: unsupported musical element <slur>`, `tie stop has no contiguous matching start`, and `optional-chord marker must identify exactly one harmony`. Diagnostics identify the file, part, measure, and unsupported element or invalid relationship. Examples include `part P1, measure 4: unsupported musical element <slur>`, `tie stop has no contiguous matching start`, and `optional-chord marker must identify exactly one harmony`.
+22
View File
@@ -12,6 +12,28 @@ find_package(ligo CONFIG REQUIRED)
find_package(noto CONFIG REQUIRED) find_package(noto CONFIG REQUIRED)
find_package(luma CONFIG REQUIRED) find_package(luma CONFIG REQUIRED)
find_package(sonus CONFIG REQUIRED) find_package(sonus CONFIG REQUIRED)
include(CheckCXXSourceCompiles)
include(CMakePushCheckState)
cmake_push_check_state(RESET)
set(CMAKE_TRY_COMPILE_CONFIGURATION "${CMAKE_BUILD_TYPE}")
set(CMAKE_REQUIRED_LIBRARIES sonus::sonus luma::luma)
unset(VAMP_DEPENDENCY_APIS_AVAILABLE CACHE)
check_cxx_source_compiles("
#include <sonus.hpp>
#include <luma.hpp>
#include <utility>
static_assert(requires(sonus::engine& engine, sonus::prepared_sequence sequence) {
engine.play_sequence(std::move(sequence));
});
static_assert(requires(luma::application& application, std::string path) {
application.load_texture_uncached(path);
});
int main() { return 0; }
" VAMP_DEPENDENCY_APIS_AVAILABLE)
cmake_pop_check_state()
if(NOT VAMP_DEPENDENCY_APIS_AVAILABLE)
message(FATAL_ERROR "Product Vamp requires current Sonus sequence and Luma uncached-texture APIs. Run ./modulus.py configure --update after publishing the updated dependencies, or create their Conan packages locally before configuring --no-remote. See CMake's configure log for the compiler diagnostic.")
endif()
include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/font.cmake) include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/font.cmake)
execute_process( execute_process(
+5 -1
View File
@@ -19,8 +19,11 @@ class product_vamp_conan(conan.ConanFile):
"with_tests": False, "with_tests": False,
} }
python_requires = "modulus/latest" python_requires = "modulus/latest"
exports = "modulus.json" exports = "modulus.json", "!build", "!.git", "!test_package/build"
exports_sources = ( exports_sources = (
"!build",
"!.git",
"!test_package/build",
".gitea/workflows/ci.yml", ".gitea/workflows/ci.yml",
"CMakeLists.txt", "CMakeLists.txt",
"modulus.py", "modulus.py",
@@ -43,6 +46,7 @@ class product_vamp_conan(conan.ConanFile):
"include/vamp_zero/*.hpp", "include/vamp_zero/*.hpp",
"package/assets/*", "package/assets/*",
"package/assets/**/*", "package/assets/**/*",
"package/Dockerfile.build",
"package/config/*.json", "package/config/*.json",
"package/manifest.json.in", "package/manifest.json.in",
"package/pico/*", "package/pico/*",
+14 -5
View File
@@ -11,7 +11,8 @@ from generator.practice.generate import generate_practice_scores
from generator.score import count_json_files, validate_generated_tree from generator.score import count_json_files, validate_generated_tree
from generator.summary import validate_expected_performance_summary, validate_practice_parts from generator.summary import validate_expected_performance_summary, validate_practice_parts
from generator.parts import source_paths from generator.parts import source_paths
from generator.engraving import engrave_tree from generator.engraving import Fonts, Style, engrave_tree
from generator.engraving.cache import renderer_fingerprint
from generator.playback import compile_tree from generator.playback import compile_tree
from generator.catalog import build_catalog from generator.catalog import build_catalog
@@ -26,14 +27,18 @@ def copy_runtime_fonts(static_assets_dir: Path, output_dir: Path) -> None:
def parse_args() -> argparse.Namespace: def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Generate Product Vamp package score assets.") parser = argparse.ArgumentParser(description="Generate Product Vamp package score assets.")
parser.add_argument("--data-dir", type=Path, required=True) parser.add_argument("--data-dir", type=Path)
parser.add_argument("--output-dir", type=Path, required=True) parser.add_argument("--output-dir", type=Path)
parser.add_argument("--renderer-fingerprint", action="store_true", help="Print the engraving cache identity without generating assets.")
parser.add_argument("--static-assets-dir", type=Path, default=Path("package/assets")) parser.add_argument("--static-assets-dir", type=Path, default=Path("package/assets"))
parser.add_argument("--check", action="store_true") parser.add_argument("--check", action="store_true")
parser.add_argument("--engraving-cache", type=Path) parser.add_argument("--engraving-cache", type=Path)
parser.add_argument("--jobs", type=int, default=1, choices=range(1, 17), metavar="1-16") parser.add_argument("--jobs", type=int, default=1, choices=range(1, 17), metavar="1-16")
parser.add_argument("--progress", action="store_true") parser.add_argument("--progress", action="store_true")
return parser.parse_args() args = parser.parse_args()
if not args.renderer_fingerprint and (args.data_dir is None or args.output_dir is None):
parser.error('--data-dir and --output-dir are required for generation')
return args
def generate_assets(args: argparse.Namespace) -> None: def generate_assets(args: argparse.Namespace) -> None:
data_dir = args.data_dir.resolve() data_dir = args.data_dir.resolve()
@@ -92,7 +97,11 @@ def generate_assets(args: argparse.Namespace) -> None:
print('Generator check passed.') print('Generator check passed.')
def main() -> int: def main() -> int:
generate_assets(parse_args()) args = parse_args()
if args.renderer_fingerprint:
print(renderer_fingerprint(Fonts(args.static_assets_dir / 'fonts' / 'petaluma'), Style()))
else:
generate_assets(args)
return 0 return 0
if __name__ == "__main__": if __name__ == "__main__":
+21 -9
View File
@@ -10,7 +10,7 @@ import shutil
from PIL import Image, ImageChops from PIL import Image, ImageChops
from generator.engraving.fonts import Fonts from generator.engraving.fonts import Fonts
from generator.engraving.cache import renderer_fingerprint, restore_pages, score_fingerprint, store_pages from generator.engraving.cache import renderer_fingerprint, renderer_namespace, restore_pages, score_fingerprint, store_pages
from generator.engraving.layout import layout_score from generator.engraving.layout import layout_score
from generator.engraving.raster import rasterize_page from generator.engraving.raster import rasterize_page
from generator.engraving.types import EngravingError, Style from generator.engraving.types import EngravingError, Style
@@ -35,12 +35,11 @@ def check_page(path: Path) -> None:
if bounds[0] < 4 or bounds[1] < 4 or bounds[2] > PAGE_WIDTH - 4 or bounds[3] > PAGE_HEIGHT - 4: if bounds[0] < 4 or bounds[1] < 4 or bounds[2] > PAGE_WIDTH - 4 or bounds[3] > PAGE_HEIGHT - 4:
raise EngravingError(f'{path.name}: notation reaches the page edge; refusing potentially clipped output') raise EngravingError(f'{path.name}: notation reaches the page edge; refusing potentially clipped output')
def engrave_scores(paths: list[Path], root: Path, fonts: Path, cache_dir: Path | None, progress: bool, worker: int) -> tuple[int, int]: def engrave_scores(paths: list[Path], root: Path, fonts: Path, cache_dir: Path | None, progress: bool, worker: int, renderer: str) -> tuple[int, int, int, int]:
family = Fonts(fonts) family = Fonts(fonts)
style = Style() style = Style()
fingerprint = family.fingerprint(style) fingerprint = family.fingerprint(style)
renderer = renderer_fingerprint(family, style) if cache_dir else '' reused, missing, rejected = 0, 0, 0
reused = 0
for index, path in enumerate(paths, 1): for index, path in enumerate(paths, 1):
document = json.loads(path.read_text(encoding='utf-8')) document = json.loads(path.read_text(encoding='utf-8'))
try: try:
@@ -49,8 +48,12 @@ def engrave_scores(paths: list[Path], root: Path, fonts: Path, cache_dir: Path |
if destination.exists(): if destination.exists():
shutil.rmtree(destination) shutil.rmtree(destination)
entry = cache_dir / score_fingerprint(document, renderer) if cache_dir else None entry = cache_dir / score_fingerprint(document, renderer) if cache_dir else None
present = entry is not None and (entry.exists() or entry.is_symlink())
pages = restore_pages(entry, destination) if entry else None pages = restore_pages(entry, destination) if entry else None
if pages is None: if pages is None:
if entry:
rejected += int(present)
missing += int(not present)
if destination.exists(): if destination.exists():
shutil.rmtree(destination) shutil.rmtree(destination)
destination.mkdir() destination.mkdir()
@@ -73,10 +76,10 @@ def engrave_scores(paths: list[Path], root: Path, fonts: Path, cache_dir: Path |
bundle['playback'] = document['playback'] bundle['playback'] = document['playback']
path.write_text(json.dumps(bundle, indent=2, ensure_ascii=False) + '\n', encoding='utf-8') path.write_text(json.dumps(bundle, indent=2, ensure_ascii=False) + '\n', encoding='utf-8')
if progress and (index % 50 == 0 or index == len(paths)): if progress and (index % 50 == 0 or index == len(paths)):
print(f'Engraving worker {worker}: {index}/{len(paths)} scores, {reused} reused', flush=True) print(f'Engraving worker {worker}: {index}/{len(paths)} scores, {reused} reused, {missing} missing, {rejected} rejected', flush=True)
except ValueError as error: except ValueError as error:
raise EngravingError(f'{path.relative_to(root)}: {error}') from error raise EngravingError(f'{path.relative_to(root)}: {error}') from error
return len(paths), reused return len(paths), reused, missing, rejected
def engrave_tree(root: Path, fonts: Path, *, cache_dir: Path | None = None, jobs: int = 1, progress: bool = False) -> None: def engrave_tree(root: Path, fonts: Path, *, cache_dir: Path | None = None, jobs: int = 1, progress: bool = False) -> None:
if type(jobs) is not int or not 1 <= jobs <= 16: if type(jobs) is not int or not 1 <= jobs <= 16:
@@ -86,12 +89,21 @@ def engrave_tree(root: Path, fonts: Path, *, cache_dir: Path | None = None, jobs
paths = sorted(root.rglob('*.json')) paths = sorted(root.rglob('*.json'))
if not paths: if not paths:
return return
renderer = renderer_fingerprint(Fonts(fonts), Style()) if cache_dir else ''
if cache_dir:
cache_dir = renderer_namespace(cache_dir.resolve(), renderer)
if progress:
state = 'warm namespace' if cache_dir.is_dir() else 'cold namespace'
print(f'Engraving cache enabled: {cache_dir} ({state}; renderer {renderer})', flush=True)
elif progress:
print('Engraving cache disabled', flush=True)
jobs = min(jobs, len(paths)) jobs = min(jobs, len(paths))
if jobs == 1: if jobs == 1:
results = [engrave_scores(paths, root, fonts, cache_dir, progress, 1)] results = [engrave_scores(paths, root, fonts, cache_dir, progress, 1, renderer)]
else: else:
with ProcessPoolExecutor(max_workers=jobs, mp_context=multiprocessing.get_context('spawn')) as pool: with ProcessPoolExecutor(max_workers=jobs, mp_context=multiprocessing.get_context('spawn')) as pool:
futures = [pool.submit(engrave_scores, paths[index::jobs], root, fonts, cache_dir, progress, index + 1) for index in range(jobs)] futures = [pool.submit(engrave_scores, paths[index::jobs], root, fonts, cache_dir, progress, index + 1, renderer) for index in range(jobs)]
results = [future.result() for future in futures] results = [future.result() for future in futures]
if progress: if progress:
print(f'Engraving complete: {sum(total for total, _ in results)} scores, {sum(reused for _, reused in results)} reused', flush=True) total, reused, missing, rejected = (sum(row[index] for row in results) for index in range(4))
print(f'Engraving complete: {total} scores, {reused} reused, {missing} missing, {rejected} rejected', flush=True)
+14 -7
View File
@@ -5,20 +5,27 @@ import importlib.metadata
import json import json
from pathlib import Path from pathlib import Path
import shutil import shutil
import sys import platform
import tempfile import tempfile
from PIL import features from PIL import features
def renderer_fingerprint(fonts, style) -> str: def renderer_identity(fonts, style) -> dict:
root = Path(__file__).resolve().parents[1] root = Path(__file__).resolve().parents[1]
versions = {name: importlib.metadata.version(name) for name in ('Pillow', 'fonttools', 'aggdraw')} versions = {name: importlib.metadata.version(name) for name in ('Pillow', 'fonttools', 'aggdraw')}
versions.update({name: features.version(name) for name in features.get_supported()}) versions.update({name: features.version(name) for name in features.get_supported()})
digest = hashlib.sha256(json.dumps([fonts.fingerprint(style), sys.version, versions], sort_keys=True).encode()) sources = [*root.joinpath('engraving').glob('*.py'), root / 'model.py', root / 'harmony.py', root / 'requirements.txt']
for path in sorted([*root.joinpath('engraving').glob('*.py'), root / 'model.py', root / 'harmony.py', root / 'requirements.txt']): return {'fonts_style': fonts.fingerprint(style), 'python': {'implementation': platform.python_implementation(), 'version': platform.python_version()}, 'dependencies': versions,
digest.update(path.relative_to(root).as_posix().encode()) 'sources': {path.relative_to(root).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() for path in sorted(sources)}}
digest.update(path.read_bytes())
return digest.hexdigest() def renderer_fingerprint(fonts, style) -> str:
return hashlib.sha256(json.dumps(renderer_identity(fonts, style), sort_keys=True).encode()).hexdigest()
def renderer_namespace(cache: Path, renderer: str) -> Path:
namespace = cache / 'v2' / renderer
if namespace.parent.is_symlink() or namespace.is_symlink():
raise ValueError('engraving cache namespaces must not be symlinks')
return namespace
def score_fingerprint(document: dict, renderer: str) -> str: def score_fingerprint(document: dict, renderer: str) -> str:
music = {key: value for key, value in document.items() if key != 'playback'} music = {key: value for key, value in document.items() if key != 'playback'}
+4 -1
View File
@@ -10,6 +10,9 @@ from generator.model import SCHEMA_VERSION, validate_document
def load_json(path: Path) -> dict: def load_json(path: Path) -> dict:
document = json.loads(path.read_text(encoding='utf-8')) document = json.loads(path.read_text(encoding='utf-8'))
return validate_json(document, path)
def validate_json(document: dict, path: Path) -> dict:
if 'music' in document: if 'music' in document:
from generator.engraving import ENGINE_NAME, ENGINE_VERSION, PAGE_HEIGHT, PAGE_WIDTH, check_page from generator.engraving import ENGINE_NAME, ENGINE_VERSION, PAGE_HEIGHT, PAGE_WIDTH, check_page
presentation = document.get('presentation', {}) presentation = document.get('presentation', {})
@@ -122,8 +125,8 @@ def count_json_files(root: Path) -> int:
def validate_generated_tree(root: Path) -> None: def validate_generated_tree(root: Path) -> None:
audio_cache = {} audio_cache = {}
for path in root.rglob('*.json'): for path in root.rglob('*.json'):
load_json(path)
document = json.loads(path.read_text(encoding='utf-8')) document = json.loads(path.read_text(encoding='utf-8'))
validate_json(document, path)
if 'playback' in document: if 'playback' in document:
from generator.playback import validate_assets from generator.playback import validate_assets
validate_assets(document['playback'], root, audio_cache) validate_assets(document['playback'], root, audio_cache)
-3
View File
@@ -7,8 +7,6 @@ namespace vamp_protocol {
constexpr std::char_t root_path[] = "/"; constexpr std::char_t root_path[] = "/";
constexpr std::char_t status_path[] = "/api/status"; constexpr std::char_t status_path[] = "/api/status";
constexpr std::char_t next_page_path[] = "/api/page/next";
constexpr std::char_t previous_page_path[] = "/api/page/previous";
constexpr std::char_t remote_next_path[] = "/api/remote/next"; constexpr std::char_t remote_next_path[] = "/api/remote/next";
constexpr std::char_t remote_previous_path[] = "/api/remote/previous"; constexpr std::char_t remote_previous_path[] = "/api/remote/previous";
constexpr std::char_t remote_select_path[] = "/api/remote/select"; constexpr std::char_t remote_select_path[] = "/api/remote/select";
@@ -22,7 +20,6 @@ enum class page_command {
}; };
auto page_command_path(vamp_protocol::page_command command) -> std::char_t const *; auto page_command_path(vamp_protocol::page_command command) -> std::char_t const *;
auto page_command_legacy_path(vamp_protocol::page_command command) -> std::char_t const *;
auto page_command_name(vamp_protocol::page_command command) -> std::char_t const *; auto page_command_name(vamp_protocol::page_command command) -> std::char_t const *;
auto page_command_display_label(vamp_protocol::page_command command) -> std::char_t const *; auto page_command_display_label(vamp_protocol::page_command command) -> std::char_t const *;
+4 -1
View File
@@ -44,7 +44,10 @@
}, },
"default_target": "x86_64_gcc12_debug", "default_target": "x86_64_gcc12_debug",
"docker": { "docker": {
"image": "product-vamp-modulus-build:dev" "image": "product-vamp-modulus-build:dev",
"dockerfile": "package/Dockerfile.build",
"inputs": ["generator/requirements.txt"],
"socket_commands": ["build", "package"]
}, },
"remote": { "remote": {
"name": "khodo", "name": "khodo",
+58 -252
View File
@@ -1,18 +1,15 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# Canonical consumer bootstrap. Copy this file unchanged to a project's modulus.py.
import json import json
import hashlib
import os import os
import pathlib import pathlib
import shlex import shlex
import subprocess import subprocess
import sys import sys
import time
PACKAGE_REF = "modulus/latest" PACKAGE_REF = "modulus/latest"
CONAN_VERSION = "2.26.2" CONAN_VERSION = "2.26.2"
FETCH_IMAGE = "modulus-fetch:dev" FETCH_IMAGE = "modulus-fetch:dev"
DEFAULT_BUILD_BASE_IMAGE = "debian:trixie-slim"
PICO_SDK_VERSION = "2.3.0"
COMMANDS = ( COMMANDS = (
"configure", "configure",
"build", "build",
@@ -29,6 +26,7 @@ RUNTIME_DIR = COMMON_DIR / "modulus"
CONAN_HOME_DIR = COMMON_DIR / "conan_cache" CONAN_HOME_DIR = COMMON_DIR / "conan_cache"
DEFAULT_REMOTE_NAME = "khodo" DEFAULT_REMOTE_NAME = "khodo"
DEFAULT_REMOTE_URL = "https://git.thyme-labs.com/api/packages/khodo/conan" DEFAULT_REMOTE_URL = "https://git.thyme-labs.com/api/packages/khodo/conan"
DEFAULT_DOCKER_MARKER = "/opt/modulus-build-image"
def repo_root(): def repo_root():
root = pathlib.Path(__file__).resolve().parent root = pathlib.Path(__file__).resolve().parent
@@ -63,7 +61,6 @@ def runtime_ready(root):
(root / TOOLS_DIR / "configure.py").is_file() (root / TOOLS_DIR / "configure.py").is_file()
and (root / TOOLS_DIR / "lsp.py").is_file() and (root / TOOLS_DIR / "lsp.py").is_file()
and (root / TOOLS_DIR / "clangd.py").is_file() and (root / TOOLS_DIR / "clangd.py").is_file()
and (root / TOOLS_DIR / "run.py").is_file()
and (root / RUNTIME_DIR / "modulus" / "cli.py").is_file() and (root / RUNTIME_DIR / "modulus" / "cli.py").is_file()
and (root / RUNTIME_DIR / "modulus" / "export.py").is_file() and (root / RUNTIME_DIR / "modulus" / "export.py").is_file()
and (root / TOOLS_DIR / "format.py").is_file() and (root / TOOLS_DIR / "format.py").is_file()
@@ -71,18 +68,19 @@ def runtime_ready(root):
and (root / RUNTIME_DIR / "modulus" / "format_layout.py").is_file() and (root / RUNTIME_DIR / "modulus" / "format_layout.py").is_file()
) )
def optional_credentials():
user = os.environ.get("CONAN_USER")
token = os.environ.get("CONAN_TOKEN")
if bool(user) != bool(token):
raise SystemExit("CONAN_USER and CONAN_TOKEN must either both be set or both be unset")
return user, token
def env_flag(name): def env_flag(name):
return os.environ.get(name, "").lower() in {"1", "true", "yes", "on"} return os.environ.get(name, "").lower() in {"1", "true", "yes", "on"}
def docker_marker(root):
config = read_json(root / "modulus.json")
marker = config.get(
"tooling",
{},
).get("docker", {}).get("marker", DEFAULT_DOCKER_MARKER)
return pathlib.Path(str(marker))
def direct_mode(root): def direct_mode(root):
return env_flag("MODULUS_DIRECT") or project_docker_marker(root).exists() return env_flag("MODULUS_DIRECT") or docker_marker(root).exists()
def run(command, input_text=None, capture=False, display_command=None, env=None, check=True): def run(command, input_text=None, capture=False, display_command=None, env=None, check=True):
visible = display_command if display_command is not None else command visible = display_command if display_command is not None else command
@@ -104,20 +102,8 @@ def run(command, input_text=None, capture=False, display_command=None, env=None,
raise SystemExit(completed.returncode) raise SystemExit(completed.returncode)
return completed return completed
def run_with_retries(command, attempts=3, input_text=None, display_command=None, env=None):
completed = None
for attempt in range(1, attempts + 1):
completed = run(command, input_text=input_text, display_command=display_command, env=env, check=False)
if completed.returncode == 0:
return completed
if attempt != attempts:
delay = attempt * 20
print(f"command failed with exit code {completed.returncode}; retrying in {delay}s")
time.sleep(delay)
raise SystemExit(completed.returncode if completed is not None else 1)
def fetch_dockerfile(): def fetch_dockerfile():
return f"""FROM {DEFAULT_BUILD_BASE_IMAGE} return f"""FROM debian:bookworm-slim
ENV DEBIAN_FRONTEND=noninteractive ENV DEBIAN_FRONTEND=noninteractive
@@ -137,112 +123,23 @@ RUN python -m pip install --no-cache-dir conan=={CONAN_VERSION}
WORKDIR /workspace WORKDIR /workspace
""" """
def build_requirements_digest(): def ensure_fetch_image(offline=False):
return hashlib.sha256((repo_root() / "generator/requirements.txt").read_bytes()).hexdigest() inspected = subprocess.run(
[
def product_build_dockerfile(): "docker",
return f"""FROM {DEFAULT_BUILD_BASE_IMAGE} "image",
"inspect",
ENV DEBIAN_FRONTEND=noninteractive FETCH_IMAGE,
],
RUN dpkg --add-architecture arm64 stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
RUN apt-get update \\ check=False,
&& apt-get install -y --no-install-recommends \\ )
bash \\ if inspected.returncode == 0 and (offline or not docker_platform_args()):
build-essential \\
ca-certificates \\
clangd-19 \\
cmake \\
curl \\
docker-buildx \\
docker-cli \\
docker.io \\
g++-12 \\
g++-12-aarch64-linux-gnu \\
gcc-12 \\
gcc-12-aarch64-linux-gnu \\
gcc-arm-none-eabi \\
git \\
libdrm-dev \\
libdrm-dev:arm64 \\
libegl-dev \\
libegl-dev:arm64 \\
libfribidi0 \\
libgbm-dev \\
libgbm-dev:arm64 \\
libgles-dev \\
libgles-dev:arm64 \\
libglvnd-dev \\
libglvnd-dev:arm64 \\
libnewlib-arm-none-eabi \\
libstdc++-arm-none-eabi-newlib \\
ninja-build \\
pkg-config \\
python3 \\
python3-venv \\
&& ln -sf /usr/bin/clangd-19 /usr/local/bin/clangd \\
&& rm -rf /var/lib/apt/lists/*
ENV CC=/usr/bin/gcc-12
ENV CXX=/usr/bin/g++-12
ENV VAMP_PICO_SDK_VERSION={PICO_SDK_VERSION}
ENV PICO_SDK_PATH=/opt/pico-sdk
RUN git clone \\
--depth 1 \\
--branch "$VAMP_PICO_SDK_VERSION" \\
--recurse-submodules \\
--shallow-submodules \\
https://github.com/raspberrypi/pico-sdk.git \\
"$PICO_SDK_PATH"
RUN python3 -m venv /opt/modulus-python
ENV PATH=/opt/modulus-python/bin:$PATH
COPY generator/requirements.txt /tmp/vamp-build-requirements.txt
ARG MODULUS_TOOLS=build/build_tools
COPY ${{MODULUS_TOOLS}}/docker/format-requirements.txt /tmp/modulus-format-requirements.txt
RUN python -m pip install --no-cache-dir \\
conan=={CONAN_VERSION} -r /tmp/vamp-build-requirements.txt -r /tmp/modulus-format-requirements.txt
LABEL org.khodo.product-vamp.build-requirements="{build_requirements_digest()}"
WORKDIR /workspace
RUN mkdir -p /opt && touch /opt/modulus-build-image
"""
def sync_product_build_dockerfile(root):
dockerfile = root / TOOLS_DIR / "docker" / "Dockerfile"
if not dockerfile.parent.is_dir():
return return
desired = product_build_dockerfile() if offline:
try: raise SystemExit('offline bootstrap requires the existing Modulus fetch image or MODULUS_DIRECT=1 with Conan installed')
current = dockerfile.read_text(encoding="utf-8") run(
except OSError:
current = None
if current != desired:
dockerfile.write_text(desired, encoding="utf-8")
def ensure_fetch_image():
if not docker_platform_args():
inspected = subprocess.run(
[
"docker",
"image",
"inspect",
FETCH_IMAGE,
],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
check=False,
)
if inspected.returncode == 0:
return
run_with_retries(
[ [
"docker", "docker",
"build", "build",
@@ -257,6 +154,10 @@ def ensure_fetch_image():
def in_container(): def in_container():
return pathlib.Path("/.dockerenv").exists() return pathlib.Path("/.dockerenv").exists()
def docker_platform_args():
platform = os.environ.get("MODULUS_DOCKER_PLATFORM", "").strip()
return ["--platform", platform] if platform else []
def docker_workspace_args(root): def docker_workspace_args(root):
if in_container(): if in_container():
container = os.environ.get("HOSTNAME", "").strip() container = os.environ.get("HOSTNAME", "").strip()
@@ -271,43 +172,26 @@ def docker_workspace_args(root):
f"{root}:{root}", f"{root}:{root}",
] ]
def docker_socket_args():
socket = pathlib.Path("/var/run/docker.sock")
if not socket.is_socket():
return []
args = [
"--volume",
"/var/run/docker.sock:/var/run/docker.sock",
]
try:
args.extend(["--group-add", str(socket.stat().st_gid)])
except OSError:
pass
return args
def docker_platform_args():
platform = os.environ.get("MODULUS_DOCKER_PLATFORM", "").strip()
if not platform:
return []
return [
"--platform",
platform,
]
def fetch_env(root): def fetch_env(root):
env = os.environ.copy() env = os.environ.copy()
for name in list(env):
if name in {"CONAN_USER", "CONAN_TOKEN"} or name.startswith(("CONAN_LOGIN_USERNAME", "CONAN_PASSWORD")):
del env[name]
env["CONAN_HOME"] = str(root / CONAN_HOME_DIR) env["CONAN_HOME"] = str(root / CONAN_HOME_DIR)
env["CONAN_NON_INTERACTIVE"] = "1"
return env return env
def run_fetch(root, command, capture=False, display_command=None, check=True): def run_fetch(root, command, capture=False, display_command=None, check=True, offline=False):
if direct_mode(root): if direct_mode(root):
return run(command, capture=capture, display_command=display_command, env=fetch_env(root), check=check) return run(command, capture=capture, display_command=display_command, env=fetch_env(root), check=check)
ensure_fetch_image() ensure_fetch_image(offline=offline)
docker_command = [ docker_command = [
"docker", "docker",
"run", "run",
"--rm", "--rm",
"--interactive", "--interactive",
*(["--pull=never", "--network=none"] if offline else []),
*docker_platform_args(),
"--user", "--user",
f"{os.getuid()}:{os.getgid()}", f"{os.getuid()}:{os.getgid()}",
"--env", "--env",
@@ -315,13 +199,8 @@ def run_fetch(root, command, capture=False, display_command=None, check=True):
"--env", "--env",
"CONAN_HOME", "CONAN_HOME",
"--env", "--env",
"CONAN_USER", "CONAN_NON_INTERACTIVE",
"--env",
"CONAN_TOKEN",
"--env",
"DOCKER_HOST",
*docker_workspace_args(root), *docker_workspace_args(root),
*docker_socket_args(),
"--workdir", "--workdir",
str(root), str(root),
FETCH_IMAGE, FETCH_IMAGE,
@@ -330,7 +209,6 @@ def run_fetch(root, command, capture=False, display_command=None, check=True):
return run(docker_command, capture=capture, display_command=display_command, env=fetch_env(root), check=check) return run(docker_command, capture=capture, display_command=display_command, env=fetch_env(root), check=check)
def configure_remote(root, remote_name, remote_url): def configure_remote(root, remote_name, remote_url):
user, token = optional_credentials()
run_fetch( run_fetch(
root, root,
[ [
@@ -342,29 +220,7 @@ def configure_remote(root, remote_name, remote_url):
remote_url, remote_url,
], ],
) )
if not user and not token: run_fetch(root, ["conan", "remote", "logout", remote_name])
return
run_fetch(
root,
[
"conan",
"remote",
"login",
remote_name,
user,
"-p",
token,
],
display_command=[
"conan",
"remote",
"login",
remote_name,
user,
"-p",
"<hidden>",
],
)
def package_refs(data): def package_refs(data):
refs = [] refs = []
@@ -401,46 +257,26 @@ def package_refs(data):
refs.append(f"{PACKAGE_REF}#{recipe_revision}:{package_id}") refs.append(f"{PACKAGE_REF}#{recipe_revision}:{package_id}")
return sorted(refs) return sorted(refs)
def remove_cached_modulus(root): def find_modulus(root, remote_name=None):
run_fetch(
root,
[
"conan",
"remove",
PACKAGE_REF + ":*",
"-c",
],
check=False,
)
def download_modulus(root, remote_name):
remove_cached_modulus(root)
run_fetch(
root,
[
"conan",
"download",
PACKAGE_REF + ":*",
"-r",
remote_name,
],
)
completed = run_fetch( completed = run_fetch(
root, root,
[ [
"conan", "conan",
"list", "list",
PACKAGE_REF + ":*", PACKAGE_REF + "#latest:*#latest",
"-c", *(["-r", remote_name] if remote_name else ["-c"]),
"-f", "-f",
"json", "json",
], ],
capture=True, capture=True,
offline=remote_name is None,
) )
refs = package_refs(json.loads(completed.stdout)) refs = package_refs(json.loads(completed.stdout))
if not refs: if not refs:
raise SystemExit(f"no downloaded {PACKAGE_REF} package found in the Conan cache") raise SystemExit(f"no {PACKAGE_REF} package available in {remote_name or 'the local Conan cache'}")
package_ref = refs[-1] package_ref = refs[-1]
if remote_name:
run_fetch(root, ["conan", "download", package_ref, "-r", remote_name])
completed = run_fetch( completed = run_fetch(
root, root,
[ [
@@ -450,40 +286,25 @@ def download_modulus(root, remote_name):
package_ref, package_ref,
], ],
capture=True, capture=True,
offline=remote_name is None,
) )
package_root = pathlib.Path(completed.stdout.strip()) package_root = pathlib.Path(completed.stdout.strip())
if not (package_root / "modulus" / "export.py").is_file() or not (package_root / "tools").is_dir(): if not (package_root / "modulus" / "export.py").is_file() or not (package_root / "tools").is_dir():
raise SystemExit(f"downloaded {PACKAGE_REF} package does not contain Modulus export files: {package_root}") raise SystemExit(f"downloaded {PACKAGE_REF} package does not contain Modulus export files: {package_root}")
return package_root return package_root
def materialize_runtime(root): def materialize_runtime(root, no_remote=False):
remote_name, remote_url = remote_config(root) remote_name, remote_url = remote_config(root)
configure_remote(root, remote_name, remote_url) if not no_remote:
package_root = download_modulus(root, remote_name) configure_remote(root, remote_name, remote_url)
package_root = find_modulus(root, None if no_remote else remote_name)
sys.path.insert(0, str(package_root)) sys.path.insert(0, str(package_root))
import modulus.export import modulus.export
modulus.export.sync_project(root, package_root) modulus.export.sync_project(root, package_root)
def ensure_runtime(root): def ensure_runtime(root, update=False, no_remote=False):
if not runtime_ready(root): if update or not runtime_ready(root):
materialize_runtime(root) materialize_runtime(root, no_remote=no_remote)
sync_product_build_dockerfile(root)
def project_docker_marker(root):
config = read_json(root / "modulus.json")
tooling = config.get("tooling", {})
if not isinstance(tooling, dict):
tooling = {}
docker = tooling.get("docker", {})
if not isinstance(docker, dict):
docker = {}
return pathlib.Path(str(docker.get("marker") or "/opt/modulus-build-image"))
def run_configure(argv):
return delegate("configure", argv)
def run_package(argv):
return delegate("package", argv)
def delegate(command, argv): def delegate(command, argv):
root = repo_root() root = repo_root()
@@ -492,19 +313,9 @@ def delegate(command, argv):
print(f"Modulus {command} tooling is missing; run ./modulus.py configure first", file=sys.stderr) print(f"Modulus {command} tooling is missing; run ./modulus.py configure first", file=sys.stderr)
return 1 return 1
else: else:
ensure_runtime(root) ensure_runtime(root, update=command == "configure" and "--update" in argv, no_remote="--no-remote" in argv)
sys.path.insert(0, str(root / RUNTIME_DIR)) sys.path.insert(0, str(root / RUNTIME_DIR))
import modulus.cli import modulus.cli
original_docker_run = modulus.cli.docker_run
def docker_run_with_host_socket(context, command, docker_args=None, env_overrides=None):
merged_docker_args = [
*docker_socket_args(),
*(docker_args or []),
]
return original_docker_run(context, command, docker_args=merged_docker_args, env_overrides=env_overrides)
modulus.cli.docker_run = docker_run_with_host_socket
previous_argv = sys.argv previous_argv = sys.argv
previous_cwd = pathlib.Path.cwd() previous_cwd = pathlib.Path.cwd()
sys.argv = [ sys.argv = [
@@ -515,7 +326,6 @@ def delegate(command, argv):
try: try:
return modulus.cli.main(command) return modulus.cli.main(command)
finally: finally:
modulus.cli.docker_run = original_docker_run
os.chdir(previous_cwd) os.chdir(previous_cwd)
sys.argv = previous_argv sys.argv = previous_argv
@@ -526,10 +336,6 @@ def main(argv):
command = argv[0] command = argv[0]
if command not in COMMANDS: if command not in COMMANDS:
raise SystemExit(f"unknown Modulus command: {command}") raise SystemExit(f"unknown Modulus command: {command}")
if command == "configure":
return run_configure(argv[1:])
if command == "package":
return run_package(argv[1:])
return delegate(command, argv[1:]) return delegate(command, argv[1:])
if __name__ == "__main__": if __name__ == "__main__":
+69
View File
@@ -0,0 +1,69 @@
FROM debian:trixie-slim
ENV DEBIAN_FRONTEND=noninteractive
RUN dpkg --add-architecture arm64
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
bash \
build-essential \
ca-certificates \
clangd-19 \
cmake \
curl \
docker-buildx \
docker-cli \
docker.io \
g++-12 \
g++-12-aarch64-linux-gnu \
gcc-12 \
gcc-12-aarch64-linux-gnu \
gcc-arm-none-eabi \
git \
libdrm-dev \
libdrm-dev:arm64 \
libegl-dev \
libegl-dev:arm64 \
libfribidi0 \
libgbm-dev \
libgbm-dev:arm64 \
libgles-dev \
libgles-dev:arm64 \
libglvnd-dev \
libglvnd-dev:arm64 \
libnewlib-arm-none-eabi \
libstdc++-arm-none-eabi-newlib \
ninja-build \
pkg-config \
python3 \
python3-venv \
&& ln -sf /usr/bin/clangd-19 /usr/local/bin/clangd \
&& rm -rf /var/lib/apt/lists/*
ENV CC=/usr/bin/gcc-12
ENV CXX=/usr/bin/g++-12
ENV VAMP_PICO_SDK_VERSION=2.3.0
ENV PICO_SDK_PATH=/opt/pico-sdk
RUN git clone \
--depth 1 \
--branch "$VAMP_PICO_SDK_VERSION" \
--recurse-submodules \
--shallow-submodules \
https://github.com/raspberrypi/pico-sdk.git \
"$PICO_SDK_PATH"
RUN python3 -m venv /opt/modulus-python
ENV PATH=/opt/modulus-python/bin:$PATH
COPY generator/requirements.txt /tmp/vamp-build-requirements.txt
ARG MODULUS_TOOLS=build/build_tools
COPY ${MODULUS_TOOLS}/docker/format-requirements.txt /tmp/modulus-format-requirements.txt
RUN python -m pip install --no-cache-dir \
conan==2.26.2 -r /tmp/vamp-build-requirements.txt -r /tmp/modulus-format-requirements.txt
WORKDIR /workspace
RUN mkdir -p /opt && touch /opt/modulus-build-image
-12
View File
@@ -13,18 +13,6 @@ auto vamp_protocol::page_command_path(vamp_protocol::page_command command) -> st
return vamp_protocol::remote_next_path; return vamp_protocol::remote_next_path;
} }
auto vamp_protocol::page_command_legacy_path(vamp_protocol::page_command command) -> std::char_t const * {
switch (command) {
case vamp_protocol::page_command::PREVIOUS:
return vamp_protocol::previous_page_path;
case vamp_protocol::page_command::NEXT:
return vamp_protocol::next_page_path;
case vamp_protocol::page_command::SELECT:
return vamp_protocol::remote_select_path;
}
return vamp_protocol::next_page_path;
}
auto vamp_protocol::page_command_name(vamp_protocol::page_command command) -> std::char_t const * { auto vamp_protocol::page_command_name(vamp_protocol::page_command command) -> std::char_t const * {
switch (command) { switch (command) {
case vamp_protocol::page_command::PREVIOUS: case vamp_protocol::page_command::PREVIOUS:
+25 -81
View File
@@ -1,4 +1,3 @@
#include <cmath>
#include <cstdint> #include <cstdint>
#include <expected> #include <expected>
#include <string> #include <string>
@@ -6,9 +5,7 @@
#include <utility> #include <utility>
#include <boost/json/object.hpp> #include <boost/json/object.hpp>
#include <boost/json/parse.hpp>
#include <boost/json/value.hpp> #include <boost/json/value.hpp>
#include <boost/system/error_code.hpp>
#include <vamp_zero/config.hpp> #include <vamp_zero/config.hpp>
#include <vamp_zero/errors.hpp> #include <vamp_zero/errors.hpp>
@@ -22,93 +19,49 @@ constexpr auto json_options = modulus::json_config::options{
.not_found_code = static_cast<std::int32_t>(vamp_zero::error_code::NOT_FOUND), .not_found_code = static_cast<std::int32_t>(vamp_zero::error_code::NOT_FOUND),
}; };
constexpr auto field_options = modulus::json_config::options{
.category = "product-vamp",
.invalid_argument_code = static_cast<std::int32_t>(vamp_zero::error_code::INVALID_ARGUMENT),
.not_found_code = static_cast<std::int32_t>(vamp_zero::error_code::NOT_FOUND),
};
auto invalid(std::string message) -> modulus::error { auto invalid(std::string message) -> modulus::error {
return vamp_zero::make_error(vamp_zero::error_code::INVALID_ARGUMENT, std::move(message)); return vamp_zero::make_error(vamp_zero::error_code::INVALID_ARGUMENT, std::move(message));
} }
auto require_object(boost::json::value const &value, std::string_view name) -> std::expected<boost::json::object const *, modulus::error> {
if (!value.is_object()) {
return std::unexpected(invalid(std::string{name} + " must be an object"));
}
return &value.as_object();
}
auto optional_object(boost::json::object const &object, std::string_view field) -> std::expected<boost::json::object const *, modulus::error> { auto optional_object(boost::json::object const &object, std::string_view field) -> std::expected<boost::json::object const *, modulus::error> {
auto found = object.find(field); auto found = object.find(field);
if (found == object.end()) { if (found == object.end()) {
return nullptr; return nullptr;
} }
return require_object(found->value(), field); return modulus::json_config::require_object(found->value(), field, field_options);
} }
auto optional_string(boost::json::object const &object, std::string_view field, std::string fallback) -> std::expected<std::string, modulus::error> { auto optional_string(boost::json::object const &object, std::string_view field, std::string fallback) -> std::expected<std::string, modulus::error> {
auto found = object.find(field); auto value = modulus::json_config::optional_string_field(object, field, std::move(fallback), field_options);
if (found == object.end()) { if (value && value->empty() && object.contains(field)) {
return fallback;
}
if (!found->value().is_string()) {
return std::unexpected(invalid(std::string{field} + " must be a string"));
}
auto value = std::string{found->value().as_string()};
if (value.empty()) {
return std::unexpected(invalid(std::string{field} + " must not be empty")); return std::unexpected(invalid(std::string{field} + " must not be empty"));
} }
return value; return value;
} }
auto optional_bool(boost::json::object const &object, std::string_view field, std::bool_t fallback) -> std::expected<std::bool_t, modulus::error> {
auto found = object.find(field);
if (found == object.end()) {
return fallback;
}
if (!found->value().is_bool()) {
return std::unexpected(invalid(std::string{field} + " must be a boolean"));
}
return found->value().as_bool();
}
auto optional_uint32(boost::json::object const &object, std::string_view field, std::uint32_t fallback, std::uint32_t min_value, std::uint32_t max_value) -> std::expected<std::uint32_t, modulus::error> { auto optional_uint32(boost::json::object const &object, std::string_view field, std::uint32_t fallback, std::uint32_t min_value, std::uint32_t max_value) -> std::expected<std::uint32_t, modulus::error> {
auto found = object.find(field); auto value = modulus::json_config::optional_uint64_field(object, field, fallback, max_value, field_options);
if (found == object.end()) { if (!value) {
return fallback; return std::unexpected(value.error());
} }
auto value = std::uint64_t{}; if (*value < min_value) {
if (found->value().is_uint64()) {
value = found->value().as_uint64();
} else if (found->value().is_int64()) {
auto signed_value = found->value().as_int64();
if (signed_value < 0) {
return std::unexpected(invalid(std::string{field} + " must not be negative"));
}
value = static_cast<std::uint64_t>(signed_value);
} else {
return std::unexpected(invalid(std::string{field} + " must be an integer"));
}
if (value < min_value || value > max_value) {
return std::unexpected(invalid(std::string{field} + " is out of range")); return std::unexpected(invalid(std::string{field} + " is out of range"));
} }
return static_cast<std::uint32_t>(value); return static_cast<std::uint32_t>(*value);
} }
auto optional_float(boost::json::object const &object, std::string_view field, std::float32_t fallback, std::float32_t min_value, std::float32_t max_value) -> std::expected<std::float32_t, modulus::error> { auto optional_float(boost::json::object const &object, std::string_view field, std::float32_t fallback, std::float32_t min_value, std::float32_t max_value) -> std::expected<std::float32_t, modulus::error> {
auto found = object.find(field); auto value = modulus::json_config::optional_float64_field(object, field, fallback, min_value, max_value, field_options);
if (found == object.end()) { if (!value) {
return fallback; return std::unexpected(value.error());
} }
auto value = std::float64_t{}; return static_cast<std::float32_t>(*value);
if (found->value().is_double()) {
value = found->value().as_double();
} else if (found->value().is_uint64()) {
value = static_cast<std::float64_t>(found->value().as_uint64());
} else if (found->value().is_int64()) {
value = static_cast<std::float64_t>(found->value().as_int64());
} else {
return std::unexpected(invalid(std::string{field} + " must be a number"));
}
if (!std::isfinite(value) || value < min_value || value > max_value) {
return std::unexpected(invalid(std::string{field} + " is out of range"));
}
return static_cast<std::float32_t>(value);
} }
auto parse_runtime(boost::json::object const &root, vamp_zero::runtime_config fallback) -> std::expected<vamp_zero::runtime_config, modulus::error> { auto parse_runtime(boost::json::object const &root, vamp_zero::runtime_config fallback) -> std::expected<vamp_zero::runtime_config, modulus::error> {
@@ -119,8 +72,8 @@ auto parse_runtime(boost::json::object const &root, vamp_zero::runtime_config fa
if (*object == nullptr) { if (*object == nullptr) {
return fallback; return fallback;
} }
auto headless = optional_bool(**object, "headless", fallback.headless); auto headless = modulus::json_config::optional_bool_field(**object, "headless", fallback.headless, field_options);
auto exit_after_startup = optional_bool(**object, "exit_after_startup", fallback.exit_after_startup); auto exit_after_startup = modulus::json_config::optional_bool_field(**object, "exit_after_startup", fallback.exit_after_startup, field_options);
auto assets_path = optional_string(**object, "assets_path", fallback.assets_path); auto assets_path = optional_string(**object, "assets_path", fallback.assets_path);
auto heartbeat_path = optional_string(**object, "heartbeat_path", fallback.heartbeat_path); auto heartbeat_path = optional_string(**object, "heartbeat_path", fallback.heartbeat_path);
if (!headless) { if (!headless) {
@@ -218,21 +171,12 @@ auto parse_audio(boost::json::object const &root, vamp_zero::audio_config fallba
if (*object == nullptr) { if (*object == nullptr) {
return fallback; return fallback;
} }
auto enabled = optional_bool(**object, "enabled", fallback.enabled);
auto playback_enabled = optional_bool(**object, "playback_enabled", fallback.playback_enabled);
auto legacy_track = optional_bool(**object, "track_enabled", fallback.playback_enabled);
if (!legacy_track) {
return std::unexpected(legacy_track.error());
}
if ((*object)->if_contains("track_enabled")) { if ((*object)->if_contains("track_enabled")) {
if ((*object)->if_contains("playback_enabled") && playback_enabled && *playback_enabled != *legacy_track) { return std::unexpected(invalid("audio.track_enabled is no longer supported; use audio.playback_enabled"));
return std::unexpected(invalid("audio.playback_enabled contradicts legacy track_enabled"));
}
if (!(*object)->if_contains("playback_enabled")) {
playback_enabled = legacy_track;
}
} }
auto metronome_enabled = optional_bool(**object, "metronome_enabled", fallback.metronome_enabled); auto enabled = modulus::json_config::optional_bool_field(**object, "enabled", fallback.enabled, field_options);
auto playback_enabled = modulus::json_config::optional_bool_field(**object, "playback_enabled", fallback.playback_enabled, field_options);
auto metronome_enabled = modulus::json_config::optional_bool_field(**object, "metronome_enabled", fallback.metronome_enabled, field_options);
auto metronome_bpm = optional_uint32(**object, "metronome_bpm", fallback.metronome_bpm, 1U, 400U); auto metronome_bpm = optional_uint32(**object, "metronome_bpm", fallback.metronome_bpm, 1U, 400U);
auto metronome_min_bpm = optional_uint32(**object, "metronome_min_bpm", fallback.metronome_min_bpm, 1U, 400U); auto metronome_min_bpm = optional_uint32(**object, "metronome_min_bpm", fallback.metronome_min_bpm, 1U, 400U);
auto metronome_max_bpm = optional_uint32(**object, "metronome_max_bpm", fallback.metronome_max_bpm, 1U, 400U); auto metronome_max_bpm = optional_uint32(**object, "metronome_max_bpm", fallback.metronome_max_bpm, 1U, 400U);
@@ -293,7 +237,7 @@ auto parse_audio(boost::json::object const &root, vamp_zero::audio_config fallba
} }
auto vamp_zero::parse_config(boost::json::value const &value) -> std::expected<vamp_zero::config, modulus::error> { auto vamp_zero::parse_config(boost::json::value const &value) -> std::expected<vamp_zero::config, modulus::error> {
auto root = require_object(value, "Product Vamp config root"); auto root = modulus::json_config::require_object(value, "Product Vamp config root", field_options);
if (!root) { if (!root) {
return std::unexpected(root.error()); return std::unexpected(root.error());
} }
-8
View File
@@ -196,13 +196,5 @@ auto vamp_zero::make_router(vamp_zero::control_state &state) -> ligo::service_ro
auto updated = state.apply(vamp_protocol::page_command::SELECT); auto updated = state.apply(vamp_protocol::page_command::SELECT);
co_return remote_response(updated); co_return remote_response(updated);
}); });
router.add_route("POST", vamp_protocol::next_page_path, [&state](ligo::request_context, boost::json::value) -> boost::asio::awaitable<std::expected<boost::json::value, modulus::error>> {
auto updated = state.apply(vamp_protocol::page_command::NEXT);
co_return vamp_zero::portal_status_json(state.config(), updated);
});
router.add_route("POST", vamp_protocol::previous_page_path, [&state](ligo::request_context, boost::json::value) -> boost::asio::awaitable<std::expected<boost::json::value, modulus::error>> {
auto updated = state.apply(vamp_protocol::page_command::PREVIOUS);
co_return vamp_zero::portal_status_json(state.config(), updated);
});
return router; return router;
} }
+6 -6
View File
@@ -1,4 +1,4 @@
import hashlib import json
import importlib.util import importlib.util
from pathlib import Path from pathlib import Path
import sys import sys
@@ -7,15 +7,15 @@ import unittest
from unittest.mock import Mock, patch from unittest.mock import Mock, patch
class BuildBootstrapTests(unittest.TestCase): class BuildBootstrapTests(unittest.TestCase):
def test_requirements_fingerprint_invalidates_cached_build_image(self): def test_build_recipe_is_declared_and_editor_commands_do_not_bootstrap(self):
root = Path(__file__).resolve().parents[1] root = Path(__file__).resolve().parents[1]
spec = importlib.util.spec_from_file_location('vamp_bootstrap', root / 'modulus.py') spec = importlib.util.spec_from_file_location('vamp_bootstrap', root / 'modulus.py')
bootstrap = importlib.util.module_from_spec(spec) bootstrap = importlib.util.module_from_spec(spec)
spec.loader.exec_module(bootstrap) spec.loader.exec_module(bootstrap)
digest = hashlib.sha256((root / 'generator/requirements.txt').read_bytes()).hexdigest() docker = json.loads((root / 'modulus.json').read_text())['tooling']['docker']
self.assertEqual(bootstrap.build_requirements_digest(), digest) self.assertEqual(docker['socket_commands'], ['build', 'package'])
self.assertIn(f'build-requirements="{digest}"', bootstrap.product_build_dockerfile()) self.assertIn('generator/requirements.txt', docker['inputs'])
self.assertIn('COPY generator/requirements.txt /tmp/vamp-build-requirements.txt', bootstrap.product_build_dockerfile()) self.assertIn('COPY generator/requirements.txt /tmp/vamp-build-requirements.txt', (root / docker['dockerfile']).read_text())
package = types.ModuleType('modulus') package = types.ModuleType('modulus')
cli = types.ModuleType('modulus.cli') cli = types.ModuleType('modulus.cli')
package.cli = cli package.cli = cli
+50 -6
View File
@@ -1,6 +1,8 @@
import argparse import argparse
import contextlib
import copy import copy
import hashlib import hashlib
import io
import json import json
from pathlib import Path from pathlib import Path
import tempfile import tempfile
@@ -9,14 +11,16 @@ from unittest.mock import patch
from generator.__main__ import generate_assets from generator.__main__ import generate_assets
from generator.engraving import EngravingError, Fonts, Style, engrave_tree, layout_score from generator.engraving import EngravingError, Fonts, Style, engrave_tree, layout_score
from generator.engraving.cache import renderer_fingerprint, restore_pages, score_fingerprint from generator.engraving.cache import renderer_fingerprint, renderer_namespace, restore_pages, score_fingerprint
from generator.score import validate_generated_tree
from tests.test_musicxml import parse, xml_note, xml_score from tests.test_musicxml import parse, xml_note, xml_score
class EngravingCacheTests(unittest.TestCase): class EngravingCacheTests(unittest.TestCase):
def test_ci_snapshot_tracks_all_generator_inputs_and_keeps_fallback(self): def test_ci_snapshot_tracks_all_generator_inputs_and_keeps_fallback(self):
workflow = Path('.gitea/workflows/ci.yml').read_text() workflow = Path('.gitea/workflows/ci.yml').read_text()
self.assertIn("hashFiles('generator/**/*.py', 'generator/*.py', 'data/**/*.xml', 'data/*.xml')", workflow) self.assertIn("hashFiles('generator/**/*.py', 'generator/*.py', 'data/**/*.xml', 'data/*.xml')", workflow)
self.assertIn('restore-keys: product-vamp-pages-v1-${{ runner.os }}-', workflow) self.assertIn('python3 -m generator --renderer-fingerprint', workflow)
self.assertIn('restore-keys: product-vamp-pages-v2-${{ steps.renderer.outputs.fingerprint }}-', workflow)
self.assertIn("if: steps.engraving-cache.outputs.cache-hit != 'true'", workflow) self.assertIn("if: steps.engraving-cache.outputs.cache-hit != 'true'", workflow)
self.assertIn('key: ${{ steps.engraving-cache.outputs.cache-primary-key }}', workflow) self.assertIn('key: ${{ steps.engraving-cache.outputs.cache-primary-key }}', workflow)
def setUp(self): def setUp(self):
@@ -36,7 +40,7 @@ class EngravingCacheTests(unittest.TestCase):
root = self.root / 'seed' root = self.root / 'seed'
self.write_scores(root, [self.document]) self.write_scores(root, [self.document])
engrave_tree(root, self.fonts, cache_dir=self.cache) engrave_tree(root, self.fonts, cache_dir=self.cache)
return next(self.cache.iterdir()) return next(self.cache.rglob('pages.json')).parent
def test_cache_hit_preserves_bundle_bytes_across_output_directories(self): def test_cache_hit_preserves_bundle_bytes_across_output_directories(self):
cold, warm = self.root / 'cold', self.root / 'warm' cold, warm = self.root / 'cold', self.root / 'warm'
self.write_scores(cold, [self.document]) self.write_scores(cold, [self.document])
@@ -65,7 +69,7 @@ class EngravingCacheTests(unittest.TestCase):
with patch('generator.engraving.layout_score', wraps=layout_score) as render: with patch('generator.engraving.layout_score', wraps=layout_score) as render:
engrave_tree(root, self.fonts, cache_dir=self.cache) engrave_tree(root, self.fonts, cache_dir=self.cache)
render.assert_called_once() render.assert_called_once()
self.assertEqual(len(list(self.cache.iterdir())), 2) self.assertEqual(len(list(self.cache.rglob('pages.json'))), 2)
def test_renderer_fingerprint_tracks_fonts_style_code_and_dependencies(self): def test_renderer_fingerprint_tracks_fonts_style_code_and_dependencies(self):
fonts, style = Fonts(self.fonts), Style() fonts, style = Fonts(self.fonts), Style()
expected = renderer_fingerprint(fonts, style) expected = renderer_fingerprint(fonts, style)
@@ -75,6 +79,8 @@ class EngravingCacheTests(unittest.TestCase):
self.assertNotEqual(expected, renderer_fingerprint(fonts, style)) self.assertNotEqual(expected, renderer_fingerprint(fonts, style))
with patch('generator.engraving.cache.importlib.metadata.version', return_value='new-version'): with patch('generator.engraving.cache.importlib.metadata.version', return_value='new-version'):
self.assertNotEqual(expected, renderer_fingerprint(fonts, style)) self.assertNotEqual(expected, renderer_fingerprint(fonts, style))
with patch('generator.engraving.cache.platform.python_version', return_value='0.0.0'):
self.assertNotEqual(expected, renderer_fingerprint(fonts, style))
read_bytes = Path.read_bytes read_bytes = Path.read_bytes
with patch.object(Path, 'read_bytes', lambda path: read_bytes(path) + (b'\n' if path.name == 'raster.py' else b'')): with patch.object(Path, 'read_bytes', lambda path: read_bytes(path) + (b'\n' if path.name == 'raster.py' else b'')):
self.assertNotEqual(expected, renderer_fingerprint(fonts, style)) self.assertNotEqual(expected, renderer_fingerprint(fonts, style))
@@ -129,8 +135,46 @@ class EngravingCacheTests(unittest.TestCase):
self.write_scores(root, documents) self.write_scores(root, documents)
engrave_tree(root, self.fonts, cache_dir=self.cache, jobs=2) engrave_tree(root, self.fonts, cache_dir=self.cache, jobs=2)
self.assertEqual(self.files(serial), self.files(root)) self.assertEqual(self.files(serial), self.files(root))
self.assertEqual(len(list(self.cache.iterdir())), 2) self.assertEqual(len(list(self.cache.rglob('pages.json'))), 2)
self.assertFalse(list(self.cache.glob('.engraving-*'))) self.assertFalse(list(self.cache.rglob('.engraving-*')))
def test_environment_change_gets_new_namespace_and_clear_progress(self):
old = self.seed_cache()
root = self.root / 'new-environment'
self.write_scores(root, [self.document])
progress = io.StringIO()
with patch('generator.engraving.cache.platform.python_version', return_value='0.0.0'), contextlib.redirect_stdout(progress):
engrave_tree(root, self.fonts, cache_dir=self.cache, progress=True)
self.assertIn('cold namespace', progress.getvalue())
self.assertIn('0 reused, 1 missing, 0 rejected', progress.getvalue())
self.assertTrue((old / 'pages.json').is_file())
self.assertEqual(len(list((self.cache / 'v2').iterdir())), 2)
self.write_scores(root, [self.document])
progress = io.StringIO()
with patch('generator.engraving.cache.platform.python_version', return_value='0.0.0'), contextlib.redirect_stdout(progress), patch('generator.engraving.layout_score', side_effect=AssertionError('cache miss')):
engrave_tree(root, self.fonts, cache_dir=self.cache, progress=True)
self.assertIn('1 reused, 0 missing, 0 rejected', progress.getvalue())
def test_namespace_symlink_is_rejected(self):
self.cache.mkdir()
(self.cache / 'v2').symlink_to(self.root, target_is_directory=True)
with self.assertRaisesRegex(ValueError, 'must not be symlinks'):
renderer_namespace(self.cache, 'renderer')
def test_only_changed_score_is_rendered_and_validation_reads_json_once(self):
root = self.root / 'scores'
self.write_scores(root, [self.document, self.document])
engrave_tree(root, self.fonts, cache_dir=self.cache)
changed = parse(xml_score(xml_note('D')))
self.write_scores(root, [self.document, changed])
with patch('generator.engraving.layout_score', wraps=layout_score) as render:
engrave_tree(root, self.fonts, cache_dir=self.cache)
render.assert_called_once()
read_text = Path.read_text
reads = []
def read(path, *args, **kwargs):
reads.append(path)
return read_text(path, *args, **kwargs)
with patch.object(Path, 'read_text', read):
validate_generated_tree(root)
self.assertCountEqual(reads, sorted(root.glob('*.json')))
def test_invalid_worker_count_and_overlapping_cache_are_rejected(self): def test_invalid_worker_count_and_overlapping_cache_are_rejected(self):
root = self.root / 'assets' root = self.root / 'assets'
self.write_scores(root, [self.document]) self.write_scores(root, [self.document])
+3 -3
View File
@@ -120,10 +120,10 @@ BOOST_AUTO_TEST_CASE(playback_worker_discards_canceled_preparation_and_reports_e
BOOST_CHECK(!audio.tick(state, loaded, VAMP_TEST_PLAYBACK_ASSETS_DIR)); BOOST_CHECK(!audio.tick(state, loaded, VAMP_TEST_PLAYBACK_ASSETS_DIR));
} }
BOOST_AUTO_TEST_CASE(playback_legacy_config_and_invalid_assets) { BOOST_AUTO_TEST_CASE(playback_rejects_legacy_config_and_invalid_assets) {
auto legacy = vamp_zero::parse_config(boost::json::object{{"audio", {{"track_enabled", true}}}}); auto legacy = vamp_zero::parse_config(boost::json::object{{"audio", {{"track_enabled", true}}}});
BOOST_REQUIRE(legacy); BOOST_REQUIRE(!legacy);
BOOST_TEST(legacy->audio.playback_enabled); BOOST_TEST(std::string{legacy.error().message()}.find("use audio.playback_enabled") != std::string::npos);
BOOST_CHECK(!vamp_zero::parse_config(boost::json::object{{"audio", {{"track_enabled", true}, {"playback_enabled", false}}}})); BOOST_CHECK(!vamp_zero::parse_config(boost::json::object{{"audio", {{"track_enabled", true}, {"playback_enabled", false}}}}));
auto score = playback_score(); auto score = playback_score();
BOOST_CHECK(!vamp_zero::prepare_playback(score, "/missing-vamp-assets", 120U)); BOOST_CHECK(!vamp_zero::prepare_playback(score, "/missing-vamp-assets", 120U));