@@ -97,25 +97,28 @@ jobs:
|
|||||||
key: product-vamp-conan-${{ runner.os }}-${{ hashFiles('conanfile.py', 'modulus.json', 'conan/profiles/*') }}
|
key: product-vamp-conan-${{ runner.os }}-${{ hashFiles('conanfile.py', 'modulus.json', 'conan/profiles/*') }}
|
||||||
restore-keys: product-vamp-conan-${{ runner.os }}-
|
restore-keys: product-vamp-conan-${{ runner.os }}-
|
||||||
- name: Configure native release
|
- name: Configure native release
|
||||||
run: python3 modulus.py configure --target x86_64_gcc12_release
|
run: python3 modulus.py configure --target x86_64_gcc12_release --update
|
||||||
- name: Check code style
|
- name: Check code style
|
||||||
run: |
|
run: |
|
||||||
python3 -m pip install --no-cache-dir -r build/build_tools/docker/format-requirements.txt
|
python3 -m pip install --no-cache-dir -r build/build_tools/docker/format-requirements.txt
|
||||||
python3 modulus.py format --check
|
python3 modulus.py format --check
|
||||||
|
- name: Identify engraving renderer
|
||||||
|
id: renderer
|
||||||
|
run: echo "fingerprint=$(python3 -m generator --renderer-fingerprint)" >> "$GITHUB_OUTPUT"
|
||||||
- name: Restore engraving cache
|
- name: Restore engraving cache
|
||||||
id: engraving-cache
|
id: engraving-cache
|
||||||
uses: actions/cache/restore@v4
|
uses: actions/cache/restore@v4
|
||||||
with:
|
with:
|
||||||
path: build/build_common/engraving-cache
|
path: build/build_common/engraving-cache/v2/${{ steps.renderer.outputs.fingerprint }}
|
||||||
key: product-vamp-pages-v1-${{ runner.os }}-${{ hashFiles('generator/engraving/*.py', 'generator/model.py', 'generator/harmony.py', 'generator/requirements.txt', 'package/assets/fonts/petaluma/*') }}-${{ hashFiles('generator/**/*.py', 'generator/*.py', 'data/**/*.xml', 'data/*.xml') }}
|
key: product-vamp-pages-v2-${{ steps.renderer.outputs.fingerprint }}-${{ hashFiles('generator/**/*.py', 'generator/*.py', 'data/**/*.xml', 'data/*.xml') }}
|
||||||
restore-keys: product-vamp-pages-v1-${{ runner.os }}-${{ hashFiles('generator/engraving/*.py', 'generator/model.py', 'generator/harmony.py', 'generator/requirements.txt', 'package/assets/fonts/petaluma/*') }}-
|
restore-keys: product-vamp-pages-v2-${{ steps.renderer.outputs.fingerprint }}-
|
||||||
- name: Build native release
|
- name: Build native release
|
||||||
run: python3 modulus.py build --target x86_64_gcc12_release
|
run: python3 modulus.py build --target x86_64_gcc12_release
|
||||||
- name: Save engraving cache
|
- name: Save engraving cache
|
||||||
if: steps.engraving-cache.outputs.cache-hit != 'true'
|
if: steps.engraving-cache.outputs.cache-hit != 'true'
|
||||||
uses: actions/cache/save@v4
|
uses: actions/cache/save@v4
|
||||||
with:
|
with:
|
||||||
path: build/build_common/engraving-cache
|
path: build/build_common/engraving-cache/v2/${{ steps.renderer.outputs.fingerprint }}
|
||||||
key: ${{ steps.engraving-cache.outputs.cache-primary-key }}
|
key: ${{ steps.engraving-cache.outputs.cache-primary-key }}
|
||||||
- name: Test native release
|
- name: Test native release
|
||||||
run: python3 modulus.py test --target x86_64_gcc12_release
|
run: python3 modulus.py test --target x86_64_gcc12_release
|
||||||
|
|||||||
@@ -33,7 +33,9 @@ python3 modulus.py build --target x86_64_gcc12_release
|
|||||||
|
|
||||||
`python3 modulus.py run` uses `package/config/vamp.development.json` for a headless readiness check against native build assets. `vamp-zero --once` checks the packaged database and referenced page/audio assets, checks runtime writes and the server bind, then exits without opening audio or display hardware. Missing or incompatible catalogs require rebuilding or reinstalling; they are never repaired at runtime. `--check-config` only validates configuration; `--health-check` probes the running service's `/api/health` endpoint without modifying files.
|
`python3 modulus.py run` uses `package/config/vamp.development.json` for a headless readiness check against native build assets. `vamp-zero --once` checks the packaged database and referenced page/audio assets, checks runtime writes and the server bind, then exits without opening audio or display hardware. Missing or incompatible catalogs require rebuilding or reinstalling; they are never repaired at runtime. `--check-config` only validates configuration; `--health-check` probes the running service's `/api/health` endpoint without modifying files.
|
||||||
|
|
||||||
Configure/build can read public packages anonymously from the khodo Conan registry.
|
Configure/build read public packages anonymously from the khodo Conan registry, even when the project cache contains an expired login. `configure --update` refreshes both the exported Modulus tooling and Conan dependencies. `--no-remote` uses only locally cached packages. Publishing is the only operation that needs credentials. CMake checks the required Sonus and Luma APIs during configuration and reports stale dependencies early.
|
||||||
|
|
||||||
|
`modulus.py` is an unchanged copy of Modulus's canonical bootstrap. The project-specific build image lives in `package/Dockerfile.build`; `tooling.docker` declares its additional inputs and permits Docker socket access only for build/package commands that assemble the Zero image.
|
||||||
|
|
||||||
Refresh clangd compile databases, including the Pico firmware database, with:
|
Refresh clangd compile databases, including the Pico firmware database, with:
|
||||||
|
|
||||||
|
|||||||
+3
-3
@@ -29,7 +29,7 @@ The API is plain HTTP on the private AP:
|
|||||||
- `POST /api/remote/previous`
|
- `POST /api/remote/previous`
|
||||||
- `POST /api/remote/select`
|
- `POST /api/remote/select`
|
||||||
|
|
||||||
Legacy `/api/page/next` and `/api/page/previous` routes are still accepted for simple manual checks.
|
Page commands use `/api/remote/next` and `/api/remote/previous`; the old `/api/page/*` routes are removed.
|
||||||
|
|
||||||
The AP setup pins NetworkManager to 2.4 GHz channel 6 with WPA2/RSN and CCMP ciphers so the Pico W can join using `CYW43_AUTH_WPA2_AES_PSK`.
|
The AP setup pins NetworkManager to 2.4 GHz channel 6 with WPA2/RSN and CCMP ciphers so the Pico W can join using `CYW43_AUTH_WPA2_AES_PSK`.
|
||||||
|
|
||||||
@@ -148,7 +148,7 @@ When grace groups steal from both ends of one donor, their adjustments accumulat
|
|||||||
|
|
||||||
At runtime the shared on-demand score loader serves both display and audio. A background worker validates and loads only the selected score's samples, converts absolute beat positions to rounded sample frames at the menu BPM, and prepares a Sonus sequence. Pitch does not change with BPM, and rounding absolute positions avoids cumulative timing drift. The I/O loop publishes the prepared sequence and polls completion without doing disk I/O or musical interpretation. Sonus mixes preallocated voices from immutable PCM and retires them on control threads; the callback never synthesizes tones or loads files. Limits are one million events, 512 distinct samples, 64 MiB decoded samples per selection, 64 simultaneous voices including releases, and 24 hours at the selected BPM. Exceeding them reports an error rather than dropping notes.
|
At runtime the shared on-demand score loader serves both display and audio. A background worker validates and loads only the selected score's samples, converts absolute beat positions to rounded sample frames at the menu BPM, and prepares a Sonus sequence. Pitch does not change with BPM, and rounding absolute positions avoids cumulative timing drift. The I/O loop publishes the prepared sequence and polls completion without doing disk I/O or musical interpretation. Sonus mixes preallocated voices from immutable PCM and retires them on control threads; the callback never synthesizes tones or loads files. Limits are one million events, 512 distinct samples, 64 MiB decoded samples per selection, 64 simultaneous voices including releases, and 24 hours at the selected BPM. Exceeding them reports an error rather than dropping notes.
|
||||||
|
|
||||||
`audio.playback_enabled` defaults to false and replaces the old inert `track_enabled` setting. The old config name is accepted for migration, but conflicting values fail. Playback suppresses the metronome without changing its stored toggle. Authored tempo marks remain notation; menu BPM is constant throughout a session. Selecting a practice or performance part starts a new session. Manual page changes never restart or seek. Natural completion, including trailing silence/release, returns to the corresponding part menu with its selection preserved. Leaving the score cancels playback; stale results cannot close a newer score. Errors remain visible rather than being mistaken for completion.
|
`audio.playback_enabled` defaults to false. The obsolete `track_enabled` setting is rejected with a diagnostic naming its replacement. Playback suppresses the metronome without changing its stored toggle. Authored tempo marks remain notation; menu BPM is constant throughout a session. Selecting a practice or performance part starts a new session. Manual page changes never restart or seek. Natural completion, including trailing silence/release, returns to the corresponding part menu with its selection preserved. Leaving the score cancels playback; stale results cannot close a newer score. Errors remain visible rather than being mistaken for completion.
|
||||||
|
|
||||||
`--once` additionally validates referenced audio without opening display/audio hardware. Runtime packages contain WAVs alongside pages and the SQLite catalog; the source XML and Python dependencies remain build-only.
|
`--once` additionally validates referenced audio without opening display/audio hardware. Runtime packages contain WAVs alongside pages and the SQLite catalog; the source XML and Python dependencies remain build-only.
|
||||||
|
|
||||||
@@ -174,7 +174,7 @@ Pages are 976x484 pixels with 12-pixel staff spacing, 28-pixel horizontal margin
|
|||||||
|
|
||||||
Generation validates sources and complete staged output before replacing the previous asset directory. Replacement is always complete; deleted sources cannot leave stale JSON, pages, or audio. CMake tracks the input list as well as file contents, so additions and removals regenerate the catalog. Runtime packages contain `catalog.db` with embedded semantic JSON, PNG pages, generated PCM WAVs, and the OTF UI text font, not MusicXML, MuseScore, or Python dependencies. Python test discovery runs model/import/export tests; the normal asset build additionally qualifies and audits every catalog score.
|
Generation validates sources and complete staged output before replacing the previous asset directory. Replacement is always complete; deleted sources cannot leave stale JSON, pages, or audio. CMake tracks the input list as well as file contents, so additions and removals regenerate the catalog. Runtime packages contain `catalog.db` with embedded semantic JSON, PNG pages, generated PCM WAVs, and the OTF UI text font, not MusicXML, MuseScore, or Python dependencies. Python test discovery runs model/import/export tests; the normal asset build additionally qualifies and audits every catalog score.
|
||||||
|
|
||||||
CI page-cache snapshot keys include every generator Python module and XML source, while retaining a renderer/font-compatible restore prefix. Individual cached pages remain validated by their semantic-content and renderer fingerprints.
|
The page cache uses `v2/<renderer fingerprint>/<score fingerprint>/` namespaces. `python3 -m generator --renderer-fingerprint` reports the same identity used by rendering and CI: source bytes, fonts, style, Python implementation/version, and rendering-library versions. Python compiler/build timestamps are excluded. CI keys append all generator and XML inputs and restore older snapshots only within the same renderer namespace; only that namespace is saved after a successful build. A renderer or environment change intentionally starts cold. Progress reports the resolved namespace and reused, missing, and rejected counts. Old flat cache entries are not migrated and can be removed when builds are idle.
|
||||||
|
|
||||||
Diagnostics identify the file, part, measure, and unsupported element or invalid relationship. Examples include `part P1, measure 4: unsupported musical element <slur>`, `tie stop has no contiguous matching start`, and `optional-chord marker must identify exactly one harmony`.
|
Diagnostics identify the file, part, measure, and unsupported element or invalid relationship. Examples include `part P1, measure 4: unsupported musical element <slur>`, `tie stop has no contiguous matching start`, and `optional-chord marker must identify exactly one harmony`.
|
||||||
|
|
||||||
|
|||||||
@@ -12,6 +12,28 @@ find_package(ligo CONFIG REQUIRED)
|
|||||||
find_package(noto CONFIG REQUIRED)
|
find_package(noto CONFIG REQUIRED)
|
||||||
find_package(luma CONFIG REQUIRED)
|
find_package(luma CONFIG REQUIRED)
|
||||||
find_package(sonus CONFIG REQUIRED)
|
find_package(sonus CONFIG REQUIRED)
|
||||||
|
include(CheckCXXSourceCompiles)
|
||||||
|
include(CMakePushCheckState)
|
||||||
|
cmake_push_check_state(RESET)
|
||||||
|
set(CMAKE_TRY_COMPILE_CONFIGURATION "${CMAKE_BUILD_TYPE}")
|
||||||
|
set(CMAKE_REQUIRED_LIBRARIES sonus::sonus luma::luma)
|
||||||
|
unset(VAMP_DEPENDENCY_APIS_AVAILABLE CACHE)
|
||||||
|
check_cxx_source_compiles("
|
||||||
|
#include <sonus.hpp>
|
||||||
|
#include <luma.hpp>
|
||||||
|
#include <utility>
|
||||||
|
static_assert(requires(sonus::engine& engine, sonus::prepared_sequence sequence) {
|
||||||
|
engine.play_sequence(std::move(sequence));
|
||||||
|
});
|
||||||
|
static_assert(requires(luma::application& application, std::string path) {
|
||||||
|
application.load_texture_uncached(path);
|
||||||
|
});
|
||||||
|
int main() { return 0; }
|
||||||
|
" VAMP_DEPENDENCY_APIS_AVAILABLE)
|
||||||
|
cmake_pop_check_state()
|
||||||
|
if(NOT VAMP_DEPENDENCY_APIS_AVAILABLE)
|
||||||
|
message(FATAL_ERROR "Product Vamp requires current Sonus sequence and Luma uncached-texture APIs. Run ./modulus.py configure --update after publishing the updated dependencies, or create their Conan packages locally before configuring --no-remote. See CMake's configure log for the compiler diagnostic.")
|
||||||
|
endif()
|
||||||
include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/font.cmake)
|
include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/font.cmake)
|
||||||
|
|
||||||
execute_process(
|
execute_process(
|
||||||
|
|||||||
+5
-1
@@ -19,8 +19,11 @@ class product_vamp_conan(conan.ConanFile):
|
|||||||
"with_tests": False,
|
"with_tests": False,
|
||||||
}
|
}
|
||||||
python_requires = "modulus/latest"
|
python_requires = "modulus/latest"
|
||||||
exports = "modulus.json"
|
exports = "modulus.json", "!build", "!.git", "!test_package/build"
|
||||||
exports_sources = (
|
exports_sources = (
|
||||||
|
"!build",
|
||||||
|
"!.git",
|
||||||
|
"!test_package/build",
|
||||||
".gitea/workflows/ci.yml",
|
".gitea/workflows/ci.yml",
|
||||||
"CMakeLists.txt",
|
"CMakeLists.txt",
|
||||||
"modulus.py",
|
"modulus.py",
|
||||||
@@ -43,6 +46,7 @@ class product_vamp_conan(conan.ConanFile):
|
|||||||
"include/vamp_zero/*.hpp",
|
"include/vamp_zero/*.hpp",
|
||||||
"package/assets/*",
|
"package/assets/*",
|
||||||
"package/assets/**/*",
|
"package/assets/**/*",
|
||||||
|
"package/Dockerfile.build",
|
||||||
"package/config/*.json",
|
"package/config/*.json",
|
||||||
"package/manifest.json.in",
|
"package/manifest.json.in",
|
||||||
"package/pico/*",
|
"package/pico/*",
|
||||||
|
|||||||
+14
-5
@@ -11,7 +11,8 @@ from generator.practice.generate import generate_practice_scores
|
|||||||
from generator.score import count_json_files, validate_generated_tree
|
from generator.score import count_json_files, validate_generated_tree
|
||||||
from generator.summary import validate_expected_performance_summary, validate_practice_parts
|
from generator.summary import validate_expected_performance_summary, validate_practice_parts
|
||||||
from generator.parts import source_paths
|
from generator.parts import source_paths
|
||||||
from generator.engraving import engrave_tree
|
from generator.engraving import Fonts, Style, engrave_tree
|
||||||
|
from generator.engraving.cache import renderer_fingerprint
|
||||||
from generator.playback import compile_tree
|
from generator.playback import compile_tree
|
||||||
from generator.catalog import build_catalog
|
from generator.catalog import build_catalog
|
||||||
|
|
||||||
@@ -26,14 +27,18 @@ def copy_runtime_fonts(static_assets_dir: Path, output_dir: Path) -> None:
|
|||||||
|
|
||||||
def parse_args() -> argparse.Namespace:
|
def parse_args() -> argparse.Namespace:
|
||||||
parser = argparse.ArgumentParser(description="Generate Product Vamp package score assets.")
|
parser = argparse.ArgumentParser(description="Generate Product Vamp package score assets.")
|
||||||
parser.add_argument("--data-dir", type=Path, required=True)
|
parser.add_argument("--data-dir", type=Path)
|
||||||
parser.add_argument("--output-dir", type=Path, required=True)
|
parser.add_argument("--output-dir", type=Path)
|
||||||
|
parser.add_argument("--renderer-fingerprint", action="store_true", help="Print the engraving cache identity without generating assets.")
|
||||||
parser.add_argument("--static-assets-dir", type=Path, default=Path("package/assets"))
|
parser.add_argument("--static-assets-dir", type=Path, default=Path("package/assets"))
|
||||||
parser.add_argument("--check", action="store_true")
|
parser.add_argument("--check", action="store_true")
|
||||||
parser.add_argument("--engraving-cache", type=Path)
|
parser.add_argument("--engraving-cache", type=Path)
|
||||||
parser.add_argument("--jobs", type=int, default=1, choices=range(1, 17), metavar="1-16")
|
parser.add_argument("--jobs", type=int, default=1, choices=range(1, 17), metavar="1-16")
|
||||||
parser.add_argument("--progress", action="store_true")
|
parser.add_argument("--progress", action="store_true")
|
||||||
return parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
if not args.renderer_fingerprint and (args.data_dir is None or args.output_dir is None):
|
||||||
|
parser.error('--data-dir and --output-dir are required for generation')
|
||||||
|
return args
|
||||||
|
|
||||||
def generate_assets(args: argparse.Namespace) -> None:
|
def generate_assets(args: argparse.Namespace) -> None:
|
||||||
data_dir = args.data_dir.resolve()
|
data_dir = args.data_dir.resolve()
|
||||||
@@ -92,7 +97,11 @@ def generate_assets(args: argparse.Namespace) -> None:
|
|||||||
print('Generator check passed.')
|
print('Generator check passed.')
|
||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
generate_assets(parse_args())
|
args = parse_args()
|
||||||
|
if args.renderer_fingerprint:
|
||||||
|
print(renderer_fingerprint(Fonts(args.static_assets_dir / 'fonts' / 'petaluma'), Style()))
|
||||||
|
else:
|
||||||
|
generate_assets(args)
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ import shutil
|
|||||||
from PIL import Image, ImageChops
|
from PIL import Image, ImageChops
|
||||||
|
|
||||||
from generator.engraving.fonts import Fonts
|
from generator.engraving.fonts import Fonts
|
||||||
from generator.engraving.cache import renderer_fingerprint, restore_pages, score_fingerprint, store_pages
|
from generator.engraving.cache import renderer_fingerprint, renderer_namespace, restore_pages, score_fingerprint, store_pages
|
||||||
from generator.engraving.layout import layout_score
|
from generator.engraving.layout import layout_score
|
||||||
from generator.engraving.raster import rasterize_page
|
from generator.engraving.raster import rasterize_page
|
||||||
from generator.engraving.types import EngravingError, Style
|
from generator.engraving.types import EngravingError, Style
|
||||||
@@ -35,12 +35,11 @@ def check_page(path: Path) -> None:
|
|||||||
if bounds[0] < 4 or bounds[1] < 4 or bounds[2] > PAGE_WIDTH - 4 or bounds[3] > PAGE_HEIGHT - 4:
|
if bounds[0] < 4 or bounds[1] < 4 or bounds[2] > PAGE_WIDTH - 4 or bounds[3] > PAGE_HEIGHT - 4:
|
||||||
raise EngravingError(f'{path.name}: notation reaches the page edge; refusing potentially clipped output')
|
raise EngravingError(f'{path.name}: notation reaches the page edge; refusing potentially clipped output')
|
||||||
|
|
||||||
def engrave_scores(paths: list[Path], root: Path, fonts: Path, cache_dir: Path | None, progress: bool, worker: int) -> tuple[int, int]:
|
def engrave_scores(paths: list[Path], root: Path, fonts: Path, cache_dir: Path | None, progress: bool, worker: int, renderer: str) -> tuple[int, int, int, int]:
|
||||||
family = Fonts(fonts)
|
family = Fonts(fonts)
|
||||||
style = Style()
|
style = Style()
|
||||||
fingerprint = family.fingerprint(style)
|
fingerprint = family.fingerprint(style)
|
||||||
renderer = renderer_fingerprint(family, style) if cache_dir else ''
|
reused, missing, rejected = 0, 0, 0
|
||||||
reused = 0
|
|
||||||
for index, path in enumerate(paths, 1):
|
for index, path in enumerate(paths, 1):
|
||||||
document = json.loads(path.read_text(encoding='utf-8'))
|
document = json.loads(path.read_text(encoding='utf-8'))
|
||||||
try:
|
try:
|
||||||
@@ -49,8 +48,12 @@ def engrave_scores(paths: list[Path], root: Path, fonts: Path, cache_dir: Path |
|
|||||||
if destination.exists():
|
if destination.exists():
|
||||||
shutil.rmtree(destination)
|
shutil.rmtree(destination)
|
||||||
entry = cache_dir / score_fingerprint(document, renderer) if cache_dir else None
|
entry = cache_dir / score_fingerprint(document, renderer) if cache_dir else None
|
||||||
|
present = entry is not None and (entry.exists() or entry.is_symlink())
|
||||||
pages = restore_pages(entry, destination) if entry else None
|
pages = restore_pages(entry, destination) if entry else None
|
||||||
if pages is None:
|
if pages is None:
|
||||||
|
if entry:
|
||||||
|
rejected += int(present)
|
||||||
|
missing += int(not present)
|
||||||
if destination.exists():
|
if destination.exists():
|
||||||
shutil.rmtree(destination)
|
shutil.rmtree(destination)
|
||||||
destination.mkdir()
|
destination.mkdir()
|
||||||
@@ -73,10 +76,10 @@ def engrave_scores(paths: list[Path], root: Path, fonts: Path, cache_dir: Path |
|
|||||||
bundle['playback'] = document['playback']
|
bundle['playback'] = document['playback']
|
||||||
path.write_text(json.dumps(bundle, indent=2, ensure_ascii=False) + '\n', encoding='utf-8')
|
path.write_text(json.dumps(bundle, indent=2, ensure_ascii=False) + '\n', encoding='utf-8')
|
||||||
if progress and (index % 50 == 0 or index == len(paths)):
|
if progress and (index % 50 == 0 or index == len(paths)):
|
||||||
print(f'Engraving worker {worker}: {index}/{len(paths)} scores, {reused} reused', flush=True)
|
print(f'Engraving worker {worker}: {index}/{len(paths)} scores, {reused} reused, {missing} missing, {rejected} rejected', flush=True)
|
||||||
except ValueError as error:
|
except ValueError as error:
|
||||||
raise EngravingError(f'{path.relative_to(root)}: {error}') from error
|
raise EngravingError(f'{path.relative_to(root)}: {error}') from error
|
||||||
return len(paths), reused
|
return len(paths), reused, missing, rejected
|
||||||
|
|
||||||
def engrave_tree(root: Path, fonts: Path, *, cache_dir: Path | None = None, jobs: int = 1, progress: bool = False) -> None:
|
def engrave_tree(root: Path, fonts: Path, *, cache_dir: Path | None = None, jobs: int = 1, progress: bool = False) -> None:
|
||||||
if type(jobs) is not int or not 1 <= jobs <= 16:
|
if type(jobs) is not int or not 1 <= jobs <= 16:
|
||||||
@@ -86,12 +89,21 @@ def engrave_tree(root: Path, fonts: Path, *, cache_dir: Path | None = None, jobs
|
|||||||
paths = sorted(root.rglob('*.json'))
|
paths = sorted(root.rglob('*.json'))
|
||||||
if not paths:
|
if not paths:
|
||||||
return
|
return
|
||||||
|
renderer = renderer_fingerprint(Fonts(fonts), Style()) if cache_dir else ''
|
||||||
|
if cache_dir:
|
||||||
|
cache_dir = renderer_namespace(cache_dir.resolve(), renderer)
|
||||||
|
if progress:
|
||||||
|
state = 'warm namespace' if cache_dir.is_dir() else 'cold namespace'
|
||||||
|
print(f'Engraving cache enabled: {cache_dir} ({state}; renderer {renderer})', flush=True)
|
||||||
|
elif progress:
|
||||||
|
print('Engraving cache disabled', flush=True)
|
||||||
jobs = min(jobs, len(paths))
|
jobs = min(jobs, len(paths))
|
||||||
if jobs == 1:
|
if jobs == 1:
|
||||||
results = [engrave_scores(paths, root, fonts, cache_dir, progress, 1)]
|
results = [engrave_scores(paths, root, fonts, cache_dir, progress, 1, renderer)]
|
||||||
else:
|
else:
|
||||||
with ProcessPoolExecutor(max_workers=jobs, mp_context=multiprocessing.get_context('spawn')) as pool:
|
with ProcessPoolExecutor(max_workers=jobs, mp_context=multiprocessing.get_context('spawn')) as pool:
|
||||||
futures = [pool.submit(engrave_scores, paths[index::jobs], root, fonts, cache_dir, progress, index + 1) for index in range(jobs)]
|
futures = [pool.submit(engrave_scores, paths[index::jobs], root, fonts, cache_dir, progress, index + 1, renderer) for index in range(jobs)]
|
||||||
results = [future.result() for future in futures]
|
results = [future.result() for future in futures]
|
||||||
if progress:
|
if progress:
|
||||||
print(f'Engraving complete: {sum(total for total, _ in results)} scores, {sum(reused for _, reused in results)} reused', flush=True)
|
total, reused, missing, rejected = (sum(row[index] for row in results) for index in range(4))
|
||||||
|
print(f'Engraving complete: {total} scores, {reused} reused, {missing} missing, {rejected} rejected', flush=True)
|
||||||
|
|||||||
@@ -5,20 +5,27 @@ import importlib.metadata
|
|||||||
import json
|
import json
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import shutil
|
import shutil
|
||||||
import sys
|
import platform
|
||||||
import tempfile
|
import tempfile
|
||||||
|
|
||||||
from PIL import features
|
from PIL import features
|
||||||
|
|
||||||
def renderer_fingerprint(fonts, style) -> str:
|
def renderer_identity(fonts, style) -> dict:
|
||||||
root = Path(__file__).resolve().parents[1]
|
root = Path(__file__).resolve().parents[1]
|
||||||
versions = {name: importlib.metadata.version(name) for name in ('Pillow', 'fonttools', 'aggdraw')}
|
versions = {name: importlib.metadata.version(name) for name in ('Pillow', 'fonttools', 'aggdraw')}
|
||||||
versions.update({name: features.version(name) for name in features.get_supported()})
|
versions.update({name: features.version(name) for name in features.get_supported()})
|
||||||
digest = hashlib.sha256(json.dumps([fonts.fingerprint(style), sys.version, versions], sort_keys=True).encode())
|
sources = [*root.joinpath('engraving').glob('*.py'), root / 'model.py', root / 'harmony.py', root / 'requirements.txt']
|
||||||
for path in sorted([*root.joinpath('engraving').glob('*.py'), root / 'model.py', root / 'harmony.py', root / 'requirements.txt']):
|
return {'fonts_style': fonts.fingerprint(style), 'python': {'implementation': platform.python_implementation(), 'version': platform.python_version()}, 'dependencies': versions,
|
||||||
digest.update(path.relative_to(root).as_posix().encode())
|
'sources': {path.relative_to(root).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() for path in sorted(sources)}}
|
||||||
digest.update(path.read_bytes())
|
|
||||||
return digest.hexdigest()
|
def renderer_fingerprint(fonts, style) -> str:
|
||||||
|
return hashlib.sha256(json.dumps(renderer_identity(fonts, style), sort_keys=True).encode()).hexdigest()
|
||||||
|
|
||||||
|
def renderer_namespace(cache: Path, renderer: str) -> Path:
|
||||||
|
namespace = cache / 'v2' / renderer
|
||||||
|
if namespace.parent.is_symlink() or namespace.is_symlink():
|
||||||
|
raise ValueError('engraving cache namespaces must not be symlinks')
|
||||||
|
return namespace
|
||||||
|
|
||||||
def score_fingerprint(document: dict, renderer: str) -> str:
|
def score_fingerprint(document: dict, renderer: str) -> str:
|
||||||
music = {key: value for key, value in document.items() if key != 'playback'}
|
music = {key: value for key, value in document.items() if key != 'playback'}
|
||||||
|
|||||||
+4
-1
@@ -10,6 +10,9 @@ from generator.model import SCHEMA_VERSION, validate_document
|
|||||||
|
|
||||||
def load_json(path: Path) -> dict:
|
def load_json(path: Path) -> dict:
|
||||||
document = json.loads(path.read_text(encoding='utf-8'))
|
document = json.loads(path.read_text(encoding='utf-8'))
|
||||||
|
return validate_json(document, path)
|
||||||
|
|
||||||
|
def validate_json(document: dict, path: Path) -> dict:
|
||||||
if 'music' in document:
|
if 'music' in document:
|
||||||
from generator.engraving import ENGINE_NAME, ENGINE_VERSION, PAGE_HEIGHT, PAGE_WIDTH, check_page
|
from generator.engraving import ENGINE_NAME, ENGINE_VERSION, PAGE_HEIGHT, PAGE_WIDTH, check_page
|
||||||
presentation = document.get('presentation', {})
|
presentation = document.get('presentation', {})
|
||||||
@@ -122,8 +125,8 @@ def count_json_files(root: Path) -> int:
|
|||||||
def validate_generated_tree(root: Path) -> None:
|
def validate_generated_tree(root: Path) -> None:
|
||||||
audio_cache = {}
|
audio_cache = {}
|
||||||
for path in root.rglob('*.json'):
|
for path in root.rglob('*.json'):
|
||||||
load_json(path)
|
|
||||||
document = json.loads(path.read_text(encoding='utf-8'))
|
document = json.loads(path.read_text(encoding='utf-8'))
|
||||||
|
validate_json(document, path)
|
||||||
if 'playback' in document:
|
if 'playback' in document:
|
||||||
from generator.playback import validate_assets
|
from generator.playback import validate_assets
|
||||||
validate_assets(document['playback'], root, audio_cache)
|
validate_assets(document['playback'], root, audio_cache)
|
||||||
|
|||||||
@@ -7,8 +7,6 @@ namespace vamp_protocol {
|
|||||||
|
|
||||||
constexpr std::char_t root_path[] = "/";
|
constexpr std::char_t root_path[] = "/";
|
||||||
constexpr std::char_t status_path[] = "/api/status";
|
constexpr std::char_t status_path[] = "/api/status";
|
||||||
constexpr std::char_t next_page_path[] = "/api/page/next";
|
|
||||||
constexpr std::char_t previous_page_path[] = "/api/page/previous";
|
|
||||||
constexpr std::char_t remote_next_path[] = "/api/remote/next";
|
constexpr std::char_t remote_next_path[] = "/api/remote/next";
|
||||||
constexpr std::char_t remote_previous_path[] = "/api/remote/previous";
|
constexpr std::char_t remote_previous_path[] = "/api/remote/previous";
|
||||||
constexpr std::char_t remote_select_path[] = "/api/remote/select";
|
constexpr std::char_t remote_select_path[] = "/api/remote/select";
|
||||||
@@ -22,7 +20,6 @@ enum class page_command {
|
|||||||
};
|
};
|
||||||
|
|
||||||
auto page_command_path(vamp_protocol::page_command command) -> std::char_t const *;
|
auto page_command_path(vamp_protocol::page_command command) -> std::char_t const *;
|
||||||
auto page_command_legacy_path(vamp_protocol::page_command command) -> std::char_t const *;
|
|
||||||
auto page_command_name(vamp_protocol::page_command command) -> std::char_t const *;
|
auto page_command_name(vamp_protocol::page_command command) -> std::char_t const *;
|
||||||
auto page_command_display_label(vamp_protocol::page_command command) -> std::char_t const *;
|
auto page_command_display_label(vamp_protocol::page_command command) -> std::char_t const *;
|
||||||
|
|
||||||
|
|||||||
+4
-1
@@ -44,7 +44,10 @@
|
|||||||
},
|
},
|
||||||
"default_target": "x86_64_gcc12_debug",
|
"default_target": "x86_64_gcc12_debug",
|
||||||
"docker": {
|
"docker": {
|
||||||
"image": "product-vamp-modulus-build:dev"
|
"image": "product-vamp-modulus-build:dev",
|
||||||
|
"dockerfile": "package/Dockerfile.build",
|
||||||
|
"inputs": ["generator/requirements.txt"],
|
||||||
|
"socket_commands": ["build", "package"]
|
||||||
},
|
},
|
||||||
"remote": {
|
"remote": {
|
||||||
"name": "khodo",
|
"name": "khodo",
|
||||||
|
|||||||
+58
-252
@@ -1,18 +1,15 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
# Canonical consumer bootstrap. Copy this file unchanged to a project's modulus.py.
|
||||||
import json
|
import json
|
||||||
import hashlib
|
|
||||||
import os
|
import os
|
||||||
import pathlib
|
import pathlib
|
||||||
import shlex
|
import shlex
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import time
|
|
||||||
|
|
||||||
PACKAGE_REF = "modulus/latest"
|
PACKAGE_REF = "modulus/latest"
|
||||||
CONAN_VERSION = "2.26.2"
|
CONAN_VERSION = "2.26.2"
|
||||||
FETCH_IMAGE = "modulus-fetch:dev"
|
FETCH_IMAGE = "modulus-fetch:dev"
|
||||||
DEFAULT_BUILD_BASE_IMAGE = "debian:trixie-slim"
|
|
||||||
PICO_SDK_VERSION = "2.3.0"
|
|
||||||
COMMANDS = (
|
COMMANDS = (
|
||||||
"configure",
|
"configure",
|
||||||
"build",
|
"build",
|
||||||
@@ -29,6 +26,7 @@ RUNTIME_DIR = COMMON_DIR / "modulus"
|
|||||||
CONAN_HOME_DIR = COMMON_DIR / "conan_cache"
|
CONAN_HOME_DIR = COMMON_DIR / "conan_cache"
|
||||||
DEFAULT_REMOTE_NAME = "khodo"
|
DEFAULT_REMOTE_NAME = "khodo"
|
||||||
DEFAULT_REMOTE_URL = "https://git.thyme-labs.com/api/packages/khodo/conan"
|
DEFAULT_REMOTE_URL = "https://git.thyme-labs.com/api/packages/khodo/conan"
|
||||||
|
DEFAULT_DOCKER_MARKER = "/opt/modulus-build-image"
|
||||||
|
|
||||||
def repo_root():
|
def repo_root():
|
||||||
root = pathlib.Path(__file__).resolve().parent
|
root = pathlib.Path(__file__).resolve().parent
|
||||||
@@ -63,7 +61,6 @@ def runtime_ready(root):
|
|||||||
(root / TOOLS_DIR / "configure.py").is_file()
|
(root / TOOLS_DIR / "configure.py").is_file()
|
||||||
and (root / TOOLS_DIR / "lsp.py").is_file()
|
and (root / TOOLS_DIR / "lsp.py").is_file()
|
||||||
and (root / TOOLS_DIR / "clangd.py").is_file()
|
and (root / TOOLS_DIR / "clangd.py").is_file()
|
||||||
and (root / TOOLS_DIR / "run.py").is_file()
|
|
||||||
and (root / RUNTIME_DIR / "modulus" / "cli.py").is_file()
|
and (root / RUNTIME_DIR / "modulus" / "cli.py").is_file()
|
||||||
and (root / RUNTIME_DIR / "modulus" / "export.py").is_file()
|
and (root / RUNTIME_DIR / "modulus" / "export.py").is_file()
|
||||||
and (root / TOOLS_DIR / "format.py").is_file()
|
and (root / TOOLS_DIR / "format.py").is_file()
|
||||||
@@ -71,18 +68,19 @@ def runtime_ready(root):
|
|||||||
and (root / RUNTIME_DIR / "modulus" / "format_layout.py").is_file()
|
and (root / RUNTIME_DIR / "modulus" / "format_layout.py").is_file()
|
||||||
)
|
)
|
||||||
|
|
||||||
def optional_credentials():
|
|
||||||
user = os.environ.get("CONAN_USER")
|
|
||||||
token = os.environ.get("CONAN_TOKEN")
|
|
||||||
if bool(user) != bool(token):
|
|
||||||
raise SystemExit("CONAN_USER and CONAN_TOKEN must either both be set or both be unset")
|
|
||||||
return user, token
|
|
||||||
|
|
||||||
def env_flag(name):
|
def env_flag(name):
|
||||||
return os.environ.get(name, "").lower() in {"1", "true", "yes", "on"}
|
return os.environ.get(name, "").lower() in {"1", "true", "yes", "on"}
|
||||||
|
|
||||||
|
def docker_marker(root):
|
||||||
|
config = read_json(root / "modulus.json")
|
||||||
|
marker = config.get(
|
||||||
|
"tooling",
|
||||||
|
{},
|
||||||
|
).get("docker", {}).get("marker", DEFAULT_DOCKER_MARKER)
|
||||||
|
return pathlib.Path(str(marker))
|
||||||
|
|
||||||
def direct_mode(root):
|
def direct_mode(root):
|
||||||
return env_flag("MODULUS_DIRECT") or project_docker_marker(root).exists()
|
return env_flag("MODULUS_DIRECT") or docker_marker(root).exists()
|
||||||
|
|
||||||
def run(command, input_text=None, capture=False, display_command=None, env=None, check=True):
|
def run(command, input_text=None, capture=False, display_command=None, env=None, check=True):
|
||||||
visible = display_command if display_command is not None else command
|
visible = display_command if display_command is not None else command
|
||||||
@@ -104,20 +102,8 @@ def run(command, input_text=None, capture=False, display_command=None, env=None,
|
|||||||
raise SystemExit(completed.returncode)
|
raise SystemExit(completed.returncode)
|
||||||
return completed
|
return completed
|
||||||
|
|
||||||
def run_with_retries(command, attempts=3, input_text=None, display_command=None, env=None):
|
|
||||||
completed = None
|
|
||||||
for attempt in range(1, attempts + 1):
|
|
||||||
completed = run(command, input_text=input_text, display_command=display_command, env=env, check=False)
|
|
||||||
if completed.returncode == 0:
|
|
||||||
return completed
|
|
||||||
if attempt != attempts:
|
|
||||||
delay = attempt * 20
|
|
||||||
print(f"command failed with exit code {completed.returncode}; retrying in {delay}s")
|
|
||||||
time.sleep(delay)
|
|
||||||
raise SystemExit(completed.returncode if completed is not None else 1)
|
|
||||||
|
|
||||||
def fetch_dockerfile():
|
def fetch_dockerfile():
|
||||||
return f"""FROM {DEFAULT_BUILD_BASE_IMAGE}
|
return f"""FROM debian:bookworm-slim
|
||||||
|
|
||||||
ENV DEBIAN_FRONTEND=noninteractive
|
ENV DEBIAN_FRONTEND=noninteractive
|
||||||
|
|
||||||
@@ -137,112 +123,23 @@ RUN python -m pip install --no-cache-dir conan=={CONAN_VERSION}
|
|||||||
WORKDIR /workspace
|
WORKDIR /workspace
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def build_requirements_digest():
|
def ensure_fetch_image(offline=False):
|
||||||
return hashlib.sha256((repo_root() / "generator/requirements.txt").read_bytes()).hexdigest()
|
inspected = subprocess.run(
|
||||||
|
[
|
||||||
def product_build_dockerfile():
|
"docker",
|
||||||
return f"""FROM {DEFAULT_BUILD_BASE_IMAGE}
|
"image",
|
||||||
|
"inspect",
|
||||||
ENV DEBIAN_FRONTEND=noninteractive
|
FETCH_IMAGE,
|
||||||
|
],
|
||||||
RUN dpkg --add-architecture arm64
|
stdout=subprocess.DEVNULL,
|
||||||
|
stderr=subprocess.DEVNULL,
|
||||||
RUN apt-get update \\
|
check=False,
|
||||||
&& apt-get install -y --no-install-recommends \\
|
)
|
||||||
bash \\
|
if inspected.returncode == 0 and (offline or not docker_platform_args()):
|
||||||
build-essential \\
|
|
||||||
ca-certificates \\
|
|
||||||
clangd-19 \\
|
|
||||||
cmake \\
|
|
||||||
curl \\
|
|
||||||
docker-buildx \\
|
|
||||||
docker-cli \\
|
|
||||||
docker.io \\
|
|
||||||
g++-12 \\
|
|
||||||
g++-12-aarch64-linux-gnu \\
|
|
||||||
gcc-12 \\
|
|
||||||
gcc-12-aarch64-linux-gnu \\
|
|
||||||
gcc-arm-none-eabi \\
|
|
||||||
git \\
|
|
||||||
libdrm-dev \\
|
|
||||||
libdrm-dev:arm64 \\
|
|
||||||
libegl-dev \\
|
|
||||||
libegl-dev:arm64 \\
|
|
||||||
libfribidi0 \\
|
|
||||||
libgbm-dev \\
|
|
||||||
libgbm-dev:arm64 \\
|
|
||||||
libgles-dev \\
|
|
||||||
libgles-dev:arm64 \\
|
|
||||||
libglvnd-dev \\
|
|
||||||
libglvnd-dev:arm64 \\
|
|
||||||
libnewlib-arm-none-eabi \\
|
|
||||||
libstdc++-arm-none-eabi-newlib \\
|
|
||||||
ninja-build \\
|
|
||||||
pkg-config \\
|
|
||||||
python3 \\
|
|
||||||
python3-venv \\
|
|
||||||
&& ln -sf /usr/bin/clangd-19 /usr/local/bin/clangd \\
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
ENV CC=/usr/bin/gcc-12
|
|
||||||
ENV CXX=/usr/bin/g++-12
|
|
||||||
ENV VAMP_PICO_SDK_VERSION={PICO_SDK_VERSION}
|
|
||||||
ENV PICO_SDK_PATH=/opt/pico-sdk
|
|
||||||
|
|
||||||
RUN git clone \\
|
|
||||||
--depth 1 \\
|
|
||||||
--branch "$VAMP_PICO_SDK_VERSION" \\
|
|
||||||
--recurse-submodules \\
|
|
||||||
--shallow-submodules \\
|
|
||||||
https://github.com/raspberrypi/pico-sdk.git \\
|
|
||||||
"$PICO_SDK_PATH"
|
|
||||||
|
|
||||||
RUN python3 -m venv /opt/modulus-python
|
|
||||||
|
|
||||||
ENV PATH=/opt/modulus-python/bin:$PATH
|
|
||||||
|
|
||||||
COPY generator/requirements.txt /tmp/vamp-build-requirements.txt
|
|
||||||
ARG MODULUS_TOOLS=build/build_tools
|
|
||||||
COPY ${{MODULUS_TOOLS}}/docker/format-requirements.txt /tmp/modulus-format-requirements.txt
|
|
||||||
|
|
||||||
RUN python -m pip install --no-cache-dir \\
|
|
||||||
conan=={CONAN_VERSION} -r /tmp/vamp-build-requirements.txt -r /tmp/modulus-format-requirements.txt
|
|
||||||
|
|
||||||
LABEL org.khodo.product-vamp.build-requirements="{build_requirements_digest()}"
|
|
||||||
|
|
||||||
WORKDIR /workspace
|
|
||||||
|
|
||||||
RUN mkdir -p /opt && touch /opt/modulus-build-image
|
|
||||||
"""
|
|
||||||
|
|
||||||
def sync_product_build_dockerfile(root):
|
|
||||||
dockerfile = root / TOOLS_DIR / "docker" / "Dockerfile"
|
|
||||||
if not dockerfile.parent.is_dir():
|
|
||||||
return
|
return
|
||||||
desired = product_build_dockerfile()
|
if offline:
|
||||||
try:
|
raise SystemExit('offline bootstrap requires the existing Modulus fetch image or MODULUS_DIRECT=1 with Conan installed')
|
||||||
current = dockerfile.read_text(encoding="utf-8")
|
run(
|
||||||
except OSError:
|
|
||||||
current = None
|
|
||||||
if current != desired:
|
|
||||||
dockerfile.write_text(desired, encoding="utf-8")
|
|
||||||
|
|
||||||
def ensure_fetch_image():
|
|
||||||
if not docker_platform_args():
|
|
||||||
inspected = subprocess.run(
|
|
||||||
[
|
|
||||||
"docker",
|
|
||||||
"image",
|
|
||||||
"inspect",
|
|
||||||
FETCH_IMAGE,
|
|
||||||
],
|
|
||||||
stdout=subprocess.DEVNULL,
|
|
||||||
stderr=subprocess.DEVNULL,
|
|
||||||
check=False,
|
|
||||||
)
|
|
||||||
if inspected.returncode == 0:
|
|
||||||
return
|
|
||||||
run_with_retries(
|
|
||||||
[
|
[
|
||||||
"docker",
|
"docker",
|
||||||
"build",
|
"build",
|
||||||
@@ -257,6 +154,10 @@ def ensure_fetch_image():
|
|||||||
def in_container():
|
def in_container():
|
||||||
return pathlib.Path("/.dockerenv").exists()
|
return pathlib.Path("/.dockerenv").exists()
|
||||||
|
|
||||||
|
def docker_platform_args():
|
||||||
|
platform = os.environ.get("MODULUS_DOCKER_PLATFORM", "").strip()
|
||||||
|
return ["--platform", platform] if platform else []
|
||||||
|
|
||||||
def docker_workspace_args(root):
|
def docker_workspace_args(root):
|
||||||
if in_container():
|
if in_container():
|
||||||
container = os.environ.get("HOSTNAME", "").strip()
|
container = os.environ.get("HOSTNAME", "").strip()
|
||||||
@@ -271,43 +172,26 @@ def docker_workspace_args(root):
|
|||||||
f"{root}:{root}",
|
f"{root}:{root}",
|
||||||
]
|
]
|
||||||
|
|
||||||
def docker_socket_args():
|
|
||||||
socket = pathlib.Path("/var/run/docker.sock")
|
|
||||||
if not socket.is_socket():
|
|
||||||
return []
|
|
||||||
args = [
|
|
||||||
"--volume",
|
|
||||||
"/var/run/docker.sock:/var/run/docker.sock",
|
|
||||||
]
|
|
||||||
try:
|
|
||||||
args.extend(["--group-add", str(socket.stat().st_gid)])
|
|
||||||
except OSError:
|
|
||||||
pass
|
|
||||||
return args
|
|
||||||
|
|
||||||
def docker_platform_args():
|
|
||||||
platform = os.environ.get("MODULUS_DOCKER_PLATFORM", "").strip()
|
|
||||||
if not platform:
|
|
||||||
return []
|
|
||||||
return [
|
|
||||||
"--platform",
|
|
||||||
platform,
|
|
||||||
]
|
|
||||||
|
|
||||||
def fetch_env(root):
|
def fetch_env(root):
|
||||||
env = os.environ.copy()
|
env = os.environ.copy()
|
||||||
|
for name in list(env):
|
||||||
|
if name in {"CONAN_USER", "CONAN_TOKEN"} or name.startswith(("CONAN_LOGIN_USERNAME", "CONAN_PASSWORD")):
|
||||||
|
del env[name]
|
||||||
env["CONAN_HOME"] = str(root / CONAN_HOME_DIR)
|
env["CONAN_HOME"] = str(root / CONAN_HOME_DIR)
|
||||||
|
env["CONAN_NON_INTERACTIVE"] = "1"
|
||||||
return env
|
return env
|
||||||
|
|
||||||
def run_fetch(root, command, capture=False, display_command=None, check=True):
|
def run_fetch(root, command, capture=False, display_command=None, check=True, offline=False):
|
||||||
if direct_mode(root):
|
if direct_mode(root):
|
||||||
return run(command, capture=capture, display_command=display_command, env=fetch_env(root), check=check)
|
return run(command, capture=capture, display_command=display_command, env=fetch_env(root), check=check)
|
||||||
ensure_fetch_image()
|
ensure_fetch_image(offline=offline)
|
||||||
docker_command = [
|
docker_command = [
|
||||||
"docker",
|
"docker",
|
||||||
"run",
|
"run",
|
||||||
"--rm",
|
"--rm",
|
||||||
"--interactive",
|
"--interactive",
|
||||||
|
*(["--pull=never", "--network=none"] if offline else []),
|
||||||
|
*docker_platform_args(),
|
||||||
"--user",
|
"--user",
|
||||||
f"{os.getuid()}:{os.getgid()}",
|
f"{os.getuid()}:{os.getgid()}",
|
||||||
"--env",
|
"--env",
|
||||||
@@ -315,13 +199,8 @@ def run_fetch(root, command, capture=False, display_command=None, check=True):
|
|||||||
"--env",
|
"--env",
|
||||||
"CONAN_HOME",
|
"CONAN_HOME",
|
||||||
"--env",
|
"--env",
|
||||||
"CONAN_USER",
|
"CONAN_NON_INTERACTIVE",
|
||||||
"--env",
|
|
||||||
"CONAN_TOKEN",
|
|
||||||
"--env",
|
|
||||||
"DOCKER_HOST",
|
|
||||||
*docker_workspace_args(root),
|
*docker_workspace_args(root),
|
||||||
*docker_socket_args(),
|
|
||||||
"--workdir",
|
"--workdir",
|
||||||
str(root),
|
str(root),
|
||||||
FETCH_IMAGE,
|
FETCH_IMAGE,
|
||||||
@@ -330,7 +209,6 @@ def run_fetch(root, command, capture=False, display_command=None, check=True):
|
|||||||
return run(docker_command, capture=capture, display_command=display_command, env=fetch_env(root), check=check)
|
return run(docker_command, capture=capture, display_command=display_command, env=fetch_env(root), check=check)
|
||||||
|
|
||||||
def configure_remote(root, remote_name, remote_url):
|
def configure_remote(root, remote_name, remote_url):
|
||||||
user, token = optional_credentials()
|
|
||||||
run_fetch(
|
run_fetch(
|
||||||
root,
|
root,
|
||||||
[
|
[
|
||||||
@@ -342,29 +220,7 @@ def configure_remote(root, remote_name, remote_url):
|
|||||||
remote_url,
|
remote_url,
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
if not user and not token:
|
run_fetch(root, ["conan", "remote", "logout", remote_name])
|
||||||
return
|
|
||||||
run_fetch(
|
|
||||||
root,
|
|
||||||
[
|
|
||||||
"conan",
|
|
||||||
"remote",
|
|
||||||
"login",
|
|
||||||
remote_name,
|
|
||||||
user,
|
|
||||||
"-p",
|
|
||||||
token,
|
|
||||||
],
|
|
||||||
display_command=[
|
|
||||||
"conan",
|
|
||||||
"remote",
|
|
||||||
"login",
|
|
||||||
remote_name,
|
|
||||||
user,
|
|
||||||
"-p",
|
|
||||||
"<hidden>",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
def package_refs(data):
|
def package_refs(data):
|
||||||
refs = []
|
refs = []
|
||||||
@@ -401,46 +257,26 @@ def package_refs(data):
|
|||||||
refs.append(f"{PACKAGE_REF}#{recipe_revision}:{package_id}")
|
refs.append(f"{PACKAGE_REF}#{recipe_revision}:{package_id}")
|
||||||
return sorted(refs)
|
return sorted(refs)
|
||||||
|
|
||||||
def remove_cached_modulus(root):
|
def find_modulus(root, remote_name=None):
|
||||||
run_fetch(
|
|
||||||
root,
|
|
||||||
[
|
|
||||||
"conan",
|
|
||||||
"remove",
|
|
||||||
PACKAGE_REF + ":*",
|
|
||||||
"-c",
|
|
||||||
],
|
|
||||||
check=False,
|
|
||||||
)
|
|
||||||
|
|
||||||
def download_modulus(root, remote_name):
|
|
||||||
remove_cached_modulus(root)
|
|
||||||
run_fetch(
|
|
||||||
root,
|
|
||||||
[
|
|
||||||
"conan",
|
|
||||||
"download",
|
|
||||||
PACKAGE_REF + ":*",
|
|
||||||
"-r",
|
|
||||||
remote_name,
|
|
||||||
],
|
|
||||||
)
|
|
||||||
completed = run_fetch(
|
completed = run_fetch(
|
||||||
root,
|
root,
|
||||||
[
|
[
|
||||||
"conan",
|
"conan",
|
||||||
"list",
|
"list",
|
||||||
PACKAGE_REF + ":*",
|
PACKAGE_REF + "#latest:*#latest",
|
||||||
"-c",
|
*(["-r", remote_name] if remote_name else ["-c"]),
|
||||||
"-f",
|
"-f",
|
||||||
"json",
|
"json",
|
||||||
],
|
],
|
||||||
capture=True,
|
capture=True,
|
||||||
|
offline=remote_name is None,
|
||||||
)
|
)
|
||||||
refs = package_refs(json.loads(completed.stdout))
|
refs = package_refs(json.loads(completed.stdout))
|
||||||
if not refs:
|
if not refs:
|
||||||
raise SystemExit(f"no downloaded {PACKAGE_REF} package found in the Conan cache")
|
raise SystemExit(f"no {PACKAGE_REF} package available in {remote_name or 'the local Conan cache'}")
|
||||||
package_ref = refs[-1]
|
package_ref = refs[-1]
|
||||||
|
if remote_name:
|
||||||
|
run_fetch(root, ["conan", "download", package_ref, "-r", remote_name])
|
||||||
completed = run_fetch(
|
completed = run_fetch(
|
||||||
root,
|
root,
|
||||||
[
|
[
|
||||||
@@ -450,40 +286,25 @@ def download_modulus(root, remote_name):
|
|||||||
package_ref,
|
package_ref,
|
||||||
],
|
],
|
||||||
capture=True,
|
capture=True,
|
||||||
|
offline=remote_name is None,
|
||||||
)
|
)
|
||||||
package_root = pathlib.Path(completed.stdout.strip())
|
package_root = pathlib.Path(completed.stdout.strip())
|
||||||
if not (package_root / "modulus" / "export.py").is_file() or not (package_root / "tools").is_dir():
|
if not (package_root / "modulus" / "export.py").is_file() or not (package_root / "tools").is_dir():
|
||||||
raise SystemExit(f"downloaded {PACKAGE_REF} package does not contain Modulus export files: {package_root}")
|
raise SystemExit(f"downloaded {PACKAGE_REF} package does not contain Modulus export files: {package_root}")
|
||||||
return package_root
|
return package_root
|
||||||
|
|
||||||
def materialize_runtime(root):
|
def materialize_runtime(root, no_remote=False):
|
||||||
remote_name, remote_url = remote_config(root)
|
remote_name, remote_url = remote_config(root)
|
||||||
configure_remote(root, remote_name, remote_url)
|
if not no_remote:
|
||||||
package_root = download_modulus(root, remote_name)
|
configure_remote(root, remote_name, remote_url)
|
||||||
|
package_root = find_modulus(root, None if no_remote else remote_name)
|
||||||
sys.path.insert(0, str(package_root))
|
sys.path.insert(0, str(package_root))
|
||||||
import modulus.export
|
import modulus.export
|
||||||
modulus.export.sync_project(root, package_root)
|
modulus.export.sync_project(root, package_root)
|
||||||
|
|
||||||
def ensure_runtime(root):
|
def ensure_runtime(root, update=False, no_remote=False):
|
||||||
if not runtime_ready(root):
|
if update or not runtime_ready(root):
|
||||||
materialize_runtime(root)
|
materialize_runtime(root, no_remote=no_remote)
|
||||||
sync_product_build_dockerfile(root)
|
|
||||||
|
|
||||||
def project_docker_marker(root):
|
|
||||||
config = read_json(root / "modulus.json")
|
|
||||||
tooling = config.get("tooling", {})
|
|
||||||
if not isinstance(tooling, dict):
|
|
||||||
tooling = {}
|
|
||||||
docker = tooling.get("docker", {})
|
|
||||||
if not isinstance(docker, dict):
|
|
||||||
docker = {}
|
|
||||||
return pathlib.Path(str(docker.get("marker") or "/opt/modulus-build-image"))
|
|
||||||
|
|
||||||
def run_configure(argv):
|
|
||||||
return delegate("configure", argv)
|
|
||||||
|
|
||||||
def run_package(argv):
|
|
||||||
return delegate("package", argv)
|
|
||||||
|
|
||||||
def delegate(command, argv):
|
def delegate(command, argv):
|
||||||
root = repo_root()
|
root = repo_root()
|
||||||
@@ -492,19 +313,9 @@ def delegate(command, argv):
|
|||||||
print(f"Modulus {command} tooling is missing; run ./modulus.py configure first", file=sys.stderr)
|
print(f"Modulus {command} tooling is missing; run ./modulus.py configure first", file=sys.stderr)
|
||||||
return 1
|
return 1
|
||||||
else:
|
else:
|
||||||
ensure_runtime(root)
|
ensure_runtime(root, update=command == "configure" and "--update" in argv, no_remote="--no-remote" in argv)
|
||||||
sys.path.insert(0, str(root / RUNTIME_DIR))
|
sys.path.insert(0, str(root / RUNTIME_DIR))
|
||||||
import modulus.cli
|
import modulus.cli
|
||||||
original_docker_run = modulus.cli.docker_run
|
|
||||||
|
|
||||||
def docker_run_with_host_socket(context, command, docker_args=None, env_overrides=None):
|
|
||||||
merged_docker_args = [
|
|
||||||
*docker_socket_args(),
|
|
||||||
*(docker_args or []),
|
|
||||||
]
|
|
||||||
return original_docker_run(context, command, docker_args=merged_docker_args, env_overrides=env_overrides)
|
|
||||||
|
|
||||||
modulus.cli.docker_run = docker_run_with_host_socket
|
|
||||||
previous_argv = sys.argv
|
previous_argv = sys.argv
|
||||||
previous_cwd = pathlib.Path.cwd()
|
previous_cwd = pathlib.Path.cwd()
|
||||||
sys.argv = [
|
sys.argv = [
|
||||||
@@ -515,7 +326,6 @@ def delegate(command, argv):
|
|||||||
try:
|
try:
|
||||||
return modulus.cli.main(command)
|
return modulus.cli.main(command)
|
||||||
finally:
|
finally:
|
||||||
modulus.cli.docker_run = original_docker_run
|
|
||||||
os.chdir(previous_cwd)
|
os.chdir(previous_cwd)
|
||||||
sys.argv = previous_argv
|
sys.argv = previous_argv
|
||||||
|
|
||||||
@@ -526,10 +336,6 @@ def main(argv):
|
|||||||
command = argv[0]
|
command = argv[0]
|
||||||
if command not in COMMANDS:
|
if command not in COMMANDS:
|
||||||
raise SystemExit(f"unknown Modulus command: {command}")
|
raise SystemExit(f"unknown Modulus command: {command}")
|
||||||
if command == "configure":
|
|
||||||
return run_configure(argv[1:])
|
|
||||||
if command == "package":
|
|
||||||
return run_package(argv[1:])
|
|
||||||
return delegate(command, argv[1:])
|
return delegate(command, argv[1:])
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -0,0 +1,69 @@
|
|||||||
|
FROM debian:trixie-slim
|
||||||
|
|
||||||
|
ENV DEBIAN_FRONTEND=noninteractive
|
||||||
|
|
||||||
|
RUN dpkg --add-architecture arm64
|
||||||
|
|
||||||
|
RUN apt-get update \
|
||||||
|
&& apt-get install -y --no-install-recommends \
|
||||||
|
bash \
|
||||||
|
build-essential \
|
||||||
|
ca-certificates \
|
||||||
|
clangd-19 \
|
||||||
|
cmake \
|
||||||
|
curl \
|
||||||
|
docker-buildx \
|
||||||
|
docker-cli \
|
||||||
|
docker.io \
|
||||||
|
g++-12 \
|
||||||
|
g++-12-aarch64-linux-gnu \
|
||||||
|
gcc-12 \
|
||||||
|
gcc-12-aarch64-linux-gnu \
|
||||||
|
gcc-arm-none-eabi \
|
||||||
|
git \
|
||||||
|
libdrm-dev \
|
||||||
|
libdrm-dev:arm64 \
|
||||||
|
libegl-dev \
|
||||||
|
libegl-dev:arm64 \
|
||||||
|
libfribidi0 \
|
||||||
|
libgbm-dev \
|
||||||
|
libgbm-dev:arm64 \
|
||||||
|
libgles-dev \
|
||||||
|
libgles-dev:arm64 \
|
||||||
|
libglvnd-dev \
|
||||||
|
libglvnd-dev:arm64 \
|
||||||
|
libnewlib-arm-none-eabi \
|
||||||
|
libstdc++-arm-none-eabi-newlib \
|
||||||
|
ninja-build \
|
||||||
|
pkg-config \
|
||||||
|
python3 \
|
||||||
|
python3-venv \
|
||||||
|
&& ln -sf /usr/bin/clangd-19 /usr/local/bin/clangd \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
ENV CC=/usr/bin/gcc-12
|
||||||
|
ENV CXX=/usr/bin/g++-12
|
||||||
|
ENV VAMP_PICO_SDK_VERSION=2.3.0
|
||||||
|
ENV PICO_SDK_PATH=/opt/pico-sdk
|
||||||
|
|
||||||
|
RUN git clone \
|
||||||
|
--depth 1 \
|
||||||
|
--branch "$VAMP_PICO_SDK_VERSION" \
|
||||||
|
--recurse-submodules \
|
||||||
|
--shallow-submodules \
|
||||||
|
https://github.com/raspberrypi/pico-sdk.git \
|
||||||
|
"$PICO_SDK_PATH"
|
||||||
|
|
||||||
|
RUN python3 -m venv /opt/modulus-python
|
||||||
|
|
||||||
|
ENV PATH=/opt/modulus-python/bin:$PATH
|
||||||
|
|
||||||
|
COPY generator/requirements.txt /tmp/vamp-build-requirements.txt
|
||||||
|
ARG MODULUS_TOOLS=build/build_tools
|
||||||
|
COPY ${MODULUS_TOOLS}/docker/format-requirements.txt /tmp/modulus-format-requirements.txt
|
||||||
|
|
||||||
|
RUN python -m pip install --no-cache-dir \
|
||||||
|
conan==2.26.2 -r /tmp/vamp-build-requirements.txt -r /tmp/modulus-format-requirements.txt
|
||||||
|
WORKDIR /workspace
|
||||||
|
|
||||||
|
RUN mkdir -p /opt && touch /opt/modulus-build-image
|
||||||
@@ -13,18 +13,6 @@ auto vamp_protocol::page_command_path(vamp_protocol::page_command command) -> st
|
|||||||
return vamp_protocol::remote_next_path;
|
return vamp_protocol::remote_next_path;
|
||||||
}
|
}
|
||||||
|
|
||||||
auto vamp_protocol::page_command_legacy_path(vamp_protocol::page_command command) -> std::char_t const * {
|
|
||||||
switch (command) {
|
|
||||||
case vamp_protocol::page_command::PREVIOUS:
|
|
||||||
return vamp_protocol::previous_page_path;
|
|
||||||
case vamp_protocol::page_command::NEXT:
|
|
||||||
return vamp_protocol::next_page_path;
|
|
||||||
case vamp_protocol::page_command::SELECT:
|
|
||||||
return vamp_protocol::remote_select_path;
|
|
||||||
}
|
|
||||||
return vamp_protocol::next_page_path;
|
|
||||||
}
|
|
||||||
|
|
||||||
auto vamp_protocol::page_command_name(vamp_protocol::page_command command) -> std::char_t const * {
|
auto vamp_protocol::page_command_name(vamp_protocol::page_command command) -> std::char_t const * {
|
||||||
switch (command) {
|
switch (command) {
|
||||||
case vamp_protocol::page_command::PREVIOUS:
|
case vamp_protocol::page_command::PREVIOUS:
|
||||||
|
|||||||
+25
-81
@@ -1,4 +1,3 @@
|
|||||||
#include <cmath>
|
|
||||||
#include <cstdint>
|
#include <cstdint>
|
||||||
#include <expected>
|
#include <expected>
|
||||||
#include <string>
|
#include <string>
|
||||||
@@ -6,9 +5,7 @@
|
|||||||
#include <utility>
|
#include <utility>
|
||||||
|
|
||||||
#include <boost/json/object.hpp>
|
#include <boost/json/object.hpp>
|
||||||
#include <boost/json/parse.hpp>
|
|
||||||
#include <boost/json/value.hpp>
|
#include <boost/json/value.hpp>
|
||||||
#include <boost/system/error_code.hpp>
|
|
||||||
|
|
||||||
#include <vamp_zero/config.hpp>
|
#include <vamp_zero/config.hpp>
|
||||||
#include <vamp_zero/errors.hpp>
|
#include <vamp_zero/errors.hpp>
|
||||||
@@ -22,93 +19,49 @@ constexpr auto json_options = modulus::json_config::options{
|
|||||||
.not_found_code = static_cast<std::int32_t>(vamp_zero::error_code::NOT_FOUND),
|
.not_found_code = static_cast<std::int32_t>(vamp_zero::error_code::NOT_FOUND),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
constexpr auto field_options = modulus::json_config::options{
|
||||||
|
.category = "product-vamp",
|
||||||
|
.invalid_argument_code = static_cast<std::int32_t>(vamp_zero::error_code::INVALID_ARGUMENT),
|
||||||
|
.not_found_code = static_cast<std::int32_t>(vamp_zero::error_code::NOT_FOUND),
|
||||||
|
};
|
||||||
|
|
||||||
auto invalid(std::string message) -> modulus::error {
|
auto invalid(std::string message) -> modulus::error {
|
||||||
return vamp_zero::make_error(vamp_zero::error_code::INVALID_ARGUMENT, std::move(message));
|
return vamp_zero::make_error(vamp_zero::error_code::INVALID_ARGUMENT, std::move(message));
|
||||||
}
|
}
|
||||||
|
|
||||||
auto require_object(boost::json::value const &value, std::string_view name) -> std::expected<boost::json::object const *, modulus::error> {
|
|
||||||
if (!value.is_object()) {
|
|
||||||
return std::unexpected(invalid(std::string{name} + " must be an object"));
|
|
||||||
}
|
|
||||||
return &value.as_object();
|
|
||||||
}
|
|
||||||
|
|
||||||
auto optional_object(boost::json::object const &object, std::string_view field) -> std::expected<boost::json::object const *, modulus::error> {
|
auto optional_object(boost::json::object const &object, std::string_view field) -> std::expected<boost::json::object const *, modulus::error> {
|
||||||
auto found = object.find(field);
|
auto found = object.find(field);
|
||||||
if (found == object.end()) {
|
if (found == object.end()) {
|
||||||
return nullptr;
|
return nullptr;
|
||||||
}
|
}
|
||||||
return require_object(found->value(), field);
|
return modulus::json_config::require_object(found->value(), field, field_options);
|
||||||
}
|
}
|
||||||
|
|
||||||
auto optional_string(boost::json::object const &object, std::string_view field, std::string fallback) -> std::expected<std::string, modulus::error> {
|
auto optional_string(boost::json::object const &object, std::string_view field, std::string fallback) -> std::expected<std::string, modulus::error> {
|
||||||
auto found = object.find(field);
|
auto value = modulus::json_config::optional_string_field(object, field, std::move(fallback), field_options);
|
||||||
if (found == object.end()) {
|
if (value && value->empty() && object.contains(field)) {
|
||||||
return fallback;
|
|
||||||
}
|
|
||||||
if (!found->value().is_string()) {
|
|
||||||
return std::unexpected(invalid(std::string{field} + " must be a string"));
|
|
||||||
}
|
|
||||||
auto value = std::string{found->value().as_string()};
|
|
||||||
if (value.empty()) {
|
|
||||||
return std::unexpected(invalid(std::string{field} + " must not be empty"));
|
return std::unexpected(invalid(std::string{field} + " must not be empty"));
|
||||||
}
|
}
|
||||||
return value;
|
return value;
|
||||||
}
|
}
|
||||||
|
|
||||||
auto optional_bool(boost::json::object const &object, std::string_view field, std::bool_t fallback) -> std::expected<std::bool_t, modulus::error> {
|
|
||||||
auto found = object.find(field);
|
|
||||||
if (found == object.end()) {
|
|
||||||
return fallback;
|
|
||||||
}
|
|
||||||
if (!found->value().is_bool()) {
|
|
||||||
return std::unexpected(invalid(std::string{field} + " must be a boolean"));
|
|
||||||
}
|
|
||||||
return found->value().as_bool();
|
|
||||||
}
|
|
||||||
|
|
||||||
auto optional_uint32(boost::json::object const &object, std::string_view field, std::uint32_t fallback, std::uint32_t min_value, std::uint32_t max_value) -> std::expected<std::uint32_t, modulus::error> {
|
auto optional_uint32(boost::json::object const &object, std::string_view field, std::uint32_t fallback, std::uint32_t min_value, std::uint32_t max_value) -> std::expected<std::uint32_t, modulus::error> {
|
||||||
auto found = object.find(field);
|
auto value = modulus::json_config::optional_uint64_field(object, field, fallback, max_value, field_options);
|
||||||
if (found == object.end()) {
|
if (!value) {
|
||||||
return fallback;
|
return std::unexpected(value.error());
|
||||||
}
|
}
|
||||||
auto value = std::uint64_t{};
|
if (*value < min_value) {
|
||||||
if (found->value().is_uint64()) {
|
|
||||||
value = found->value().as_uint64();
|
|
||||||
} else if (found->value().is_int64()) {
|
|
||||||
auto signed_value = found->value().as_int64();
|
|
||||||
if (signed_value < 0) {
|
|
||||||
return std::unexpected(invalid(std::string{field} + " must not be negative"));
|
|
||||||
}
|
|
||||||
value = static_cast<std::uint64_t>(signed_value);
|
|
||||||
} else {
|
|
||||||
return std::unexpected(invalid(std::string{field} + " must be an integer"));
|
|
||||||
}
|
|
||||||
if (value < min_value || value > max_value) {
|
|
||||||
return std::unexpected(invalid(std::string{field} + " is out of range"));
|
return std::unexpected(invalid(std::string{field} + " is out of range"));
|
||||||
}
|
}
|
||||||
return static_cast<std::uint32_t>(value);
|
return static_cast<std::uint32_t>(*value);
|
||||||
}
|
}
|
||||||
|
|
||||||
auto optional_float(boost::json::object const &object, std::string_view field, std::float32_t fallback, std::float32_t min_value, std::float32_t max_value) -> std::expected<std::float32_t, modulus::error> {
|
auto optional_float(boost::json::object const &object, std::string_view field, std::float32_t fallback, std::float32_t min_value, std::float32_t max_value) -> std::expected<std::float32_t, modulus::error> {
|
||||||
auto found = object.find(field);
|
auto value = modulus::json_config::optional_float64_field(object, field, fallback, min_value, max_value, field_options);
|
||||||
if (found == object.end()) {
|
if (!value) {
|
||||||
return fallback;
|
return std::unexpected(value.error());
|
||||||
}
|
}
|
||||||
auto value = std::float64_t{};
|
return static_cast<std::float32_t>(*value);
|
||||||
if (found->value().is_double()) {
|
|
||||||
value = found->value().as_double();
|
|
||||||
} else if (found->value().is_uint64()) {
|
|
||||||
value = static_cast<std::float64_t>(found->value().as_uint64());
|
|
||||||
} else if (found->value().is_int64()) {
|
|
||||||
value = static_cast<std::float64_t>(found->value().as_int64());
|
|
||||||
} else {
|
|
||||||
return std::unexpected(invalid(std::string{field} + " must be a number"));
|
|
||||||
}
|
|
||||||
if (!std::isfinite(value) || value < min_value || value > max_value) {
|
|
||||||
return std::unexpected(invalid(std::string{field} + " is out of range"));
|
|
||||||
}
|
|
||||||
return static_cast<std::float32_t>(value);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
auto parse_runtime(boost::json::object const &root, vamp_zero::runtime_config fallback) -> std::expected<vamp_zero::runtime_config, modulus::error> {
|
auto parse_runtime(boost::json::object const &root, vamp_zero::runtime_config fallback) -> std::expected<vamp_zero::runtime_config, modulus::error> {
|
||||||
@@ -119,8 +72,8 @@ auto parse_runtime(boost::json::object const &root, vamp_zero::runtime_config fa
|
|||||||
if (*object == nullptr) {
|
if (*object == nullptr) {
|
||||||
return fallback;
|
return fallback;
|
||||||
}
|
}
|
||||||
auto headless = optional_bool(**object, "headless", fallback.headless);
|
auto headless = modulus::json_config::optional_bool_field(**object, "headless", fallback.headless, field_options);
|
||||||
auto exit_after_startup = optional_bool(**object, "exit_after_startup", fallback.exit_after_startup);
|
auto exit_after_startup = modulus::json_config::optional_bool_field(**object, "exit_after_startup", fallback.exit_after_startup, field_options);
|
||||||
auto assets_path = optional_string(**object, "assets_path", fallback.assets_path);
|
auto assets_path = optional_string(**object, "assets_path", fallback.assets_path);
|
||||||
auto heartbeat_path = optional_string(**object, "heartbeat_path", fallback.heartbeat_path);
|
auto heartbeat_path = optional_string(**object, "heartbeat_path", fallback.heartbeat_path);
|
||||||
if (!headless) {
|
if (!headless) {
|
||||||
@@ -218,21 +171,12 @@ auto parse_audio(boost::json::object const &root, vamp_zero::audio_config fallba
|
|||||||
if (*object == nullptr) {
|
if (*object == nullptr) {
|
||||||
return fallback;
|
return fallback;
|
||||||
}
|
}
|
||||||
auto enabled = optional_bool(**object, "enabled", fallback.enabled);
|
|
||||||
auto playback_enabled = optional_bool(**object, "playback_enabled", fallback.playback_enabled);
|
|
||||||
auto legacy_track = optional_bool(**object, "track_enabled", fallback.playback_enabled);
|
|
||||||
if (!legacy_track) {
|
|
||||||
return std::unexpected(legacy_track.error());
|
|
||||||
}
|
|
||||||
if ((*object)->if_contains("track_enabled")) {
|
if ((*object)->if_contains("track_enabled")) {
|
||||||
if ((*object)->if_contains("playback_enabled") && playback_enabled && *playback_enabled != *legacy_track) {
|
return std::unexpected(invalid("audio.track_enabled is no longer supported; use audio.playback_enabled"));
|
||||||
return std::unexpected(invalid("audio.playback_enabled contradicts legacy track_enabled"));
|
|
||||||
}
|
|
||||||
if (!(*object)->if_contains("playback_enabled")) {
|
|
||||||
playback_enabled = legacy_track;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
auto metronome_enabled = optional_bool(**object, "metronome_enabled", fallback.metronome_enabled);
|
auto enabled = modulus::json_config::optional_bool_field(**object, "enabled", fallback.enabled, field_options);
|
||||||
|
auto playback_enabled = modulus::json_config::optional_bool_field(**object, "playback_enabled", fallback.playback_enabled, field_options);
|
||||||
|
auto metronome_enabled = modulus::json_config::optional_bool_field(**object, "metronome_enabled", fallback.metronome_enabled, field_options);
|
||||||
auto metronome_bpm = optional_uint32(**object, "metronome_bpm", fallback.metronome_bpm, 1U, 400U);
|
auto metronome_bpm = optional_uint32(**object, "metronome_bpm", fallback.metronome_bpm, 1U, 400U);
|
||||||
auto metronome_min_bpm = optional_uint32(**object, "metronome_min_bpm", fallback.metronome_min_bpm, 1U, 400U);
|
auto metronome_min_bpm = optional_uint32(**object, "metronome_min_bpm", fallback.metronome_min_bpm, 1U, 400U);
|
||||||
auto metronome_max_bpm = optional_uint32(**object, "metronome_max_bpm", fallback.metronome_max_bpm, 1U, 400U);
|
auto metronome_max_bpm = optional_uint32(**object, "metronome_max_bpm", fallback.metronome_max_bpm, 1U, 400U);
|
||||||
@@ -293,7 +237,7 @@ auto parse_audio(boost::json::object const &root, vamp_zero::audio_config fallba
|
|||||||
}
|
}
|
||||||
|
|
||||||
auto vamp_zero::parse_config(boost::json::value const &value) -> std::expected<vamp_zero::config, modulus::error> {
|
auto vamp_zero::parse_config(boost::json::value const &value) -> std::expected<vamp_zero::config, modulus::error> {
|
||||||
auto root = require_object(value, "Product Vamp config root");
|
auto root = modulus::json_config::require_object(value, "Product Vamp config root", field_options);
|
||||||
if (!root) {
|
if (!root) {
|
||||||
return std::unexpected(root.error());
|
return std::unexpected(root.error());
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -196,13 +196,5 @@ auto vamp_zero::make_router(vamp_zero::control_state &state) -> ligo::service_ro
|
|||||||
auto updated = state.apply(vamp_protocol::page_command::SELECT);
|
auto updated = state.apply(vamp_protocol::page_command::SELECT);
|
||||||
co_return remote_response(updated);
|
co_return remote_response(updated);
|
||||||
});
|
});
|
||||||
router.add_route("POST", vamp_protocol::next_page_path, [&state](ligo::request_context, boost::json::value) -> boost::asio::awaitable<std::expected<boost::json::value, modulus::error>> {
|
|
||||||
auto updated = state.apply(vamp_protocol::page_command::NEXT);
|
|
||||||
co_return vamp_zero::portal_status_json(state.config(), updated);
|
|
||||||
});
|
|
||||||
router.add_route("POST", vamp_protocol::previous_page_path, [&state](ligo::request_context, boost::json::value) -> boost::asio::awaitable<std::expected<boost::json::value, modulus::error>> {
|
|
||||||
auto updated = state.apply(vamp_protocol::page_command::PREVIOUS);
|
|
||||||
co_return vamp_zero::portal_status_json(state.config(), updated);
|
|
||||||
});
|
|
||||||
return router;
|
return router;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import hashlib
|
import json
|
||||||
import importlib.util
|
import importlib.util
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import sys
|
import sys
|
||||||
@@ -7,15 +7,15 @@ import unittest
|
|||||||
from unittest.mock import Mock, patch
|
from unittest.mock import Mock, patch
|
||||||
|
|
||||||
class BuildBootstrapTests(unittest.TestCase):
|
class BuildBootstrapTests(unittest.TestCase):
|
||||||
def test_requirements_fingerprint_invalidates_cached_build_image(self):
|
def test_build_recipe_is_declared_and_editor_commands_do_not_bootstrap(self):
|
||||||
root = Path(__file__).resolve().parents[1]
|
root = Path(__file__).resolve().parents[1]
|
||||||
spec = importlib.util.spec_from_file_location('vamp_bootstrap', root / 'modulus.py')
|
spec = importlib.util.spec_from_file_location('vamp_bootstrap', root / 'modulus.py')
|
||||||
bootstrap = importlib.util.module_from_spec(spec)
|
bootstrap = importlib.util.module_from_spec(spec)
|
||||||
spec.loader.exec_module(bootstrap)
|
spec.loader.exec_module(bootstrap)
|
||||||
digest = hashlib.sha256((root / 'generator/requirements.txt').read_bytes()).hexdigest()
|
docker = json.loads((root / 'modulus.json').read_text())['tooling']['docker']
|
||||||
self.assertEqual(bootstrap.build_requirements_digest(), digest)
|
self.assertEqual(docker['socket_commands'], ['build', 'package'])
|
||||||
self.assertIn(f'build-requirements="{digest}"', bootstrap.product_build_dockerfile())
|
self.assertIn('generator/requirements.txt', docker['inputs'])
|
||||||
self.assertIn('COPY generator/requirements.txt /tmp/vamp-build-requirements.txt', bootstrap.product_build_dockerfile())
|
self.assertIn('COPY generator/requirements.txt /tmp/vamp-build-requirements.txt', (root / docker['dockerfile']).read_text())
|
||||||
package = types.ModuleType('modulus')
|
package = types.ModuleType('modulus')
|
||||||
cli = types.ModuleType('modulus.cli')
|
cli = types.ModuleType('modulus.cli')
|
||||||
package.cli = cli
|
package.cli = cli
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
import argparse
|
import argparse
|
||||||
|
import contextlib
|
||||||
import copy
|
import copy
|
||||||
import hashlib
|
import hashlib
|
||||||
|
import io
|
||||||
import json
|
import json
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import tempfile
|
import tempfile
|
||||||
@@ -9,14 +11,16 @@ from unittest.mock import patch
|
|||||||
|
|
||||||
from generator.__main__ import generate_assets
|
from generator.__main__ import generate_assets
|
||||||
from generator.engraving import EngravingError, Fonts, Style, engrave_tree, layout_score
|
from generator.engraving import EngravingError, Fonts, Style, engrave_tree, layout_score
|
||||||
from generator.engraving.cache import renderer_fingerprint, restore_pages, score_fingerprint
|
from generator.engraving.cache import renderer_fingerprint, renderer_namespace, restore_pages, score_fingerprint
|
||||||
|
from generator.score import validate_generated_tree
|
||||||
from tests.test_musicxml import parse, xml_note, xml_score
|
from tests.test_musicxml import parse, xml_note, xml_score
|
||||||
|
|
||||||
class EngravingCacheTests(unittest.TestCase):
|
class EngravingCacheTests(unittest.TestCase):
|
||||||
def test_ci_snapshot_tracks_all_generator_inputs_and_keeps_fallback(self):
|
def test_ci_snapshot_tracks_all_generator_inputs_and_keeps_fallback(self):
|
||||||
workflow = Path('.gitea/workflows/ci.yml').read_text()
|
workflow = Path('.gitea/workflows/ci.yml').read_text()
|
||||||
self.assertIn("hashFiles('generator/**/*.py', 'generator/*.py', 'data/**/*.xml', 'data/*.xml')", workflow)
|
self.assertIn("hashFiles('generator/**/*.py', 'generator/*.py', 'data/**/*.xml', 'data/*.xml')", workflow)
|
||||||
self.assertIn('restore-keys: product-vamp-pages-v1-${{ runner.os }}-', workflow)
|
self.assertIn('python3 -m generator --renderer-fingerprint', workflow)
|
||||||
|
self.assertIn('restore-keys: product-vamp-pages-v2-${{ steps.renderer.outputs.fingerprint }}-', workflow)
|
||||||
self.assertIn("if: steps.engraving-cache.outputs.cache-hit != 'true'", workflow)
|
self.assertIn("if: steps.engraving-cache.outputs.cache-hit != 'true'", workflow)
|
||||||
self.assertIn('key: ${{ steps.engraving-cache.outputs.cache-primary-key }}', workflow)
|
self.assertIn('key: ${{ steps.engraving-cache.outputs.cache-primary-key }}', workflow)
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
@@ -36,7 +40,7 @@ class EngravingCacheTests(unittest.TestCase):
|
|||||||
root = self.root / 'seed'
|
root = self.root / 'seed'
|
||||||
self.write_scores(root, [self.document])
|
self.write_scores(root, [self.document])
|
||||||
engrave_tree(root, self.fonts, cache_dir=self.cache)
|
engrave_tree(root, self.fonts, cache_dir=self.cache)
|
||||||
return next(self.cache.iterdir())
|
return next(self.cache.rglob('pages.json')).parent
|
||||||
def test_cache_hit_preserves_bundle_bytes_across_output_directories(self):
|
def test_cache_hit_preserves_bundle_bytes_across_output_directories(self):
|
||||||
cold, warm = self.root / 'cold', self.root / 'warm'
|
cold, warm = self.root / 'cold', self.root / 'warm'
|
||||||
self.write_scores(cold, [self.document])
|
self.write_scores(cold, [self.document])
|
||||||
@@ -65,7 +69,7 @@ class EngravingCacheTests(unittest.TestCase):
|
|||||||
with patch('generator.engraving.layout_score', wraps=layout_score) as render:
|
with patch('generator.engraving.layout_score', wraps=layout_score) as render:
|
||||||
engrave_tree(root, self.fonts, cache_dir=self.cache)
|
engrave_tree(root, self.fonts, cache_dir=self.cache)
|
||||||
render.assert_called_once()
|
render.assert_called_once()
|
||||||
self.assertEqual(len(list(self.cache.iterdir())), 2)
|
self.assertEqual(len(list(self.cache.rglob('pages.json'))), 2)
|
||||||
def test_renderer_fingerprint_tracks_fonts_style_code_and_dependencies(self):
|
def test_renderer_fingerprint_tracks_fonts_style_code_and_dependencies(self):
|
||||||
fonts, style = Fonts(self.fonts), Style()
|
fonts, style = Fonts(self.fonts), Style()
|
||||||
expected = renderer_fingerprint(fonts, style)
|
expected = renderer_fingerprint(fonts, style)
|
||||||
@@ -75,6 +79,8 @@ class EngravingCacheTests(unittest.TestCase):
|
|||||||
self.assertNotEqual(expected, renderer_fingerprint(fonts, style))
|
self.assertNotEqual(expected, renderer_fingerprint(fonts, style))
|
||||||
with patch('generator.engraving.cache.importlib.metadata.version', return_value='new-version'):
|
with patch('generator.engraving.cache.importlib.metadata.version', return_value='new-version'):
|
||||||
self.assertNotEqual(expected, renderer_fingerprint(fonts, style))
|
self.assertNotEqual(expected, renderer_fingerprint(fonts, style))
|
||||||
|
with patch('generator.engraving.cache.platform.python_version', return_value='0.0.0'):
|
||||||
|
self.assertNotEqual(expected, renderer_fingerprint(fonts, style))
|
||||||
read_bytes = Path.read_bytes
|
read_bytes = Path.read_bytes
|
||||||
with patch.object(Path, 'read_bytes', lambda path: read_bytes(path) + (b'\n' if path.name == 'raster.py' else b'')):
|
with patch.object(Path, 'read_bytes', lambda path: read_bytes(path) + (b'\n' if path.name == 'raster.py' else b'')):
|
||||||
self.assertNotEqual(expected, renderer_fingerprint(fonts, style))
|
self.assertNotEqual(expected, renderer_fingerprint(fonts, style))
|
||||||
@@ -129,8 +135,46 @@ class EngravingCacheTests(unittest.TestCase):
|
|||||||
self.write_scores(root, documents)
|
self.write_scores(root, documents)
|
||||||
engrave_tree(root, self.fonts, cache_dir=self.cache, jobs=2)
|
engrave_tree(root, self.fonts, cache_dir=self.cache, jobs=2)
|
||||||
self.assertEqual(self.files(serial), self.files(root))
|
self.assertEqual(self.files(serial), self.files(root))
|
||||||
self.assertEqual(len(list(self.cache.iterdir())), 2)
|
self.assertEqual(len(list(self.cache.rglob('pages.json'))), 2)
|
||||||
self.assertFalse(list(self.cache.glob('.engraving-*')))
|
self.assertFalse(list(self.cache.rglob('.engraving-*')))
|
||||||
|
def test_environment_change_gets_new_namespace_and_clear_progress(self):
|
||||||
|
old = self.seed_cache()
|
||||||
|
root = self.root / 'new-environment'
|
||||||
|
self.write_scores(root, [self.document])
|
||||||
|
progress = io.StringIO()
|
||||||
|
with patch('generator.engraving.cache.platform.python_version', return_value='0.0.0'), contextlib.redirect_stdout(progress):
|
||||||
|
engrave_tree(root, self.fonts, cache_dir=self.cache, progress=True)
|
||||||
|
self.assertIn('cold namespace', progress.getvalue())
|
||||||
|
self.assertIn('0 reused, 1 missing, 0 rejected', progress.getvalue())
|
||||||
|
self.assertTrue((old / 'pages.json').is_file())
|
||||||
|
self.assertEqual(len(list((self.cache / 'v2').iterdir())), 2)
|
||||||
|
self.write_scores(root, [self.document])
|
||||||
|
progress = io.StringIO()
|
||||||
|
with patch('generator.engraving.cache.platform.python_version', return_value='0.0.0'), contextlib.redirect_stdout(progress), patch('generator.engraving.layout_score', side_effect=AssertionError('cache miss')):
|
||||||
|
engrave_tree(root, self.fonts, cache_dir=self.cache, progress=True)
|
||||||
|
self.assertIn('1 reused, 0 missing, 0 rejected', progress.getvalue())
|
||||||
|
def test_namespace_symlink_is_rejected(self):
|
||||||
|
self.cache.mkdir()
|
||||||
|
(self.cache / 'v2').symlink_to(self.root, target_is_directory=True)
|
||||||
|
with self.assertRaisesRegex(ValueError, 'must not be symlinks'):
|
||||||
|
renderer_namespace(self.cache, 'renderer')
|
||||||
|
def test_only_changed_score_is_rendered_and_validation_reads_json_once(self):
|
||||||
|
root = self.root / 'scores'
|
||||||
|
self.write_scores(root, [self.document, self.document])
|
||||||
|
engrave_tree(root, self.fonts, cache_dir=self.cache)
|
||||||
|
changed = parse(xml_score(xml_note('D')))
|
||||||
|
self.write_scores(root, [self.document, changed])
|
||||||
|
with patch('generator.engraving.layout_score', wraps=layout_score) as render:
|
||||||
|
engrave_tree(root, self.fonts, cache_dir=self.cache)
|
||||||
|
render.assert_called_once()
|
||||||
|
read_text = Path.read_text
|
||||||
|
reads = []
|
||||||
|
def read(path, *args, **kwargs):
|
||||||
|
reads.append(path)
|
||||||
|
return read_text(path, *args, **kwargs)
|
||||||
|
with patch.object(Path, 'read_text', read):
|
||||||
|
validate_generated_tree(root)
|
||||||
|
self.assertCountEqual(reads, sorted(root.glob('*.json')))
|
||||||
def test_invalid_worker_count_and_overlapping_cache_are_rejected(self):
|
def test_invalid_worker_count_and_overlapping_cache_are_rejected(self):
|
||||||
root = self.root / 'assets'
|
root = self.root / 'assets'
|
||||||
self.write_scores(root, [self.document])
|
self.write_scores(root, [self.document])
|
||||||
|
|||||||
@@ -120,10 +120,10 @@ BOOST_AUTO_TEST_CASE(playback_worker_discards_canceled_preparation_and_reports_e
|
|||||||
BOOST_CHECK(!audio.tick(state, loaded, VAMP_TEST_PLAYBACK_ASSETS_DIR));
|
BOOST_CHECK(!audio.tick(state, loaded, VAMP_TEST_PLAYBACK_ASSETS_DIR));
|
||||||
}
|
}
|
||||||
|
|
||||||
BOOST_AUTO_TEST_CASE(playback_legacy_config_and_invalid_assets) {
|
BOOST_AUTO_TEST_CASE(playback_rejects_legacy_config_and_invalid_assets) {
|
||||||
auto legacy = vamp_zero::parse_config(boost::json::object{{"audio", {{"track_enabled", true}}}});
|
auto legacy = vamp_zero::parse_config(boost::json::object{{"audio", {{"track_enabled", true}}}});
|
||||||
BOOST_REQUIRE(legacy);
|
BOOST_REQUIRE(!legacy);
|
||||||
BOOST_TEST(legacy->audio.playback_enabled);
|
BOOST_TEST(std::string{legacy.error().message()}.find("use audio.playback_enabled") != std::string::npos);
|
||||||
BOOST_CHECK(!vamp_zero::parse_config(boost::json::object{{"audio", {{"track_enabled", true}, {"playback_enabled", false}}}}));
|
BOOST_CHECK(!vamp_zero::parse_config(boost::json::object{{"audio", {{"track_enabled", true}, {"playback_enabled", false}}}}));
|
||||||
auto score = playback_score();
|
auto score = playback_score();
|
||||||
BOOST_CHECK(!vamp_zero::prepare_playback(score, "/missing-vamp-assets", 120U));
|
BOOST_CHECK(!vamp_zero::prepare_playback(score, "/missing-vamp-assets", 120U));
|
||||||
|
|||||||
Reference in New Issue
Block a user