khodo/product-vamp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Cleanup
ci / checks (push) Failing after 1h6m47s
Details

Browse Source
This commit is contained in:
khodowany
2026-09-24 19:44:05 -05:00
parent 1be98e14aa
commit 3f47c0c5f3
19 changed files with 309 additions and 404 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modulus.py
+58 -252
View File
@@ -1,18 +1,15 @@
#!/usr/bin/env python3
# Canonical consumer bootstrap. Copy this file unchanged to a project's modulus.py.
import json
import hashlib
import os
import pathlib
import shlex
import subprocess
import sys
import time
PACKAGE_REF = "modulus/latest"
CONAN_VERSION = "2.26.2"
FETCH_IMAGE = "modulus-fetch:dev"
DEFAULT_BUILD_BASE_IMAGE = "debian:trixie-slim"
PICO_SDK_VERSION = "2.3.0"
COMMANDS = (
"configure",
"build",
@@ -29,6 +26,7 @@ RUNTIME_DIR = COMMON_DIR / "modulus"
CONAN_HOME_DIR = COMMON_DIR / "conan_cache"
DEFAULT_REMOTE_NAME = "khodo"
DEFAULT_REMOTE_URL = "https://git.thyme-labs.com/api/packages/khodo/conan"
DEFAULT_DOCKER_MARKER = "/opt/modulus-build-image"
def repo_root():
root = pathlib.Path(__file__).resolve().parent
@@ -63,7 +61,6 @@ def runtime_ready(root):
(root / TOOLS_DIR / "configure.py").is_file()
and (root / TOOLS_DIR / "lsp.py").is_file()
and (root / TOOLS_DIR / "clangd.py").is_file()
and (root / TOOLS_DIR / "run.py").is_file()
and (root / RUNTIME_DIR / "modulus" / "cli.py").is_file()
and (root / RUNTIME_DIR / "modulus" / "export.py").is_file()
and (root / TOOLS_DIR / "format.py").is_file()
@@ -71,18 +68,19 @@ def runtime_ready(root):
and (root / RUNTIME_DIR / "modulus" / "format_layout.py").is_file()
)
def optional_credentials():
user = os.environ.get("CONAN_USER")
token = os.environ.get("CONAN_TOKEN")
if bool(user) != bool(token):
raise SystemExit("CONAN_USER and CONAN_TOKEN must either both be set or both be unset")
return user, token
def env_flag(name):
return os.environ.get(name, "").lower() in {"1", "true", "yes", "on"}
def docker_marker(root):
config = read_json(root / "modulus.json")
marker = config.get(
"tooling",
{},
).get("docker", {}).get("marker", DEFAULT_DOCKER_MARKER)
return pathlib.Path(str(marker))
def direct_mode(root):
return env_flag("MODULUS_DIRECT") or project_docker_marker(root).exists()
return env_flag("MODULUS_DIRECT") or docker_marker(root).exists()
def run(command, input_text=None, capture=False, display_command=None, env=None, check=True):
visible = display_command if display_command is not None else command
@@ -104,20 +102,8 @@ def run(command, input_text=None, capture=False, display_command=None, env=None,
raise SystemExit(completed.returncode)
return completed
def run_with_retries(command, attempts=3, input_text=None, display_command=None, env=None):
completed = None
for attempt in range(1, attempts + 1):
completed = run(command, input_text=input_text, display_command=display_command, env=env, check=False)
if completed.returncode == 0:
return completed
if attempt != attempts:
delay = attempt * 20
print(f"command failed with exit code {completed.returncode}; retrying in {delay}s")
time.sleep(delay)
raise SystemExit(completed.returncode if completed is not None else 1)
def fetch_dockerfile():
return f"""FROM {DEFAULT_BUILD_BASE_IMAGE}
return f"""FROM debian:bookworm-slim
ENV DEBIAN_FRONTEND=noninteractive
@@ -137,112 +123,23 @@ RUN python -m pip install --no-cache-dir conan=={CONAN_VERSION}
WORKDIR /workspace
"""
def build_requirements_digest():
return hashlib.sha256((repo_root() / "generator/requirements.txt").read_bytes()).hexdigest()
def product_build_dockerfile():
return f"""FROM {DEFAULT_BUILD_BASE_IMAGE}
ENV DEBIAN_FRONTEND=noninteractive
RUN dpkg --add-architecture arm64
RUN apt-get update \\
&& apt-get install -y --no-install-recommends \\
bash \\
build-essential \\
ca-certificates \\
clangd-19 \\
cmake \\
curl \\
docker-buildx \\
docker-cli \\
docker.io \\
g++-12 \\
g++-12-aarch64-linux-gnu \\
gcc-12 \\
gcc-12-aarch64-linux-gnu \\
gcc-arm-none-eabi \\
git \\
libdrm-dev \\
libdrm-dev:arm64 \\
libegl-dev \\
libegl-dev:arm64 \\
libfribidi0 \\
libgbm-dev \\
libgbm-dev:arm64 \\
libgles-dev \\
libgles-dev:arm64 \\
libglvnd-dev \\
libglvnd-dev:arm64 \\
libnewlib-arm-none-eabi \\
libstdc++-arm-none-eabi-newlib \\
ninja-build \\
pkg-config \\
python3 \\
python3-venv \\
&& ln -sf /usr/bin/clangd-19 /usr/local/bin/clangd \\
&& rm -rf /var/lib/apt/lists/*
ENV CC=/usr/bin/gcc-12
ENV CXX=/usr/bin/g++-12
ENV VAMP_PICO_SDK_VERSION={PICO_SDK_VERSION}
ENV PICO_SDK_PATH=/opt/pico-sdk
RUN git clone \\
--depth 1 \\
--branch "$VAMP_PICO_SDK_VERSION" \\
--recurse-submodules \\
--shallow-submodules \\
https://github.com/raspberrypi/pico-sdk.git \\
"$PICO_SDK_PATH"
RUN python3 -m venv /opt/modulus-python
ENV PATH=/opt/modulus-python/bin:$PATH
COPY generator/requirements.txt /tmp/vamp-build-requirements.txt
ARG MODULUS_TOOLS=build/build_tools
COPY ${{MODULUS_TOOLS}}/docker/format-requirements.txt /tmp/modulus-format-requirements.txt
RUN python -m pip install --no-cache-dir \\
conan=={CONAN_VERSION} -r /tmp/vamp-build-requirements.txt -r /tmp/modulus-format-requirements.txt
LABEL org.khodo.product-vamp.build-requirements="{build_requirements_digest()}"
WORKDIR /workspace
RUN mkdir -p /opt && touch /opt/modulus-build-image
"""
def sync_product_build_dockerfile(root):
dockerfile = root / TOOLS_DIR / "docker" / "Dockerfile"
if not dockerfile.parent.is_dir():
def ensure_fetch_image(offline=False):
inspected = subprocess.run(
[
"docker",
"image",
"inspect",
FETCH_IMAGE,
],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
check=False,
)
if inspected.returncode == 0 and (offline or not docker_platform_args()):
return
desired = product_build_dockerfile()
try:
current = dockerfile.read_text(encoding="utf-8")
except OSError:
current = None
if current != desired:
dockerfile.write_text(desired, encoding="utf-8")
def ensure_fetch_image():
if not docker_platform_args():
inspected = subprocess.run(
[
"docker",
"image",
"inspect",
FETCH_IMAGE,
],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
check=False,
)
if inspected.returncode == 0:
return
run_with_retries(
if offline:
raise SystemExit('offline bootstrap requires the existing Modulus fetch image or MODULUS_DIRECT=1 with Conan installed')
run(
[
"docker",
"build",
@@ -257,6 +154,10 @@ def ensure_fetch_image():
def in_container():
return pathlib.Path("/.dockerenv").exists()
def docker_platform_args():
platform = os.environ.get("MODULUS_DOCKER_PLATFORM", "").strip()
return ["--platform", platform] if platform else []
def docker_workspace_args(root):
if in_container():
container = os.environ.get("HOSTNAME", "").strip()
@@ -271,43 +172,26 @@ def docker_workspace_args(root):
f"{root}:{root}",
]
def docker_socket_args():
socket = pathlib.Path("/var/run/docker.sock")
if not socket.is_socket():
return []
args = [
"--volume",
"/var/run/docker.sock:/var/run/docker.sock",
]
try:
args.extend(["--group-add", str(socket.stat().st_gid)])
except OSError:
pass
return args
def docker_platform_args():
platform = os.environ.get("MODULUS_DOCKER_PLATFORM", "").strip()
if not platform:
return []
return [
"--platform",
platform,
]
def fetch_env(root):
env = os.environ.copy()
for name in list(env):
if name in {"CONAN_USER", "CONAN_TOKEN"} or name.startswith(("CONAN_LOGIN_USERNAME", "CONAN_PASSWORD")):
del env[name]
env["CONAN_HOME"] = str(root / CONAN_HOME_DIR)
env["CONAN_NON_INTERACTIVE"] = "1"
return env
def run_fetch(root, command, capture=False, display_command=None, check=True):
def run_fetch(root, command, capture=False, display_command=None, check=True, offline=False):
if direct_mode(root):
return run(command, capture=capture, display_command=display_command, env=fetch_env(root), check=check)
ensure_fetch_image()
ensure_fetch_image(offline=offline)
docker_command = [
"docker",
"run",
"--rm",
"--interactive",
*(["--pull=never", "--network=none"] if offline else []),
*docker_platform_args(),
"--user",
f"{os.getuid()}:{os.getgid()}",
"--env",
@@ -315,13 +199,8 @@ def run_fetch(root, command, capture=False, display_command=None, check=True):
"--env",
"CONAN_HOME",
"--env",
"CONAN_USER",
"--env",
"CONAN_TOKEN",
"--env",
"DOCKER_HOST",
"CONAN_NON_INTERACTIVE",
*docker_workspace_args(root),
*docker_socket_args(),
"--workdir",
str(root),
FETCH_IMAGE,
@@ -330,7 +209,6 @@ def run_fetch(root, command, capture=False, display_command=None, check=True):
return run(docker_command, capture=capture, display_command=display_command, env=fetch_env(root), check=check)
def configure_remote(root, remote_name, remote_url):
user, token = optional_credentials()
run_fetch(
root,
[
@@ -342,29 +220,7 @@ def configure_remote(root, remote_name, remote_url):
remote_url,
],
)
if not user and not token:
return
run_fetch(
root,
[
"conan",
"remote",
"login",
remote_name,
user,
"-p",
token,
],
display_command=[
"conan",
"remote",
"login",
remote_name,
user,
"-p",
"<hidden>",
],
)
run_fetch(root, ["conan", "remote", "logout", remote_name])
def package_refs(data):
refs = []
@@ -401,46 +257,26 @@ def package_refs(data):
refs.append(f"{PACKAGE_REF}#{recipe_revision}:{package_id}")
return sorted(refs)
def remove_cached_modulus(root):
run_fetch(
root,
[
"conan",
"remove",
PACKAGE_REF + ":*",
"-c",
],
check=False,
)
def download_modulus(root, remote_name):
remove_cached_modulus(root)
run_fetch(
root,
[
"conan",
"download",
PACKAGE_REF + ":*",
"-r",
remote_name,
],
)
def find_modulus(root, remote_name=None):
completed = run_fetch(
root,
[
"conan",
"list",
PACKAGE_REF + ":*",
"-c",
PACKAGE_REF + "#latest:*#latest",
*(["-r", remote_name] if remote_name else ["-c"]),
"-f",
"json",
],
capture=True,
offline=remote_name is None,
)
refs = package_refs(json.loads(completed.stdout))
if not refs:
raise SystemExit(f"no downloaded {PACKAGE_REF} package found in the Conan cache")
raise SystemExit(f"no {PACKAGE_REF} package available in {remote_name or 'the local Conan cache'}")
package_ref = refs[-1]
if remote_name:
run_fetch(root, ["conan", "download", package_ref, "-r", remote_name])
completed = run_fetch(
root,
[
@@ -450,40 +286,25 @@ def download_modulus(root, remote_name):
package_ref,
],
capture=True,
offline=remote_name is None,
)
package_root = pathlib.Path(completed.stdout.strip())
if not (package_root / "modulus" / "export.py").is_file() or not (package_root / "tools").is_dir():
raise SystemExit(f"downloaded {PACKAGE_REF} package does not contain Modulus export files: {package_root}")
return package_root
def materialize_runtime(root):
def materialize_runtime(root, no_remote=False):
remote_name, remote_url = remote_config(root)
configure_remote(root, remote_name, remote_url)
package_root = download_modulus(root, remote_name)
if not no_remote:
configure_remote(root, remote_name, remote_url)
package_root = find_modulus(root, None if no_remote else remote_name)
sys.path.insert(0, str(package_root))
import modulus.export
modulus.export.sync_project(root, package_root)
def ensure_runtime(root):
if not runtime_ready(root):
materialize_runtime(root)
sync_product_build_dockerfile(root)
def project_docker_marker(root):
config = read_json(root / "modulus.json")
tooling = config.get("tooling", {})
if not isinstance(tooling, dict):
tooling = {}
docker = tooling.get("docker", {})
if not isinstance(docker, dict):
docker = {}
return pathlib.Path(str(docker.get("marker") or "/opt/modulus-build-image"))
def run_configure(argv):
return delegate("configure", argv)
def run_package(argv):
return delegate("package", argv)
def ensure_runtime(root, update=False, no_remote=False):
if update or not runtime_ready(root):
materialize_runtime(root, no_remote=no_remote)
def delegate(command, argv):
root = repo_root()
@@ -492,19 +313,9 @@ def delegate(command, argv):
print(f"Modulus {command} tooling is missing; run ./modulus.py configure first", file=sys.stderr)
return 1
else:
ensure_runtime(root)
ensure_runtime(root, update=command == "configure" and "--update" in argv, no_remote="--no-remote" in argv)
sys.path.insert(0, str(root / RUNTIME_DIR))
import modulus.cli
original_docker_run = modulus.cli.docker_run
def docker_run_with_host_socket(context, command, docker_args=None, env_overrides=None):
merged_docker_args = [
*docker_socket_args(),
*(docker_args or []),
]
return original_docker_run(context, command, docker_args=merged_docker_args, env_overrides=env_overrides)
modulus.cli.docker_run = docker_run_with_host_socket
previous_argv = sys.argv
previous_cwd = pathlib.Path.cwd()
sys.argv = [
@@ -515,7 +326,6 @@ def delegate(command, argv):
try:
return modulus.cli.main(command)
finally:
modulus.cli.docker_run = original_docker_run
os.chdir(previous_cwd)
sys.argv = previous_argv
@@ -526,10 +336,6 @@ def main(argv):
command = argv[0]
if command not in COMMANDS:
raise SystemExit(f"unknown Modulus command: {command}")
if command == "configure":
return run_configure(argv[1:])
if command == "package":
return run_package(argv[1:])
return delegate(command, argv[1:])
if __name__ == "__main__":