From 0c9ba1bff48db0ac049a7ed653b6e9d9638854b4 Mon Sep 17 00:00:00 2001 From: khodowany Date: Thu, 24 Sep 2026 19:40:03 -0500 Subject: [PATCH] Cleanup --- .gitea/workflows/ci.yml | 4 +- README.md | 6 +- conanfile.py | 5 +- include/modulus/json_config.hpp | 1 + modulus/bootstrap.py | 342 ++++++++++++++++++++++++++++++++ modulus/clangd.py | 5 +- modulus/cli.py | 72 ++++--- modulus/config.py | 12 ++ modulus/export.py | 29 ++- schemas/modulus.schema.json | 12 ++ src/json_config.cpp | 18 ++ tests/test.cpp | 20 ++ tests/test_bootstrap.py | 125 ++++++++++++ tests/test_clangd.py | 1 + tests/test_cli.py | 29 +-- 15 files changed, 628 insertions(+), 53 deletions(-) create mode 100755 modulus/bootstrap.py create mode 100644 tests/test_bootstrap.py diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 46dd297..2a3f56e 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -47,7 +47,7 @@ jobs: key: conan-${{ runner.os }}-${{ hashFiles('conanfile.py', 'modulus.json', 'tools/conan/profiles/*') }} restore-keys: conan-${{ runner.os }}- - name: Configure native release - run: ./tools/configure.py --target x86_64_gcc12_release + run: ./tools/configure.py --target x86_64_gcc12_release --update - name: Test tooling run: python3 -m unittest discover -s tests -p 'test_*.py' - name: Check code style @@ -65,4 +65,4 @@ jobs: CONAN_TOKEN: ${{ secrets.CONAN_TOKEN }} MODULUS_CONAN_UPLOAD: "1" MODULUS_ALLOW_PACKAGE_REPLACE: "1" - run: ./tools/package.py --all --replace + run: ./tools/package.py --all --replace --update diff --git a/README.md b/README.md index 33a68d8..5283db1 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,9 @@ MODULUS_DIRECT=1 ./tools/configure.py --target x86_64_gcc12_release Outside that environment, downstream repos use `python3 modulus.py ` to fetch the public `modulus/latest` package from the khodo Conan registry and export the tool wrappers into `build/build_tools/`. -`build` automatically configures a target when CMake metadata is missing. Run `configure` explicitly when you want to force a reconfigure or refresh dependencies. +`build` automatically configures a target when CMake metadata is missing. `configure --update` refreshes dependencies and, in consumer repos, the exported Python tooling. Failed downloads/exports leave the previous tooling in place. `--no-remote` uses only the project-local cache, including initial bootstrap; it never downloads packages. Each consumer's tracked `modulus.py` is copied unchanged from `modulus/bootstrap.py`. Update that canonical source and synchronize consumer copies together. + +Projects can select a tracked build recipe using `tooling.docker.dockerfile` and list additional build-context inputs in `tooling.docker.inputs`. Those files, the Dockerfile, and the shared formatting requirements determine the image fingerprint used by builds and clangd. Paths are project-relative. `tooling.docker.socket_commands` optionally permits host Docker socket access for `build` and/or `package`; the default is no socket access. Editor, configure, and test commands never receive the socket. Refresh clangd compile databases with: @@ -129,7 +131,7 @@ Generic application artifacts default to: https://git.thyme-labs.com/api/packages/khodo/generic ``` -Configure/build/test do not require package credentials for public packages. +Configure/build/test do not require package credentials for public packages. Read operations log out only the configured public remote in the project-local Conan home and strip credential environment variables, including native Conan username/password variables. Authentication failures are noninteractive errors, not password prompts. Package contents and global Conan configuration are untouched. Credentials are validated only for publishing, redacted in command logs, and the remote is logged out in a `finally` cleanup after every upload attempt so CI caches do not retain the session. ## License diff --git a/conanfile.py b/conanfile.py index e43eedf..a4763c6 100644 --- a/conanfile.py +++ b/conanfile.py @@ -63,8 +63,11 @@ class modulus_recipe(conan.ConanFile): "glfw/*:with_wayland": False, "glfw/*:with_x11": False, } - exports = "modulus/*.py", "modulus.json" + exports = "modulus/*.py", "modulus.json", "!build", "!.git", "!test_package/build" exports_sources = ( + "!build", + "!.git", + "!test_package/build", "CMakeLists.txt", "cmake/*.cmake", "cmake/*.in", diff --git a/include/modulus/json_config.hpp b/include/modulus/json_config.hpp index 7053ce5..79b15aa 100644 --- a/include/modulus/json_config.hpp +++ b/include/modulus/json_config.hpp @@ -39,6 +39,7 @@ auto required_string_field(boost::json::object const &object, std::string_view n auto optional_uint64_field(boost::json::object const &object, std::string_view name, std::uint64_t fallback, std::uint64_t max_value, modulus::json_config::options config = {}) -> std::expected; auto positive_uint64_field(boost::json::object const &object, std::string_view name, std::uint64_t fallback, std::uint64_t max_value, modulus::json_config::options config = {}) -> std::expected; auto optional_int32_field(boost::json::object const &object, std::string_view name, std::int32_t fallback, std::int32_t min_value, std::int32_t max_value, modulus::json_config::options config = {}) -> std::expected; +auto optional_float64_field(boost::json::object const &object, std::string_view name, std::float64_t fallback, std::float64_t min_value, std::float64_t max_value, modulus::json_config::options config = {}) -> std::expected; auto positive_milliseconds_field(boost::json::object const &object, std::string_view name, std::chrono::milliseconds fallback, std::uint64_t max_value = static_cast(std::numeric_limits::max()), modulus::json_config::options config = {}) -> std::expected; template diff --git a/modulus/bootstrap.py b/modulus/bootstrap.py new file mode 100755 index 0000000..856cec7 --- /dev/null +++ b/modulus/bootstrap.py @@ -0,0 +1,342 @@ +#!/usr/bin/env python3 +# Canonical consumer bootstrap. Copy this file unchanged to a project's modulus.py. +import json +import os +import pathlib +import shlex +import subprocess +import sys + +PACKAGE_REF = "modulus/latest" +CONAN_VERSION = "2.26.2" +FETCH_IMAGE = "modulus-fetch:dev" +COMMANDS = ( + "configure", + "build", + "test", + "package", + "lsp", + "clangd", + "format", + "run", +) +COMMON_DIR = pathlib.Path("build") / "build_common" +TOOLS_DIR = pathlib.Path("build") / "build_tools" +RUNTIME_DIR = COMMON_DIR / "modulus" +CONAN_HOME_DIR = COMMON_DIR / "conan_cache" +DEFAULT_REMOTE_NAME = "khodo" +DEFAULT_REMOTE_URL = "https://git.thyme-labs.com/api/packages/khodo/conan" +DEFAULT_DOCKER_MARKER = "/opt/modulus-build-image" + +def repo_root(): + root = pathlib.Path(__file__).resolve().parent + if not (root / "modulus.json").is_file(): + raise SystemExit("modulus.py must be run from a Modulus-managed repository root") + return root + +def read_json(path): + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise SystemExit(f"failed to read {path}: {error}") from error + if not isinstance(data, dict): + raise SystemExit(f"{path} must contain a JSON object") + return data + +def remote_config(root): + config = read_json(root / "modulus.json") + remote = config.get( + "tooling", + {}, + ).get("remote", {}) + if not isinstance(remote, dict): + remote = {} + return ( + str(remote.get("name") or DEFAULT_REMOTE_NAME), + str(remote.get("url") or DEFAULT_REMOTE_URL), + ) + +def runtime_ready(root): + return ( + (root / TOOLS_DIR / "configure.py").is_file() + and (root / TOOLS_DIR / "lsp.py").is_file() + and (root / TOOLS_DIR / "clangd.py").is_file() + and (root / RUNTIME_DIR / "modulus" / "cli.py").is_file() + and (root / RUNTIME_DIR / "modulus" / "export.py").is_file() + and (root / TOOLS_DIR / "format.py").is_file() + and (root / RUNTIME_DIR / "modulus" / "formatting.py").is_file() + and (root / RUNTIME_DIR / "modulus" / "format_layout.py").is_file() + ) + +def env_flag(name): + return os.environ.get(name, "").lower() in {"1", "true", "yes", "on"} + +def docker_marker(root): + config = read_json(root / "modulus.json") + marker = config.get( + "tooling", + {}, + ).get("docker", {}).get("marker", DEFAULT_DOCKER_MARKER) + return pathlib.Path(str(marker)) + +def direct_mode(root): + return env_flag("MODULUS_DIRECT") or docker_marker(root).exists() + +def run(command, input_text=None, capture=False, display_command=None, env=None, check=True): + visible = display_command if display_command is not None else command + print("+", " ".join(shlex.quote(str(part)) for part in visible)) + completed = subprocess.run( + [str(part) for part in command], + input=input_text, + stdout=subprocess.PIPE if capture else None, + stderr=subprocess.PIPE if capture else None, + text=True, + check=False, + env=env, + ) + if check and completed.returncode != 0: + if capture and completed.stdout: + print(completed.stdout, end="") + if capture and completed.stderr: + print(completed.stderr, end="", file=sys.stderr) + raise SystemExit(completed.returncode) + return completed + +def fetch_dockerfile(): + return f"""FROM debian:bookworm-slim + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update \\ + && apt-get install -y --no-install-recommends \\ + ca-certificates \\ + python3 \\ + python3-venv \\ + && rm -rf /var/lib/apt/lists/* + +RUN python3 -m venv /opt/modulus-python + +ENV PATH=/opt/modulus-python/bin:$PATH + +RUN python -m pip install --no-cache-dir conan=={CONAN_VERSION} + +WORKDIR /workspace +""" + +def ensure_fetch_image(offline=False): + inspected = subprocess.run( + [ + "docker", + "image", + "inspect", + FETCH_IMAGE, + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + if inspected.returncode == 0 and (offline or not docker_platform_args()): + return + if offline: + raise SystemExit('offline bootstrap requires the existing Modulus fetch image or MODULUS_DIRECT=1 with Conan installed') + run( + [ + "docker", + "build", + *docker_platform_args(), + "--tag", + FETCH_IMAGE, + "-", + ], + input_text=fetch_dockerfile(), + ) + +def in_container(): + return pathlib.Path("/.dockerenv").exists() + +def docker_platform_args(): + platform = os.environ.get("MODULUS_DOCKER_PLATFORM", "").strip() + return ["--platform", platform] if platform else [] + +def docker_workspace_args(root): + if in_container(): + container = os.environ.get("HOSTNAME", "").strip() + if not container: + raise SystemExit("HOSTNAME is required to share the CI workspace with the Modulus fetch container") + return [ + "--volumes-from", + container, + ] + return [ + "--volume", + f"{root}:{root}", + ] + +def fetch_env(root): + env = os.environ.copy() + for name in list(env): + if name in {"CONAN_USER", "CONAN_TOKEN"} or name.startswith(("CONAN_LOGIN_USERNAME", "CONAN_PASSWORD")): + del env[name] + env["CONAN_HOME"] = str(root / CONAN_HOME_DIR) + env["CONAN_NON_INTERACTIVE"] = "1" + return env + +def run_fetch(root, command, capture=False, display_command=None, check=True, offline=False): + if direct_mode(root): + return run(command, capture=capture, display_command=display_command, env=fetch_env(root), check=check) + ensure_fetch_image(offline=offline) + docker_command = [ + "docker", + "run", + "--rm", + "--interactive", + *(["--pull=never", "--network=none"] if offline else []), + *docker_platform_args(), + "--user", + f"{os.getuid()}:{os.getgid()}", + "--env", + "HOME=/tmp", + "--env", + "CONAN_HOME", + "--env", + "CONAN_NON_INTERACTIVE", + *docker_workspace_args(root), + "--workdir", + str(root), + FETCH_IMAGE, + *command, + ] + return run(docker_command, capture=capture, display_command=display_command, env=fetch_env(root), check=check) + +def configure_remote(root, remote_name, remote_url): + run_fetch( + root, + [ + "conan", + "remote", + "add", + "-f", + remote_name, + remote_url, + ], + ) + run_fetch(root, ["conan", "remote", "logout", remote_name]) + +def package_refs(data): + refs = [] + for location_data in data.values(): + if not isinstance(location_data, dict): + continue + for ref, ref_data in location_data.items(): + if not ref.startswith(PACKAGE_REF) or not isinstance(ref_data, dict): + continue + revisions = ref_data.get( + "revisions", + {}, + ) + if not isinstance(revisions, dict): + continue + for recipe_revision, revision_data in revisions.items(): + if not isinstance(revision_data, dict): + continue + packages = revision_data.get( + "packages", + {}, + ) + if not isinstance(packages, dict): + continue + for package_id, package_data in packages.items(): + package_revisions = package_data.get( + "revisions", + {}, + )if isinstance(package_data, dict) else {} + if isinstance(package_revisions, dict) and package_revisions: + for package_revision in package_revisions: + refs.append(f"{PACKAGE_REF}#{recipe_revision}:{package_id}#{package_revision}") + else: + refs.append(f"{PACKAGE_REF}#{recipe_revision}:{package_id}") + return sorted(refs) + +def find_modulus(root, remote_name=None): + completed = run_fetch( + root, + [ + "conan", + "list", + PACKAGE_REF + "#latest:*#latest", + *(["-r", remote_name] if remote_name else ["-c"]), + "-f", + "json", + ], + capture=True, + offline=remote_name is None, + ) + refs = package_refs(json.loads(completed.stdout)) + if not refs: + raise SystemExit(f"no {PACKAGE_REF} package available in {remote_name or 'the local Conan cache'}") + package_ref = refs[-1] + if remote_name: + run_fetch(root, ["conan", "download", package_ref, "-r", remote_name]) + completed = run_fetch( + root, + [ + "conan", + "cache", + "path", + package_ref, + ], + capture=True, + offline=remote_name is None, + ) + package_root = pathlib.Path(completed.stdout.strip()) + if not (package_root / "modulus" / "export.py").is_file() or not (package_root / "tools").is_dir(): + raise SystemExit(f"downloaded {PACKAGE_REF} package does not contain Modulus export files: {package_root}") + return package_root + +def materialize_runtime(root, no_remote=False): + remote_name, remote_url = remote_config(root) + if not no_remote: + configure_remote(root, remote_name, remote_url) + package_root = find_modulus(root, None if no_remote else remote_name) + sys.path.insert(0, str(package_root)) + import modulus.export + modulus.export.sync_project(root, package_root) + +def ensure_runtime(root, update=False, no_remote=False): + if update or not runtime_ready(root): + materialize_runtime(root, no_remote=no_remote) + +def delegate(command, argv): + root = repo_root() + if command in {"clangd", "format"}: + if not runtime_ready(root): + print(f"Modulus {command} tooling is missing; run ./modulus.py configure first", file=sys.stderr) + return 1 + else: + ensure_runtime(root, update=command == "configure" and "--update" in argv, no_remote="--no-remote" in argv) + sys.path.insert(0, str(root / RUNTIME_DIR)) + import modulus.cli + previous_argv = sys.argv + previous_cwd = pathlib.Path.cwd() + sys.argv = [ + str(root / "modulus.py"), + *argv, + ] + os.chdir(root) + try: + return modulus.cli.main(command) + finally: + os.chdir(previous_cwd) + sys.argv = previous_argv + +def main(argv): + if not argv or argv[0] in ("-h", "--help"): + print("usage: python3 modulus.py " + "|".join(COMMANDS) + " ...") + return 0 if argv else 2 + command = argv[0] + if command not in COMMANDS: + raise SystemExit(f"unknown Modulus command: {command}") + return delegate(command, argv[1:]) + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/modulus/clangd.py b/modulus/clangd.py index bb0abad..a304128 100644 --- a/modulus/clangd.py +++ b/modulus/clangd.py @@ -22,9 +22,8 @@ def settings(context): def recipe_digest(context): digest = hashlib.sha256(context.dockerfile.read_bytes()) - requirements = context.dockerfile.parent / "format-requirements.txt" - if requirements.is_file(): - digest.update(requirements.read_bytes()) + for path in getattr(context, "docker_inputs", [context.dockerfile.parent / "format-requirements.txt"]): + digest.update(path.read_bytes()) return digest.hexdigest() def current_image(context, architecture=None): diff --git a/modulus/cli.py b/modulus/cli.py index 6887256..2f0828b 100644 --- a/modulus/cli.py +++ b/modulus/cli.py @@ -77,7 +77,8 @@ class build_context: self.default_target = self.targets[self.default_target_name] docker = self.tooling["docker"] self.docker_image = str(docker["image"]) - self.dockerfile = self.tools_dir / "docker" / "Dockerfile" + self.dockerfile = repo_root / docker["dockerfile"] if docker.get("dockerfile") else self.tools_dir / "docker" / "Dockerfile" + self.docker_inputs = [self.tools_dir / "docker" / "format-requirements.txt", *(repo_root / path for path in docker.get("inputs", []))] self.docker_workdir = str(repo_root) self.docker_marker = pathlib.Path(str(docker["marker"])) self.upload_env = str(self.tooling["upload_env"]) @@ -148,13 +149,6 @@ def conan_token(): def conan_credentials(): return conan_user(), conan_token() -def has_conan_credentials(): - user, token = conan_credentials() - if bool(user) != bool(token): - print("CONAN_USER and CONAN_TOKEN must either both be set or both be unset", file=sys.stderr) - return None - return bool(user) - def require_conan_credentials(operation): user, token = conan_credentials() if bool(user) != bool(token): @@ -165,11 +159,20 @@ def require_conan_credentials(operation): return None return user, token +def subprocess_environment(context, command): + env = os.environ.copy() + env["CONAN_HOME"] = str(context.conan_cache_dir) + if str(command[0]) == "conan": + for name in list(env): + if name in {"CONAN_USER", "CONAN_TOKEN"} or name.startswith(("CONAN_LOGIN_USERNAME", "CONAN_PASSWORD")): + del env[name] + env["CONAN_NON_INTERACTIVE"] = "1" + return env + def run(context, command, cwd=None, env_overrides=None, env_unset=None, display_command=None): visible = display_command if display_command is not None else command print("+", " ".join(shlex.quote(str(part)) for part in visible)) - env = os.environ.copy() - env["CONAN_HOME"] = str(context.conan_cache_dir) + env = subprocess_environment(context, command) for name in env_unset or (): env.pop(name, None) if env_overrides: @@ -179,8 +182,7 @@ def run(context, command, cwd=None, env_overrides=None, env_unset=None, display_ def run_capture(context, command, cwd=None, env_overrides=None, display_command=None): visible = display_command if display_command is not None else command print("+", " ".join(shlex.quote(str(part)) for part in visible)) - env = os.environ.copy() - env["CONAN_HOME"] = str(context.conan_cache_dir) + env = subprocess_environment(context, command) if env_overrides: env.update(env_overrides) completed = subprocess.run( @@ -201,7 +203,8 @@ def run_capture(context, command, cwd=None, env_overrides=None, display_command= def configure_conan_remote(context, authenticate=False): remote = conan_remote(context) - if has_conan_credentials() is None: + credentials = require_conan_credentials("upload Conan packages") if authenticate else None + if authenticate and credentials is None: return 1 status = run( context, @@ -217,10 +220,7 @@ def configure_conan_remote(context, authenticate=False): if status != 0: return status if not authenticate: - return 0 - credentials = require_conan_credentials("upload Conan packages") - if credentials is None: - return 1 + return run(context, ["conan", "remote", "logout", remote]) user, token = credentials return run( context, @@ -244,19 +244,26 @@ def configure_conan_remote(context, authenticate=False): ], ) -def docker_passthrough_env(context): - return ( +def docker_passthrough_env(context, publishing=False): + names = ( "CI", "CONAN_HOME", - "CONAN_TOKEN", - "CONAN_USER", "MODULUS_DIRECT", "MODULUS_ALLOW_PACKAGE_REPLACE", - "PACKAGE_REGISTRY_TOKEN", - "PACKAGE_REGISTRY_USER", context.upload_env, context.package_upload_env, ) + if publishing: + names += ("CONAN_TOKEN", "CONAN_USER", "PACKAGE_REGISTRY_TOKEN", "PACKAGE_REGISTRY_USER") + return names + +def docker_socket_args(context, command): + if command not in context.tooling["docker"].get("socket_commands", []): + return [] + socket = pathlib.Path("/var/run/docker.sock") + if not socket.is_socket(): + return [] + return ["--volume", f"{socket}:{socket}", "--group-add", str(socket.stat().st_gid)] def docker_platform_arch(): platform = os.environ.get("MODULUS_DOCKER_PLATFORM", "").strip() @@ -303,6 +310,8 @@ def docker_run(context, command, docker_args=None, env_overrides=None): if status != 0: return status resolved_env_overrides = dict(env_overrides or {}) + operation = pathlib.Path(command[0]).stem + publishing = operation == "package" and "--no-remote" not in command and (should_upload(context) or should_upload_artifacts(context)) docker_command = [ "docker", "run", @@ -313,10 +322,11 @@ def docker_run(context, command, docker_args=None, env_overrides=None): f"{os.getuid()}:{os.getgid()}", "--env", "HOME=/tmp", + *docker_socket_args(context, operation), ] if docker_args: docker_command.extend(docker_args) - for name in docker_passthrough_env(context): + for name in docker_passthrough_env(context, publishing): if name in os.environ: docker_command.extend(["--env", name]) if resolved_env_overrides: @@ -435,6 +445,7 @@ def configure_one(context, target, update, no_remote=False): status = run(context, command) if status != 0: return status + clear_cmake_cache(target) return run( context, [ @@ -484,7 +495,6 @@ def configure(context, argv): update = args.update for target in context.selected_targets(args.target, args.all): target.build_dir.mkdir(parents=True, exist_ok=True) - clear_cmake_cache(target) status = configure_one(context, target, update, no_remote=args.no_remote) if status != 0: return status @@ -668,7 +678,6 @@ def lsp(context, argv): return status for target in context.selected_targets(args.target, all_targets=False): target.build_dir.mkdir(parents=True, exist_ok=True) - clear_cmake_cache(target) status = configure_one(context, target, args.update, no_remote=args.no_remote) if status != 0: return status @@ -768,10 +777,13 @@ def upload_packages(context, replace=False): print("--replace requires MODULUS_ALLOW_PACKAGE_REPLACE=1", file=sys.stderr) return 1 remote = conan_remote(context) - status = configure_conan_remote(context, authenticate=True) - if status != 0: - return status - status = upload_all_packages(context, remote, force=replace) + try: + status = configure_conan_remote(context, authenticate=True) + if status == 0: + status = upload_all_packages(context, remote, force=replace) + finally: + logout_status = run(context, ["conan", "remote", "logout", remote]) + status = status or logout_status if status != 0 and not replace: print(f"upload failed for {package_name(context)}/latest. Use --replace with MODULUS_ALLOW_PACKAGE_REPLACE=1 to intentionally replace an existing version.", file=sys.stderr) return status diff --git a/modulus/config.py b/modulus/config.py index 86ad4c1..fe1656b 100644 --- a/modulus/config.py +++ b/modulus/config.py @@ -123,6 +123,9 @@ lsp_compile_database_keys = { docker_keys = { "image", "marker", + "dockerfile", + "inputs", + "socket_commands", } remote_keys = { "name", @@ -240,6 +243,8 @@ def normalize(data): ) tooling["docker"].setdefault("image", "modulus-build:dev") tooling["docker"].setdefault("marker", "/opt/modulus-build-image") + tooling["docker"].setdefault("inputs", []) + tooling["docker"].setdefault("socket_commands", []) tooling.setdefault("upload_env", f"{environment_prefix(name)}_CONAN_UPLOAD") tooling.setdefault( "remote", @@ -428,6 +433,13 @@ def validate_tooling(tooling): validate_keys(docker, docker_keys, "tooling.docker") validate_string(docker.get("image"), "tooling.docker.image", optional=True) validate_absolute_path(docker.get("marker"), "tooling.docker.marker", optional=True) + validate_path(docker.get("dockerfile"), "tooling.docker.dockerfile", optional=True) + for key in ("inputs", "socket_commands"): + validate_string_list(docker.get(key, []), f"tooling.docker.{key}") + for path in docker.get("inputs", []): + validate_path(path, "tooling.docker.inputs") + if any(command not in {"build", "package"} for command in docker.get("socket_commands", [])): + raise config_error("tooling.docker.socket_commands may contain only build and package") remote = tooling.get("remote") if remote is not None: if not isinstance(remote, dict): diff --git a/modulus/export.py b/modulus/export.py index 0657640..fe29d76 100644 --- a/modulus/export.py +++ b/modulus/export.py @@ -2,6 +2,7 @@ import json import pathlib import shutil import sys +import tempfile import modulus.config @@ -45,9 +46,31 @@ def sync_project(repo_root, package_root): repo_root = pathlib.Path(repo_root).resolve() package_root = pathlib.Path(package_root).resolve() config = modulus.config.load_repo_config(repo_root) - sync_runtime(repo_root, package_root) - sync_tools(repo_root, package_root) - sync_editor_files(repo_root, config) + stage_parent = repo_root / common_dir + stage_parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix=".modulus-export-", dir=stage_parent) as directory: + staged = pathlib.Path(directory) + sync_runtime(staged, package_root) + sync_tools(staged, package_root) + installed = [] + backups = [] + try: + for index, relative in enumerate((runtime_dir, tools_dir)): + target = repo_root / relative + target.parent.mkdir(parents=True, exist_ok=True) + if target.exists(): + backup = staged / f"backup-{index}" + target.rename(backup) + backups.append((backup, target)) + (staged / relative).rename(target) + installed.append(target) + sync_editor_files(repo_root, config) + except BaseException: + for target in installed: + shutil.rmtree(target) + for backup, target in backups: + backup.rename(target) + raise def sync_runtime(repo_root, package_root): dst = repo_root / runtime_dir diff --git a/schemas/modulus.schema.json b/schemas/modulus.schema.json index 5432c87..85e087f 100644 --- a/schemas/modulus.schema.json +++ b/schemas/modulus.schema.json @@ -176,6 +176,18 @@ "marker": { "type": "string", "pattern": "^/" + }, + "dockerfile": { + "$ref": "#/$defs/relative_path" + }, + "inputs": { + "type": "array", + "items": {"$ref": "#/$defs/relative_path"} + }, + "socket_commands": { + "type": "array", + "uniqueItems": true, + "items": {"enum": ["build", "package"]} } } }, diff --git a/src/json_config.cpp b/src/json_config.cpp index 8e66a5e..cbfbbc9 100644 --- a/src/json_config.cpp +++ b/src/json_config.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include @@ -183,6 +184,23 @@ auto modulus::json_config::optional_int32_field(boost::json::object const &objec return static_cast(*number); } +auto modulus::json_config::optional_float64_field(boost::json::object const &object, std::string_view name, std::float64_t fallback, std::float64_t min_value, std::float64_t max_value, modulus::json_config::options config) -> std::expected { + auto found = object.find(name); + if (found == object.end()) { + return fallback; + } + auto const &value = found->value(); + if (!value.is_number()) { + return std::unexpected(modulus::json_config::invalid(config, std::string{name} + " must be a number")); + } + auto number = value.is_double() ? static_cast(value.as_double()) : value.is_uint64() ? static_cast(value.as_uint64()) + : static_cast(value.as_int64()); + if (!std::isfinite(number) || number < min_value || number > max_value) { + return std::unexpected(modulus::json_config::invalid(config, std::string{name} + " is out of range")); + } + return number; +} + auto modulus::json_config::positive_milliseconds_field(boost::json::object const &object, std::string_view name, std::chrono::milliseconds fallback, std::uint64_t max_value, modulus::json_config::options config) -> std::expected { auto parsed = modulus::json_config::positive_uint64_field(object, name, static_cast(fallback.count()), max_value, config); if (!parsed) { diff --git a/tests/test.cpp b/tests/test.cpp index 4c8a0f1..efbc86d 100644 --- a/tests/test.cpp +++ b/tests/test.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -270,3 +271,22 @@ BOOST_AUTO_TEST_CASE(modulus_load_config_handles_files) { auto invalid = modulus::load_config(invalid_path.string()); BOOST_REQUIRE(!invalid.has_value()); } + +BOOST_AUTO_TEST_CASE(modulus_json_float_fields_validate_numeric_types_and_bounds) { + auto object = boost::json::object{}; + auto options = modulus::json_config::options{.category = "domain", .invalid_argument_code = 42}; + auto read = [&] { return modulus::json_config::optional_float64_field(object, "gain", 0.5, 0.0, 1.0, options); }; + BOOST_REQUIRE(read()); + BOOST_TEST(static_cast(*read()) == 0.5); + for (auto const &value : {boost::json::value{0}, boost::json::value{std::uint64_t{1}}, boost::json::value{0.25}}) { + object["gain"] = value; + BOOST_REQUIRE(read()); + } + for (auto const &value : {boost::json::value{-1}, boost::json::value{2U}, boost::json::value{true}, boost::json::value{"0.5"}, boost::json::value{nullptr}, boost::json::value{std::numeric_limits::infinity()}, boost::json::value{std::numeric_limits::quiet_NaN()}}) { + object["gain"] = value; + auto result = read(); + BOOST_REQUIRE(!result); + BOOST_TEST(std::string{result.error().category()} == "domain"); + BOOST_TEST(result.error().code() == 42); + } +} diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py new file mode 100644 index 0000000..2a0c9b6 --- /dev/null +++ b/tests/test_bootstrap.py @@ -0,0 +1,125 @@ +import ast +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +import modulus.bootstrap +import modulus.cli +import modulus.config +import modulus.export + +class BootstrapTests(unittest.TestCase): + def test_recipe_exports_prune_generated_trees(self): + workspace = Path(modulus.bootstrap.__file__).parents[2] + for name in ('modulus', 'ligo', 'noto', 'luma', 'sonus', 'product-vamp'): + recipe = workspace / name / 'conanfile.py' + if not recipe.exists(): + continue + fields = {target.id: ast.literal_eval(node.value) for node in ast.walk(ast.parse(recipe.read_text())) if isinstance(node, ast.Assign) for target in node.targets if isinstance(target, ast.Name) and target.id in {'exports', 'exports_sources'}} + for field in ('exports', 'exports_sources'): + self.assertTrue({'!build', '!.git', '!test_package/build'}.issubset(fields[field]), f'{name}: {field}') + def test_consumers_use_canonical_bootstrap(self): + canonical = Path(modulus.bootstrap.__file__) + for name in ('ligo', 'noto', 'luma', 'sonus', 'product-vamp'): + consumer = canonical.parents[2] / name / 'modulus.py' + if consumer.exists(): + self.assertEqual(canonical.read_bytes(), consumer.read_bytes(), name) + def test_read_environments_strip_all_credential_forms(self): + credentials = dict.fromkeys(('CONAN_USER', 'CONAN_TOKEN', 'CONAN_LOGIN_USERNAME', 'CONAN_PASSWORD', 'CONAN_LOGIN_USERNAME_KHODO', 'CONAN_PASSWORD_KHODO'), 'secret') + root = Path('/tmp/project') + context = modulus.cli.build_context(root, modulus.config.normalize({'package': {'name': 'sample'}})) + with patch.dict(os.environ, credentials): + for env in (modulus.bootstrap.fetch_env(root), modulus.cli.subprocess_environment(context, ['conan', 'install'])): + self.assertFalse(set(credentials) & env.keys()) + self.assertEqual(env['CONAN_NON_INTERACTIVE'], '1') + self.assertEqual(env['CONAN_HOME'], str(root / 'build/build_common/conan_cache')) + def test_update_refreshes_runtime_and_offline_stays_offline(self): + root = Path('/tmp/project') + with patch.object(modulus.bootstrap, 'runtime_ready', return_value=True), patch.object(modulus.bootstrap, 'materialize_runtime') as refresh: + modulus.bootstrap.ensure_runtime(root) + refresh.assert_not_called() + modulus.bootstrap.ensure_runtime(root, update=True, no_remote=True) + refresh.assert_called_once_with(root, no_remote=True) + with patch.object(modulus.bootstrap, 'run_fetch', return_value=subprocess.CompletedProcess([], 0, '{}')) as fetch: + with self.assertRaisesRegex(SystemExit, 'local Conan cache'): + modulus.bootstrap.find_modulus(root) + self.assertNotIn('-r', fetch.call_args.args[1]) + def test_fetch_failure_preserves_runtime(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + runtime = root / modulus.bootstrap.RUNTIME_DIR / 'modulus/cli.py' + runtime.parent.mkdir(parents=True) + runtime.write_text('old runtime') + with patch.object(modulus.bootstrap, 'remote_config', return_value=('khodo', 'https://example.test')), patch.object(modulus.bootstrap, 'configure_remote'), patch.object(modulus.bootstrap, 'find_modulus', side_effect=SystemExit(1)): + with self.assertRaises(SystemExit): + modulus.bootstrap.materialize_runtime(root) + self.assertEqual(runtime.read_text(), 'old runtime') + def test_offline_bootstrap_never_builds_a_fetch_image(self): + with patch.object(modulus.bootstrap.subprocess, 'run', return_value=subprocess.CompletedProcess([], 1)), patch.object(modulus.bootstrap, 'run') as build: + with self.assertRaisesRegex(SystemExit, 'offline bootstrap requires'): + modulus.bootstrap.ensure_fetch_image(offline=True) + build.assert_not_called() + def test_export_rolls_back_when_editor_refresh_fails(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'modulus.json').write_text(json.dumps({'package': {'name': 'sample'}})) + runtime = root / modulus.export.runtime_dir / 'modulus/cli.py' + runtime.parent.mkdir(parents=True) + runtime.write_text('old runtime') + package = root / 'package' + (package / 'modulus').mkdir(parents=True) + (package / 'modulus/cli.py').write_text('new runtime') + (package / 'tools').mkdir() + with patch.object(modulus.export, 'sync_editor_files', side_effect=OSError('editor is read-only')): + with self.assertRaisesRegex(OSError, 'read-only'): + modulus.export.sync_project(root, package) + self.assertEqual(runtime.read_text(), 'old runtime') + self.assertFalse((root / modulus.export.tools_dir).exists()) + def test_public_remote_error_is_noninteractive_and_does_not_delete_packages(self): + completed = subprocess.CompletedProcess([], 1, '', '401 Unauthorized') + with patch.object(modulus.bootstrap, 'direct_mode', return_value=True), patch.object(modulus.bootstrap.subprocess, 'run', return_value=completed) as run: + with self.assertRaises(SystemExit): + modulus.bootstrap.find_modulus(Path('/tmp/project'), 'khodo') + self.assertEqual(run.call_args.args[0][:2], ['conan', 'list']) + self.assertEqual(run.call_args.kwargs['env']['CONAN_NON_INTERACTIVE'], '1') + def test_export_stages_all_payloads_before_replacing_runtime(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'modulus.json').write_text(json.dumps({'package': {'name': 'sample'}})) + runtime = root / modulus.export.runtime_dir / 'modulus/cli.py' + runtime.parent.mkdir(parents=True) + runtime.write_text('old runtime') + package = root / 'package' + (package / 'modulus').mkdir(parents=True) + (package / 'modulus/cli.py').write_text('new runtime') + with self.assertRaisesRegex(RuntimeError, 'missing tools'): + modulus.export.sync_project(root, package) + self.assertEqual(runtime.read_text(), 'old runtime') + def test_upload_exception_still_logs_out(self): + context = modulus.cli.build_context(Path('/tmp/project'), modulus.config.normalize({'package': {'name': 'sample'}})) + with patch.object(modulus.cli, 'configure_conan_remote', return_value=0), patch.object(modulus.cli, 'upload_all_packages', side_effect=RuntimeError('upload failed')), patch.object(modulus.cli, 'run', return_value=0) as run: + with self.assertRaisesRegex(RuntimeError, 'upload failed'): + modulus.cli.upload_packages(context) + run.assert_called_once_with(context, ['conan', 'remote', 'logout', 'khodo']) + def test_socket_is_scoped_and_docker_inputs_change_fingerprint(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + config = modulus.config.normalize({'package': {'name': 'sample'}, 'tooling': {'docker': {'dockerfile': 'package/build/Dockerfile', 'inputs': ['requirements.txt'], 'socket_commands': ['build', 'package']}}}) + context = modulus.cli.build_context(root, config) + context.dockerfile.parent.mkdir(parents=True) + context.dockerfile.write_text('FROM scratch') + context.docker_inputs[0].parent.mkdir(parents=True) + for path in context.docker_inputs: + path.write_text('old') + before = modulus.clangd.recipe_digest(context) + context.docker_inputs[-1].write_text('new') + self.assertNotEqual(before, modulus.clangd.recipe_digest(context)) + with patch.object(Path, 'is_socket', return_value=True), patch.object(Path, 'stat') as stat: + stat.return_value.st_gid = 123 + for command in ('configure', 'test', 'lsp', 'clangd', 'format'): + self.assertEqual(modulus.cli.docker_socket_args(context, command), []) + self.assertIn('/var/run/docker.sock:/var/run/docker.sock', modulus.cli.docker_socket_args(context, 'package')) diff --git a/tests/test_clangd.py b/tests/test_clangd.py index 2a0d52d..08cc8e1 100644 --- a/tests/test_clangd.py +++ b/tests/test_clangd.py @@ -23,6 +23,7 @@ class clangd_test(unittest.TestCase): self.context.docker_marker = self.root / "not-in-container" self.context.dockerfile.parent.mkdir(parents=True) self.context.dockerfile.write_text("FROM sample\n") + self.context.docker_inputs[0].write_text("autopep8==2.3.2\n") self.image = "sha256:" + "1" * 64 def test_defaults_and_custom_settings(self): self.assertEqual(modulus.clangd.settings(self.context)["executable"], "/usr/bin/clangd-19") diff --git a/tests/test_cli.py b/tests/test_cli.py index d7d1e73..fb60975 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -110,13 +110,15 @@ class upload_packages_test(unittest.TestCase): self.assertEqual(cache.read_text(encoding="utf-8"), "keep") @unittest.mock.patch.object(modulus.cli, "upload_all_packages", return_value=37) @unittest.mock.patch.object(modulus.cli, "configure_conan_remote", return_value=0) - def test_failed_upload_does_not_force_by_default(self, configure, upload): + @unittest.mock.patch.object(modulus.cli, "run", return_value=0) + def test_failed_upload_does_not_force_by_default(self, run, configure, upload): context = self.context() with unittest.mock.patch.dict(os.environ, {}, clear=True): status = modulus.cli.upload_packages(context) self.assertEqual(status, 37) configure.assert_called_once_with(context, authenticate=True) upload.assert_called_once_with(context, "origin", force=False) + run.assert_called_once_with(context, ["conan", "remote", "logout", "origin"]) @unittest.mock.patch.object(modulus.cli, "upload_all_packages") @unittest.mock.patch.object(modulus.cli, "configure_conan_remote") def test_replace_requires_guard_env(self, configure, upload): @@ -127,7 +129,8 @@ class upload_packages_test(unittest.TestCase): upload.assert_not_called() @unittest.mock.patch.object(modulus.cli, "upload_all_packages", return_value=0) @unittest.mock.patch.object(modulus.cli, "configure_conan_remote", return_value=0) - def test_replace_forces_upload_when_guarded(self, configure, upload): + @unittest.mock.patch.object(modulus.cli, "run", return_value=0) + def test_replace_forces_upload_when_guarded(self, run, configure, upload): context = self.context() with unittest.mock.patch.dict( os.environ, @@ -186,8 +189,9 @@ class upload_packages_test(unittest.TestCase): def test_docker_passthrough_includes_replace_env(self): passthrough = modulus.cli.docker_passthrough_env(self.context()) self.assertIn("MODULUS_ALLOW_PACKAGE_REPLACE", passthrough) - self.assertIn("PACKAGE_REGISTRY_USER", passthrough) - self.assertIn("PACKAGE_REGISTRY_TOKEN", passthrough) + for name in ("CONAN_USER", "CONAN_TOKEN", "PACKAGE_REGISTRY_USER", "PACKAGE_REGISTRY_TOKEN"): + self.assertNotIn(name, passthrough) + self.assertIn(name, modulus.cli.docker_passthrough_env(self.context(), publishing=True)) self.assertIn("SAMPLE_PACKAGE_UPLOAD", passthrough) self.assertNotIn("CONAN_PACKAGE_VERSION", passthrough) self.assertNotIn("MODULUS_EXPORT_RUNTIME", passthrough) @@ -368,7 +372,8 @@ class upload_packages_test(unittest.TestCase): with unittest.mock.patch.dict(os.environ, {}, clear=True): status = modulus.cli.configure_conan_remote(self.context()) self.assertEqual(status, 0) - self.assertEqual(len(run.call_args_list), 1) + self.assertEqual(len(run.call_args_list), 2) + self.assertEqual(run.call_args_list[-1].args[1], ["conan", "remote", "logout", "origin"]) self.assertEqual( run.call_args_list[0].args[1], [ @@ -381,11 +386,11 @@ class upload_packages_test(unittest.TestCase): ], ) @unittest.mock.patch.object(modulus.cli, "run", return_value=0) - def test_configure_remote_rejects_partial_conan_credentials(self, run): + def test_configure_remote_ignores_partial_conan_credentials(self, run): with unittest.mock.patch.dict(os.environ, {"CONAN_USER": "dev"}, clear=True): status = modulus.cli.configure_conan_remote(self.context()) - self.assertEqual(status, 1) - run.assert_not_called() + self.assertEqual(status, 0) + self.assertEqual(run.call_args_list[-1].args[1], ["conan", "remote", "logout", "origin"]) @unittest.mock.patch.object(modulus.cli, "run", return_value=0) def test_configure_remote_skips_login_until_authentication_is_requested(self, run): context = self.context() @@ -399,7 +404,7 @@ class upload_packages_test(unittest.TestCase): ): status = modulus.cli.configure_conan_remote(context) self.assertEqual(status, 0) - self.assertEqual(len(run.call_args_list), 1) + self.assertEqual(len(run.call_args_list), 2) self.assertEqual( run.call_args_list[0].args[1], [ @@ -744,7 +749,7 @@ class upload_packages_test(unittest.TestCase): @unittest.mock.patch.object(modulus.cli, "create_package", return_value=0) @unittest.mock.patch.object(modulus.cli, "run", return_value=0) @unittest.mock.patch.object(modulus.cli, "in_docker", return_value=True) - def test_package_rejects_partial_conan_credentials_before_create(self, docker, run, create_package): + def test_package_checks_credentials_only_at_upload(self, docker, run, create_package): context = self.context() with unittest.mock.patch.dict( os.environ, @@ -762,8 +767,8 @@ class upload_packages_test(unittest.TestCase): ], ) self.assertEqual(status, 1) - create_package.assert_not_called() - run.assert_not_called() + create_package.assert_called_once() + self.assertEqual(run.call_args_list[-1].args[1], ["conan", "remote", "logout", "origin"]) docker.assert_called_once() @unittest.mock.patch.object(modulus.cli, "clear_conan_deps") @unittest.mock.patch.object(modulus.cli, "run", return_value=0)