diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 861c96a..32a6c94 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -38,7 +38,7 @@ jobs: python3 \ python3-venv python3 -m venv build/build_common/ci_python - build/build_common/ci_python/bin/python -m pip install --no-cache-dir conan==2.26.2 -r tools/docker/format-requirements.txt + build/build_common/ci_python/bin/python -m pip install --no-cache-dir conan==2.26.2 -r tools/docker/format_requirements.txt echo "$PWD/build/build_common/ci_python/bin" >> "$GITHUB_PATH" - name: Restore Conan cache uses: actions/cache@v4 @@ -52,7 +52,7 @@ jobs: run: python3 -m unittest discover -s tests -p 'test_*.py' - name: Check code style run: | - python3 -m pip install --no-cache-dir -r tools/docker/format-requirements.txt + python3 -m pip install --no-cache-dir -r tools/docker/format_requirements.txt python3 tools/format.py --check - name: Build native release run: ./tools/build.py --target x86_64_gcc12_release diff --git a/CMakeLists.txt b/CMakeLists.txt index 33f86a3..d12da8a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,7 +51,7 @@ if(BUILD_TESTING) endif() install(TARGETS modulus - EXPORT modulus-targets + EXPORT modulus_targets ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} @@ -59,7 +59,7 @@ install(TARGETS modulus install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) -install(EXPORT modulus-targets +install(EXPORT modulus_targets NAMESPACE modulus:: DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/modulus ) @@ -71,18 +71,18 @@ install(FILES configure_package_config_file( ${CMAKE_CURRENT_SOURCE_DIR}/cmake/modulus_config.cmake.in - ${CMAKE_CURRENT_BINARY_DIR}/modulus-config.cmake + ${CMAKE_CURRENT_BINARY_DIR}/modulusConfig.cmake INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/modulus ) write_basic_package_version_file( - ${CMAKE_CURRENT_BINARY_DIR}/modulus-config-version.cmake + ${CMAKE_CURRENT_BINARY_DIR}/modulusConfigVersion.cmake VERSION ${PROJECT_VERSION} COMPATIBILITY SameMajorVersion ) install(FILES - ${CMAKE_CURRENT_BINARY_DIR}/modulus-config.cmake - ${CMAKE_CURRENT_BINARY_DIR}/modulus-config-version.cmake + ${CMAKE_CURRENT_BINARY_DIR}/modulusConfig.cmake + ${CMAKE_CURRENT_BINARY_DIR}/modulusConfigVersion.cmake DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/modulus ) diff --git a/README.md b/README.md index 1b6260c..379c68a 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,16 @@ Refresh clangd compile databases with: Product repos can add typed extra CMake compile databases under `tooling.lsp.compile_databases`; this is intended for cases like Product Vamp's Pico firmware tree. +## Build Python + +Build infrastructure is available as exported `modulus` modules. `modulus.filesystem.publish_directories()` replaces validated directories with rollback; `modulus.artifacts` owns archive naming, checksums, and explicit generic-registry publishing. Product-specific generators and asset policies stay in the consumer. + +Projects needing asset tooling opt in with `"tooling": {"python": {"groups": ["assets"]}}`. The pinned Pillow, defusedxml, FontTools, and aggdraw requirements live in `modulus/assets_requirements.txt`. Configure prepares an isolated environment under `build/build_common/python`, fingerprinted by the interpreter, platform, and dependency manifest. Builds and offline commands reuse it without downloading; missing or stale dependencies require configure. C++-only repos use no optional environment, and Python dependencies never enter runtime packages. + +Run `./modulus.py python -m generator` (or another project module) through the exported launcher. It adds the exported Modulus package before the consumer's `modulus.py`, without changing the working directory. Configure supplies `Python3_EXECUTABLE`, `MODULUS_PYTHON_RUNNER`, and `MODULUS_PYTHON_ROOT` for CMake commands. Conan consumers can use `python_cmake_variables(recipe)` from the Modulus Python requirement. + +`./modulus.py publish` validates all configured archives and checksums before accessing the registry. Credentials are forwarded only for explicit publishing or enabled package uploads, never for configure/build downloads. Project-owned artifact and build paths use underscores; dependency names, CLI flags, Git URLs, and standard tool filenames retain their upstream spelling. + ## Editor Configuration `modulus.json` is the source for generated `.clangd`, `.clang-format`, and `pyrightconfig.json` files. Configure, LSP refresh, and tooling export regenerate them only when their content changes. Keep these generated files ignored; customize `tooling.editor` instead: @@ -64,7 +74,7 @@ Each supplied entry replaces that file's entire configuration. Omitted entries u Run `python3 tools/format.py` in Modulus, or `./modulus.py format` in a consumer. Optional repository-relative paths restrict the operation. `format --check` reports layout differences and prohibited aliases without writing files; CI runs this check. `format --stdin-filename /absolute/buffer.cpp` reads stdin and writes only formatted text, for editor integrations. -The build image pins clang-format 19.1.7, autopep8 2.3.2, and the syntax parser dependencies in `tools/docker/format-requirements.txt`. Python uses two-space indentation for both blocks and continuations, including multiline tuples, lists, dictionaries, imports, and calls. Continuations indent two spaces from their opening line rather than aligning to a column; closing brackets align with the opening line. Line-length and aggressive semantic fixes are disabled. C++ has no column limit, uses braces, and preserves include order and groups. The shared syntax-aware pass joins simple signatures/calls, removes blank lines inside functions and class/struct bodies, separates definitions outside classes by one blank line, and pads namespace interiors without indenting their bodies. Class members, including access labels, defaulted/deleted constructors, inline methods, and nested types, have no intervening blank lines. Blank lines within literal and comment contents are preserved. Comments, multiline strings, structured literals, and preprocessor content are not joined. Python formatting verifies AST equivalence before writing. C++ formatting supplements clang-format using concrete syntax boundaries, including common test macros; it is not a C++ type checker. +The build image pins clang-format 19.1.7, autopep8 2.3.2, and the syntax parser dependencies in `tools/docker/format_requirements.txt`. Python uses two-space indentation for both blocks and continuations, including multiline tuples, lists, dictionaries, imports, and calls. Continuations indent two spaces from their opening line rather than aligning to a column; closing brackets align with the opening line. Line-length and aggressive semantic fixes are disabled. C++ has no column limit, uses braces, and preserves include order and groups. The shared syntax-aware pass joins simple signatures/calls, removes blank lines inside functions and class/struct bodies, separates definitions outside classes by one blank line, and pads namespace interiors without indenting their bodies. Class members, including access labels, defaulted/deleted constructors, inline methods, and nested types, have no intervening blank lines. Blank lines within literal and comment contents are preserved. Comments, multiline strings, structured literals, and preprocessor content are not joined. Python formatting verifies AST equivalence before writing. C++ formatting supplements clang-format using concrete syntax boundaries, including common test macros; it is not a C++ type checker. Use full type/module names rather than C++ type/namespace aliases, Python type aliases, or renamed imports. The existing `std::*_t` portability definitions in `include/modulus/types.hpp` are the sole local exception; third-party SDK definitions are not rewritten. Formatting changes layout only, so alias diagnostics require explicit code changes. diff --git a/cmake/modulus_config.cmake.in b/cmake/modulus_config.cmake.in index d8e8cde..420cf00 100644 --- a/cmake/modulus_config.cmake.in +++ b/cmake/modulus_config.cmake.in @@ -8,7 +8,7 @@ if(NOT TARGET Boost::json) set(modulus_NOT_FOUND_MESSAGE "Modulus requires Boost.JSON") return() endif() -include("${CMAKE_CURRENT_LIST_DIR}/modulus-targets.cmake") +include("${CMAKE_CURRENT_LIST_DIR}/modulus_targets.cmake") foreach(_modulus_component IN LISTS modulus_FIND_COMPONENTS) set(modulus_${_modulus_component}_FOUND FALSE) if(_modulus_component STREQUAL "modulus" OR _modulus_component STREQUAL "boost_json") diff --git a/conanfile.py b/conanfile.py index a4763c6..8784c0e 100644 --- a/conanfile.py +++ b/conanfile.py @@ -7,6 +7,16 @@ import conan.tools.cmake import conan.tools.files import modulus.config +import modulus.python +import modulus.cli + +def python_cmake_variables(recipe): + root = pathlib.Path(recipe.dependencies['modulus'].package_folder) + context = modulus.cli.build_context(pathlib.Path(recipe.source_folder), modulus.config.load_repo_config(recipe.source_folder)) + context.tools_dir = root / 'tools' + context.common_dir = pathlib.Path(os.environ.get('MODULUS_PYTHON_COMMON_DIR', str(pathlib.Path(recipe.build_folder) / 'build_python'))) + modulus.python.interpreter(context, install=True, offline=os.environ.get('MODULUS_PYTHON_OFFLINE') == '1') + return modulus.python.cmake_definitions(context) class modulus_recipe(conan.ConanFile): name = "modulus" @@ -63,7 +73,7 @@ class modulus_recipe(conan.ConanFile): "glfw/*:with_wayland": False, "glfw/*:with_x11": False, } - exports = "modulus/*.py", "modulus.json", "!build", "!.git", "!test_package/build" + exports = "modulus/*.py", "modulus/*.txt", "modulus.json", "!build", "!.git", "!test_package/build" exports_sources = ( "!build", "!.git", @@ -148,6 +158,7 @@ class modulus_recipe(conan.ConanFile): cmake.install() package = pathlib.Path(self.package_folder) conan.tools.files.copy(self, "*.py", src=pathlib.Path(self.recipe_folder) / "modulus", dst=package / "modulus", keep_path=True) + conan.tools.files.copy(self, "*.txt", src=pathlib.Path(self.recipe_folder) / "modulus", dst=package / "modulus", keep_path=True) conan.tools.files.copy(self, "*", src=pathlib.Path(self.source_folder) / "tools", dst=package / "tools", keep_path=True) def package_info(self): cpp_info = self.cpp_info diff --git a/modulus/artifacts.py b/modulus/artifacts.py new file mode 100644 index 0000000..66b0004 --- /dev/null +++ b/modulus/artifacts.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +import argparse +import base64 +import hashlib +import json +import os +import pathlib +import sys +import tarfile +import urllib.error +import urllib.parse +import urllib.request + +REGISTRY_SLOT = "latest" +PAGE_LIMIT = 50 + +def archive_name(package_name, target): + return f'{package_name}_{target}.tar.gz' + +def create_archive(source_dir, archive): + with tarfile.open(archive, 'w:gz') as output: + for path in sorted(source_dir.rglob('*')): + output.add(path, arcname=path.relative_to(source_dir), recursive=False) + +def checksum_text(archive): + with archive.open('rb') as stream: + digest = hashlib.file_digest(stream, 'sha256').hexdigest() + return f'{digest} {archive.name}\n' + +def write_checksum(archive): + checksum = archive.with_name(archive.name + '.sha256') + checksum.write_text(checksum_text(archive), encoding='utf-8') + return checksum + +def parse_registry_url(registry_url): + parsed = urllib.parse.urlparse(registry_url.rstrip("/")) + parts = [urllib.parse.unquote(part) for part in parsed.path.split("/") if part] + if len(parts) < 4 or parts[-4] != "api" or parts[-3] != "packages" or parts[-1] != "generic": + raise SystemExit("package artifact registry_url must end with /api/packages/{owner}/generic") + owner = parts[-2] + prefix = "/" + "/".join(urllib.parse.quote(part, safe="") for part in parts[:-4]) if parts[:-4] else "" + return parsed, prefix, owner + +def build_url(parsed, path, query=""): + return urllib.parse.urlunparse(( + parsed.scheme, + parsed.netloc, + path, + "", + query, + "", + )) + +def quote_path(*parts): + return "/".join(urllib.parse.quote(part, safe="") for part in parts) + +def version_list_url(parsed, prefix, owner, package_name, page): + path = f"{prefix}/api/v1/packages/{quote_path(owner, 'generic', package_name)}" + query = urllib.parse.urlencode({ + "page": page, + "limit": PAGE_LIMIT, + }) + return build_url(parsed, path, query) + +def version_delete_url(parsed, prefix, owner, package_name, version): + path = f"{prefix}/api/packages/{quote_path(owner, 'generic', package_name, version)}" + return build_url(parsed, path) + +def file_upload_url(registry_url, package_name, path): + return "/".join(( + registry_url.rstrip("/"), + urllib.parse.quote(package_name, safe=""), + REGISTRY_SLOT, + urllib.parse.quote(path.name, safe=""), + )) + +def authorization(user, token): + encoded = base64.b64encode(f"{user}:{token}".encode("utf-8")).decode("ascii") + return f"Basic {encoded}" + +def request(method, url, auth, data=None, headers=None, missing_ok=False): + request_headers = { + "Authorization": auth, + **(headers or {}), + } + request = urllib.request.Request(url, data=data, method=method, headers=request_headers) + try: + return urllib.request.urlopen(request, timeout=120) + except urllib.error.HTTPError as error: + if missing_ok and error.code == 404: + return None + message = f"{method} {url} failed with HTTP {error.code}" + raise SystemExit(message) from error + except urllib.error.URLError as error: + raise SystemExit(f"{method} {url} failed: {error.reason}") from error + +def list_versions(parsed, prefix, owner, package_name, auth): + versions = [] + page = 1 + while True: + response = request("GET", version_list_url(parsed, prefix, owner, package_name, page), auth, missing_ok=True) + if response is None: + return [] + with response: + data = json.loads(response.read().decode("utf-8")) + if not isinstance(data, list): + raise SystemExit("Gitea package version list response was not an array") + if not data: + break + for item in data: + if isinstance(item, dict) and isinstance(item.get("version"), str): + versions.append(item["version"]) + if len(data) < PAGE_LIMIT: + break + page += 1 + return sorted(set(versions)) + +def delete_version(parsed, prefix, owner, package_name, version, auth): + url = version_delete_url(parsed, prefix, owner, package_name, version) + response = request("DELETE", url, auth, missing_ok=True) + if response is None: + print(f"Package version already absent: {package_name}/{version}") + return + with response: + if response.status != 204: + raise SystemExit(f"DELETE {url} returned HTTP {response.status}") + print(f"Deleted package version: {package_name}/{version}") + +def discover_artifacts(dist_dir, package_name, targets): + artifacts = [] + for target in sorted(set(targets)): + archive = dist_dir / archive_name(package_name, target) + artifacts.extend((archive, archive.with_name(archive.name + ".sha256"))) + missing = [path for path in artifacts if not path.is_file()] + if missing: + found = sorted(path.name for path in dist_dir.glob(f"{package_name}*.tar.gz*") if path.is_file()) + raise SystemExit("missing artifact files: " + ", ".join(str(path) for path in missing) + + "\nFound: " + (", ".join(found) or "none") + + "\nPublish the current Modulus tooling, run configure --update, and recreate the package so the builder and publisher use the same filenames.") + for archive, checksum in zip(artifacts[::2], artifacts[1::2]): + if archive.is_symlink() or checksum.is_symlink(): + raise SystemExit('artifact files must not be symlinks') + if checksum.read_text(encoding='utf-8') != checksum_text(archive): + raise SystemExit(f'artifact checksum mismatch: {archive}') + return artifacts + +def upload_file(registry_url, package_name, path, auth): + url = file_upload_url(registry_url, package_name, path) + headers = { + "Content-Length": str(path.stat().st_size), + "Content-Type": "application/octet-stream", + } + with path.open("rb") as input_file: + response = request("PUT", url, auth, data=input_file, headers=headers) + with response: + if response.status != 201: + raise SystemExit(f"PUT {url} returned HTTP {response.status}") + print(f"Uploaded package file: {package_name}/{path.name}") + +def parse_args(argv): + parser = argparse.ArgumentParser(description="Publish the configured rolling Gitea generic package.") + parser.add_argument("--dist-dir", type=pathlib.Path) + return parser.parse_args(argv) + +def main(context, argv): + args = parse_args(argv) + if not context.artifact['enabled']: + raise SystemExit('artifact publishing is not enabled in modulus.json') + package_name, registry_url = context.package['name'], context.artifact['registry_url'] + targets = context.artifact['targets'] or [context.default_target_name] + artifacts = discover_artifacts(args.dist_dir or context.build_root / 'packages', package_name, targets) + user = os_environ("PACKAGE_REGISTRY_USER") + token = os_environ("PACKAGE_REGISTRY_TOKEN") + auth = authorization(user, token) + parsed, prefix, owner = parse_registry_url(registry_url) + existing_versions = list_versions(parsed, prefix, owner, package_name, auth) + # Gitea rejects duplicate file names within a generic package version, so + # replacing the rolling release requires deleting the old version first. + if REGISTRY_SLOT in existing_versions: + delete_version(parsed, prefix, owner, package_name, REGISTRY_SLOT, auth) + for artifact in artifacts: + upload_file(registry_url, package_name, artifact, auth) + for version in existing_versions: + if version != REGISTRY_SLOT: + delete_version(parsed, prefix, owner, package_name, version, auth) + return 0 + +def os_environ(name): + value = os.environ.get(name) + if not value: + raise SystemExit(f"{name} is required") + return value diff --git a/modulus/assets_requirements.txt b/modulus/assets_requirements.txt new file mode 100644 index 0000000..6e1f01e --- /dev/null +++ b/modulus/assets_requirements.txt @@ -0,0 +1,4 @@ +Pillow==12.3.0 +defusedxml==0.7.1 +fonttools==4.65.0 +aggdraw==1.4.1 diff --git a/modulus/bootstrap.py b/modulus/bootstrap.py index 856cec7..78060be 100755 --- a/modulus/bootstrap.py +++ b/modulus/bootstrap.py @@ -9,7 +9,7 @@ import sys PACKAGE_REF = "modulus/latest" CONAN_VERSION = "2.26.2" -FETCH_IMAGE = "modulus-fetch:dev" +FETCH_IMAGE = "modulus_fetch:dev" COMMANDS = ( "configure", "build", @@ -19,6 +19,8 @@ COMMANDS = ( "clangd", "format", "run", + "python", + "publish", ) COMMON_DIR = pathlib.Path("build") / "build_common" TOOLS_DIR = pathlib.Path("build") / "build_tools" @@ -26,7 +28,7 @@ RUNTIME_DIR = COMMON_DIR / "modulus" CONAN_HOME_DIR = COMMON_DIR / "conan_cache" DEFAULT_REMOTE_NAME = "khodo" DEFAULT_REMOTE_URL = "https://git.thyme-labs.com/api/packages/khodo/conan" -DEFAULT_DOCKER_MARKER = "/opt/modulus-build-image" +DEFAULT_DOCKER_MARKER = "/opt/modulus_build_image" def repo_root(): root = pathlib.Path(__file__).resolve().parent @@ -66,6 +68,9 @@ def runtime_ready(root): and (root / TOOLS_DIR / "format.py").is_file() and (root / RUNTIME_DIR / "modulus" / "formatting.py").is_file() and (root / RUNTIME_DIR / "modulus" / "format_layout.py").is_file() + and (root / RUNTIME_DIR / "modulus" / "python.py").is_file() + and (root / RUNTIME_DIR / "modulus" / "artifacts.py").is_file() + and (root / TOOLS_DIR / "python_runner.py").is_file() ) def env_flag(name): @@ -114,9 +119,9 @@ RUN apt-get update \\ python3-venv \\ && rm -rf /var/lib/apt/lists/* -RUN python3 -m venv /opt/modulus-python +RUN python3 -m venv /opt/modulus_python -ENV PATH=/opt/modulus-python/bin:$PATH +ENV PATH=/opt/modulus_python/bin:$PATH RUN python -m pip install --no-cache-dir conan=={CONAN_VERSION} diff --git a/modulus/clangd.py b/modulus/clangd.py index a304128..123b2ca 100644 --- a/modulus/clangd.py +++ b/modulus/clangd.py @@ -22,7 +22,7 @@ def settings(context): def recipe_digest(context): digest = hashlib.sha256(context.dockerfile.read_bytes()) - for path in getattr(context, "docker_inputs", [context.dockerfile.parent / "format-requirements.txt"]): + for path in getattr(context, "docker_inputs", [context.dockerfile.parent / "format_requirements.txt"]): digest.update(path.read_bytes()) return digest.hexdigest() @@ -107,12 +107,12 @@ def prepare(context): return 0 probe = context.tools_dir / "clangd_headers.py" result = subprocess.run( - [*isolated_run(image), "--volume", f"{probe}:/modulus-headers.py:ro", image, - "python3", "/modulus-headers.py", json.dumps(settings(context))], + [*isolated_run(image), "--volume", f"{probe}:/modulus_headers.py:ro", image, + "python3", "/modulus_headers.py", json.dumps(settings(context))], stdout=subprocess.PIPE, text=True, check=True, ) roots = json.loads(result.stdout) - with tempfile.TemporaryDirectory(prefix=".headers-", dir=cache.parent) as temporary: + with tempfile.TemporaryDirectory(prefix=".headers_", dir=cache.parent) as temporary: stage = pathlib.Path(temporary) / "cache" stage.mkdir() command = [*isolated_run(image), image, "tar", "--dereference", "--hard-dereference", @@ -177,7 +177,7 @@ def launch(context, argv): roots = read_manifest(cache) except (OSError, ValueError, KeyError) as error: raise RuntimeError("header cache is missing or invalid; run the lsp command") from error - name = "modulus-clangd-" + uuid.uuid4().hex + name = "modulus_clangd_" + uuid.uuid4().hex command = launch_command(context, argv, image, cache, roots, name) return run_server(command, name) except (OSError, ValueError, RuntimeError) as error: diff --git a/modulus/cli.py b/modulus/cli.py index 6fc5205..c1958c6 100644 --- a/modulus/cli.py +++ b/modulus/cli.py @@ -1,18 +1,18 @@ import argparse -import hashlib import os import pathlib import shlex import shutil import subprocess import sys -import tarfile import urllib.parse import modulus.config import modulus.export import modulus.clangd import modulus.formatting +import modulus.python +import modulus.artifacts build_root = pathlib.Path("build") common_dir = build_root / "build_common" @@ -49,6 +49,12 @@ def main(command): return product_run(context, sys.argv[1:]) if command == "lsp": return lsp(context, sys.argv[1:]) + if command in {'python', 'publish'}: + if not in_docker(context): + return docker_run(context, [f'./{context.tools_rel}/{command}.py', *sys.argv[1:]]) + if command == 'python': + return modulus.python.launch(context, sys.argv[1:]) + return modulus.artifacts.main(context, sys.argv[1:]) raise SystemExit(f"unknown modulus command: {command}") def find_repo_root(start): @@ -78,7 +84,7 @@ class build_context: docker = self.tooling["docker"] self.docker_image = str(docker["image"]) self.dockerfile = repo_root / docker["dockerfile"] if docker.get("dockerfile") else self.tools_dir / "docker" / "Dockerfile" - self.docker_inputs = [self.tools_dir / "docker" / "format-requirements.txt", *(repo_root / path for path in docker.get("inputs", []))] + self.docker_inputs = [self.tools_dir / "docker" / "format_requirements.txt", *(repo_root / path for path in docker.get("inputs", []))] self.docker_workdir = str(repo_root) self.docker_marker = pathlib.Path(str(docker["marker"])) self.upload_env = str(self.tooling["upload_env"]) @@ -167,6 +173,8 @@ def subprocess_environment(context, command): if name in {"CONAN_USER", "CONAN_TOKEN"} or name.startswith(("CONAN_LOGIN_USERNAME", "CONAN_PASSWORD")): del env[name] env["CONAN_NON_INTERACTIVE"] = "1" + env["MODULUS_PYTHON_COMMON_DIR"] = str(context.common_dir) + env["MODULUS_PYTHON_OFFLINE"] = "1" if "--no-remote" in command else "0" return env def run(context, command, cwd=None, env_overrides=None, env_unset=None, display_command=None): @@ -311,7 +319,7 @@ def docker_run(context, command, docker_args=None, env_overrides=None): return status resolved_env_overrides = dict(env_overrides or {}) operation = pathlib.Path(command[0]).stem - publishing = operation == "package" and "--no-remote" not in command and (should_upload(context) or should_upload_artifacts(context)) + publishing = operation == "publish" or (operation == "package" and "--no-remote" not in command and (should_upload(context) or should_upload_artifacts(context))) docker_command = [ "docker", "run", @@ -426,6 +434,7 @@ def dependency_build_args(context): return ["--build=missing"] def configure_one(context, target, update, no_remote=False): + modulus.python.interpreter(context, install=True, offline=no_remote) clear_conan_deps(target) command = [ "conan", @@ -461,6 +470,7 @@ def configure_one(context, target, update, no_remote=False): "-DCMAKE_EXPORT_COMPILE_COMMANDS=ON", f"-DBUILD_TESTING={'ON' if target.can_run else 'OFF'}", f"-DCMAKE_PREFIX_PATH={target.build_dir / 'conan'}", + *cmake_definition_args(modulus.python.cmake_definitions(context)), *cmake_definition_args(target.cmake_definitions), ], ) @@ -618,6 +628,7 @@ def validate_build_directory(repo_root, source_dir, build_dir): raise modulus.config.config_error(f"build directory must differ from the project and source roots: {build_dir}") def configure_lsp_database(context, database): + modulus.python.interpreter(context, install=True) build_dir = context.repo_root / str(database["build_dir"]) source_dir = context.repo_root / str(database.get("source_dir", ".")) validate_build_directory(context.repo_root, source_dir, build_dir) @@ -634,6 +645,7 @@ def configure_lsp_database(context, database): "-G", str(database.get("generator", "Ninja")), "-DCMAKE_EXPORT_COMPILE_COMMANDS=ON", + *cmake_definition_args(modulus.python.cmake_definitions(context)), *cmake_definition_args(database.get("cmake_definitions", {})), ], env_overrides=dict(database.get("env", {})), @@ -853,21 +865,8 @@ def package_artifact_install_prefix(context, stage_root): return stage_root return stage_root / install_prefix -def create_archive(source_dir, archive): - with tarfile.open(archive, "w:gz") as output: - for path in sorted(source_dir.rglob("*")): - output.add(path, arcname=path.relative_to(source_dir), recursive=False) - -def write_checksum(archive): - digest = hashlib.sha256() - with archive.open("rb") as input_file: - for chunk in iter(lambda: input_file.read(1024 * 1024), b""): - digest.update(chunk) - checksum = archive.with_name(archive.name + ".sha256") - checksum.write_text(f"{digest.hexdigest()} {archive.name}\n", encoding="utf-8") - return checksum - def create_artifact_package(context, target, update, no_remote=False): + modulus.python.interpreter(context, install=True, offline=no_remote) root = package_artifact_build_root(context, target) build_dir = root / "build" stage_root = root / "stage" @@ -912,6 +911,7 @@ def create_artifact_package(context, target, update, no_remote=False): "-DBUILD_TESTING=OFF", f"-DCMAKE_PREFIX_PATH={conan_dir}", f"-DCMAKE_INSTALL_PREFIX={install_prefix}", + *cmake_definition_args(modulus.python.cmake_definitions(context)), *package_artifact_cmake_definition_args(context), ], ) @@ -941,65 +941,15 @@ def create_artifact_package(context, target, update, no_remote=False): return status, [] dist_dir = package_artifact_dist_dir(context) dist_dir.mkdir(parents=True, exist_ok=True) - archive = dist_dir / f"{package_name(context)}-{target.name}.tar.gz" + archive = dist_dir / modulus.artifacts.archive_name(package_name(context), target.name) if archive.exists(): archive.unlink() - create_archive(stage_root, archive) - checksum = write_checksum(archive) + modulus.artifacts.create_archive(stage_root, archive) + checksum = modulus.artifacts.write_checksum(archive) print(f"Created {archive}") print(f"Created {checksum}") return 0, [archive, checksum] -def generic_package_upload_url(context, path): - base = str(context.artifact["registry_url"]).rstrip("/") - package = urllib.parse.quote(package_name(context), safe="") - filename = urllib.parse.quote(path.name, safe="") - return f"{base}/{package}/latest/{filename}" - -def upload_generic_artifact(context, path): - if not path.is_file(): - print(f"missing artifact: {path}", file=sys.stderr) - return 1 - credentials = package_registry_credentials() - if credentials is None: - return 1 - user, token = credentials - url = generic_package_upload_url(context, path) - command = ["curl", "--silent", "--show-error", "--user", f"{user}:{token}", "--request", "DELETE", "--output", os.devnull, "--write-out", "%{http_code}", url] - visible = [*command] - visible[4] = f"{user}:" - deleted = run_capture(context, command, display_command=visible) - if deleted.returncode != 0: - return deleted.returncode - if deleted.stdout.strip() not in {"204", "404"}: - print(f"cannot replace artifact {path.name}: HTTP {deleted.stdout.strip()}", file=sys.stderr) - return 1 - return run( - context, - [ - "curl", - "--fail", - "--silent", - "--show-error", - "--user", - f"{user}:{token}", - "--upload-file", - path, - url, - ], - display_command=[ - "curl", - "--fail", - "--silent", - "--show-error", - "--user", - f"{user}:", - "--upload-file", - path, - url, - ], - ) - def package_artifacts(context, argv): if not in_docker(context): return docker_run( @@ -1035,10 +985,7 @@ def package_artifacts(context, argv): return status artifacts.extend(created) if should_upload_artifacts(context) and not args.no_remote: - for artifact in artifacts: - status = upload_generic_artifact(context, artifact) - if status != 0: - return status + return modulus.artifacts.main(context, []) return 0 def package(context, argv): diff --git a/modulus/config.py b/modulus/config.py index a8ced6e..05d57d6 100644 --- a/modulus/config.py +++ b/modulus/config.py @@ -5,6 +5,7 @@ import re import pathlib import modulus.format_layout +import modulus.python class config_error(RuntimeError): pass @@ -79,6 +80,7 @@ test_keys = { "name", } tooling_keys = { + "python", "editor", "default_target", "targets", @@ -230,6 +232,10 @@ def normalize(data): ) if not isinstance(tooling, dict): raise config_error("modulus.json tooling must be an object") + python = tooling.setdefault("python", {}) + if not isinstance(python, dict): + raise config_error("modulus.json tooling.python must be an object") + python.setdefault("groups", []) formatting = tooling.setdefault("format", {}) if not isinstance(formatting, dict): raise config_error("modulus.json tooling.format must be an object") @@ -241,8 +247,8 @@ def normalize(data): "docker", {}, ) - tooling["docker"].setdefault("image", "modulus-build:dev") - tooling["docker"].setdefault("marker", "/opt/modulus-build-image") + tooling["docker"].setdefault("image", "modulus_build:dev") + tooling["docker"].setdefault("marker", "/opt/modulus_build_image") tooling["docker"].setdefault("inputs", []) tooling["docker"].setdefault("socket_commands", []) tooling.setdefault("upload_env", f"{environment_prefix(name)}_CONAN_UPLOAD") @@ -410,6 +416,14 @@ def validate_tests(tests): validate_cmake_name(tests.get("name"), "tests.name", optional=True) def validate_tooling(tooling): + python = tooling.get('python', {}) + if not isinstance(python, dict): + raise config_error('modulus.json tooling.python must be an object') + validate_keys(python, {'groups'}, 'tooling.python') + groups = python.get('groups', []) + validate_string_list(groups, 'tooling.python.groups') + if len(set(groups)) != len(groups) or any(group not in modulus.python.GROUPS for group in groups): + raise config_error('tooling.python.groups must contain unique supported groups: ' + ', '.join(modulus.python.GROUPS)) if "editor" in tooling: validate_editor(tooling["editor"]) validate_cmake_name(tooling.get("default_target"), "tooling.default_target", optional=True) diff --git a/modulus/export.py b/modulus/export.py index fe29d76..8160af4 100644 --- a/modulus/export.py +++ b/modulus/export.py @@ -5,6 +5,7 @@ import sys import tempfile import modulus.config +import modulus.filesystem build_dir = pathlib.Path("build") common_dir = build_dir / "build_common" @@ -48,29 +49,12 @@ def sync_project(repo_root, package_root): config = modulus.config.load_repo_config(repo_root) stage_parent = repo_root / common_dir stage_parent.mkdir(parents=True, exist_ok=True) - with tempfile.TemporaryDirectory(prefix=".modulus-export-", dir=stage_parent) as directory: + with tempfile.TemporaryDirectory(prefix=".modulus_export_", dir=stage_parent) as directory: staged = pathlib.Path(directory) sync_runtime(staged, package_root) sync_tools(staged, package_root) - installed = [] - backups = [] - try: - for index, relative in enumerate((runtime_dir, tools_dir)): - target = repo_root / relative - target.parent.mkdir(parents=True, exist_ok=True) - if target.exists(): - backup = staged / f"backup-{index}" - target.rename(backup) - backups.append((backup, target)) - (staged / relative).rename(target) - installed.append(target) + with modulus.filesystem.publish_directories([(staged / relative, repo_root / relative) for relative in (runtime_dir, tools_dir)]): sync_editor_files(repo_root, config) - except BaseException: - for target in installed: - shutil.rmtree(target) - for backup, target in backups: - backup.rename(target) - raise def sync_runtime(repo_root, package_root): dst = repo_root / runtime_dir diff --git a/modulus/filesystem.py b/modulus/filesystem.py new file mode 100644 index 0000000..be7726b --- /dev/null +++ b/modulus/filesystem.py @@ -0,0 +1,56 @@ +"""Publish validated build directories with rollback on a failed replacement.""" + +import contextlib +import pathlib +import shutil +import tempfile + +def has_symlink(path): + path = pathlib.Path(path).absolute() + return any(parent.is_symlink() for parent in (path, *path.parents)) + +@contextlib.contextmanager +def publish_directories(replacements): + pairs = [(pathlib.Path(source).absolute(), pathlib.Path(target).absolute()) for source, target in replacements] + paths = [path for pair in pairs for path in pair] + for path in paths: + if has_symlink(path): + raise ValueError(f'publication paths must not contain symlinks: {path}') + pairs = [(source.resolve(), target.resolve()) for source, target in pairs] + paths = [path for pair in pairs for path in pair] + for index, path in enumerate(paths): + if any(path == other or path.is_relative_to(other) or other.is_relative_to(path) for other in paths[:index]): + raise ValueError('publication paths must not overlap') + for source, target in pairs: + if not source.is_dir() or (target.exists() and not target.is_dir()): + raise ValueError('publication requires directories') + target.parent.mkdir(parents=True, exist_ok=True) + if source.stat().st_dev != target.parent.stat().st_dev: + raise ValueError('publication must remain on the same filesystem') + installed, backups, recovery = [], [], [] + try: + for source, target in pairs: + temporary = pathlib.Path(tempfile.mkdtemp(prefix='.publish_', dir=target.parent)) + recovery.append(temporary) + if target.exists(): + backup = temporary / 'previous' + target.rename(backup) + backups.append((backup, target)) + source.rename(target) + installed.append(target) + yield + except BaseException: + try: + for target in reversed(installed): + if target.exists(): + shutil.rmtree(target) + for backup, target in reversed(backups): + backup.rename(target) + except BaseException as error: + raise RuntimeError(f'publication rollback failed; recovery directories retained: {recovery}') from error + for temporary in recovery: + shutil.rmtree(temporary) + raise + else: + for temporary in recovery: + shutil.rmtree(temporary) diff --git a/modulus/python.py b/modulus/python.py new file mode 100644 index 0000000..3ef202b --- /dev/null +++ b/modulus/python.py @@ -0,0 +1,56 @@ +"""Optional, build-host Python dependencies and exported module execution.""" + +import hashlib +import json +import pathlib +import platform +import subprocess +import sys + +import modulus.filesystem + +GROUPS = ('assets',) + +def requirement_files(context): + return [pathlib.Path(__file__).with_name(f'{group}_requirements.txt') for group in context.tooling['python']['groups']] + +def identity(files): + data = [sys.version, str(pathlib.Path(sys.executable).resolve()), platform.system(), platform.machine(), [(path.name, path.read_text()) for path in files]] + return hashlib.sha256(json.dumps(data, sort_keys=True).encode()).hexdigest() + +def interpreter(context, *, install=False, offline=False): + files = requirement_files(context) + if not files: + return pathlib.Path(sys.executable) + root = context.common_dir / 'python' + if modulus.filesystem.has_symlink(root): + raise ValueError('build Python environment must not contain symlinks in its directory path') + executable = root / 'bin' / 'python3' + marker = root / 'requirements.sha256' + fingerprint = identity(files) + if executable.is_file() and marker.is_file() and marker.read_text().strip() == fingerprint: + return executable + if not install or offline: + raise RuntimeError('Build Python dependencies are unavailable or changed; run ./modulus.py configure with network access first') + root.parent.mkdir(parents=True, exist_ok=True) + subprocess.run([sys.executable, '-m', 'venv', '--clear', str(root)], check=True) + subprocess.run([str(executable), '-m', 'pip', 'install', '--disable-pip-version-check', *(item for path in files for item in ('-r', str(path)))], check=True) + marker.write_text(fingerprint + '\n') + return executable + +def module_root(context): + return context.tools_dir.parent if (context.tools_dir.parent / 'modulus' / 'cli.py').is_file() else context.common_dir / 'modulus' + +def command(context, arguments): + return [str(interpreter(context)), str(context.tools_dir / 'python_runner.py'), str(module_root(context)), *arguments] + +def cmake_definitions(context): + return { + 'Python3_EXECUTABLE': str(interpreter(context)), + 'MODULUS_PYTHON_RUNNER': str(context.tools_dir / 'python_runner.py'), + 'MODULUS_PYTHON_ROOT': str(module_root(context)), + 'MODULUS_PYTHON_REQUIREMENTS': ';'.join(str(path) for path in requirement_files(context)), + } + +def launch(context, arguments): + return subprocess.run(command(context, arguments), cwd=context.repo_root, check=False).returncode diff --git a/schemas/modulus.schema.json b/schemas/modulus.schema.json index 85e087f..34c0d69 100644 --- a/schemas/modulus.schema.json +++ b/schemas/modulus.schema.json @@ -134,6 +134,17 @@ "type": "object", "additionalProperties": false, "properties": { + "python": { + "type": "object", + "additionalProperties": false, + "properties": { + "groups": { + "type": "array", + "uniqueItems": true, + "items": {"enum": ["assets"]} + } + } + }, "format": { "type": "object", "additionalProperties": false, diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 23a0ace..4db1d82 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,20 +1,20 @@ -add_executable(modulus-test +add_executable(modulus_test test.cpp ) -target_include_directories(modulus-test PRIVATE +target_include_directories(modulus_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ) -target_link_libraries(modulus-test PRIVATE +target_link_libraries(modulus_test PRIVATE modulus::modulus ) -target_compile_features(modulus-test PRIVATE cxx_std_23) -target_compile_definitions(modulus-test PRIVATE +target_compile_features(modulus_test PRIVATE cxx_std_23) +target_compile_definitions(modulus_test PRIVATE $<$:MODULUS_EXPECT_DEBUG_ASSERTS> $<$:MODULUS_EXPECT_NDEBUG> $<$:MODULUS_EXPECT_NDEBUG> $<$:MODULUS_EXPECT_NDEBUG> ) -add_test(NAME modulus COMMAND modulus-test) +add_test(NAME modulus COMMAND modulus_test) set_tests_properties(modulus PROPERTIES TIMEOUT 20) add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND} diff --git a/tests/test_artifacts.py b/tests/test_artifacts.py new file mode 100644 index 0000000..c3c2299 --- /dev/null +++ b/tests/test_artifacts.py @@ -0,0 +1,56 @@ +import os +import pathlib +import tempfile +import unittest +import unittest.mock +import urllib.error + +import modulus.artifacts +import modulus.cli +import modulus.config + +class ArtifactTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = pathlib.Path(self.temporary.name) + config = modulus.config.normalize({'package': {'name': 'example', 'artifact': {'enabled': True, 'targets': ['arm_64_gcc12_release']}}}) + self.context = modulus.cli.build_context(self.root, config) + self.directory = self.root / 'build/packages' + self.directory.mkdir(parents=True) + self.archive = self.directory / modulus.artifacts.archive_name('example', 'arm_64_gcc12_release') + self.archive.write_bytes(b'package') + self.checksum = modulus.artifacts.write_checksum(self.archive) + self.enterContext(unittest.mock.patch.dict(os.environ, {'PACKAGE_REGISTRY_USER': 'test', 'PACKAGE_REGISTRY_TOKEN': 'secret'})) + def test_discovery_uses_the_same_names_as_creation(self): + self.assertEqual(self.archive.name, 'example_arm_64_gcc12_release.tar.gz') + self.assertEqual(modulus.artifacts.discover_artifacts(self.directory, 'example', ['arm_64_gcc12_release']), [self.archive, self.checksum]) + def test_missing_or_invalid_checksums_never_touch_registry(self): + with unittest.mock.patch.object(modulus.artifacts, 'request') as request: + self.archive.write_bytes(b'corrupt') + with self.assertRaisesRegex(SystemExit, 'checksum mismatch'): + modulus.artifacts.main(self.context, []) + self.checksum.unlink() + with self.assertRaisesRegex(SystemExit, 'missing artifact'): + modulus.artifacts.main(self.context, []) + request.assert_not_called() + def test_replacement_and_cleanup_are_ordered_after_validation(self): + events = [] + with unittest.mock.patch.object(modulus.artifacts, 'list_versions', return_value=['latest', 'old']), \ + unittest.mock.patch.object(modulus.artifacts, 'delete_version', side_effect=lambda *args: events.append(('delete', args[-2]))), \ + unittest.mock.patch.object(modulus.artifacts, 'upload_file', side_effect=lambda *args: events.append(('upload', args[2].name))): + self.assertEqual(modulus.artifacts.main(self.context, []), 0) + self.assertEqual(events, [('delete', 'latest'), ('upload', self.archive.name), ('upload', self.checksum.name), ('delete', 'old')]) + def test_failed_upload_does_not_delete_other_slots(self): + with unittest.mock.patch.object(modulus.artifacts, 'list_versions', return_value=['latest', 'old']), \ + unittest.mock.patch.object(modulus.artifacts, 'delete_version') as delete, \ + unittest.mock.patch.object(modulus.artifacts, 'upload_file', side_effect=SystemExit('upload failed')): + with self.assertRaisesRegex(SystemExit, 'upload failed'): + modulus.artifacts.main(self.context, []) + self.assertEqual([call.args[-2] for call in delete.call_args_list], ['latest']) + def test_request_errors_and_credentials_are_not_logged(self): + for code in (401, 500): + with unittest.mock.patch.object(modulus.artifacts.urllib.request, 'urlopen', side_effect=urllib.error.HTTPError('https://example.test', code, 'failed', {}, None)): + with self.assertRaises(SystemExit) as caught: + modulus.artifacts.request('PUT', 'https://example.test', 'secret') + self.assertNotIn('secret', str(caught.exception)) diff --git a/tests/test_clangd.py b/tests/test_clangd.py index 0af77b5..4520d7a 100644 --- a/tests/test_clangd.py +++ b/tests/test_clangd.py @@ -114,7 +114,7 @@ class clangd_test(unittest.TestCase): patch.object(subprocess, "Popen", return_value=process), contextlib.redirect_stderr(io.StringIO()): self.assertEqual(modulus.clangd.prepare(self.context), 1) self.assertFalse(cache.exists()) - self.assertEqual(list(cache.parent.glob(".headers-*")), []) + self.assertEqual(list(cache.parent.glob(".headers_*")), []) def test_cleanup_always_waits_for_child_even_if_docker_removal_times_out(self): process = MagicMock() process.wait.side_effect = [0, subprocess.TimeoutExpired("docker", 5), 0] diff --git a/tests/test_clangd_integration.py b/tests/test_clangd_integration.py index 6ab6b6e..7608b5c 100644 --- a/tests/test_clangd_integration.py +++ b/tests/test_clangd_integration.py @@ -138,7 +138,7 @@ class container_clangd_test(unittest.TestCase): session.close() def test_eof_and_termination_remove_container(self): def containers(): - return set(subprocess.check_output(["docker", "ps", "-q", "--filter", "name=modulus-clangd-"], text=True).split()) + return set(subprocess.check_output(["docker", "ps", "-q", "--filter", "name=modulus_clangd_"], text=True).split()) before = containers() for terminate in (False, True): session = lsp_session(self.roots()[0]) diff --git a/tests/test_cli.py b/tests/test_cli.py index c85edc3..1b847b7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -58,7 +58,7 @@ class upload_packages_test(unittest.TestCase): "name": "sample", }, } - return modulus.cli.build_context(repo_root or pathlib.Path(os.getcwd()), config) + return modulus.cli.build_context(repo_root or pathlib.Path(os.getcwd()), modulus.config.normalize(config)) def test_cli_import_does_not_import_conan_recipe_module(self): repo = pathlib.Path(__file__).resolve().parents[1] command = [ @@ -345,6 +345,17 @@ class upload_packages_test(unittest.TestCase): def test_conan_remote_url_uses_config_by_default(self): with unittest.mock.patch.dict(os.environ, {}, clear=True): self.assertEqual(modulus.cli.conan_remote_url(self.context()), "https://example.test/conan") + @unittest.mock.patch.object(modulus.cli, "ensure_build_image", return_value=0) + @unittest.mock.patch.object(modulus.cli, "run", return_value=0) + @unittest.mock.patch.object(modulus.cli, "in_container", return_value=False) + def test_publish_forwards_credentials_but_python_does_not(self, in_container, run, ensure_image): + context = self.context() + with unittest.mock.patch.dict(os.environ, {"PACKAGE_REGISTRY_TOKEN": "secret"}, clear=True): + modulus.cli.docker_run(context, ["tools/publish.py"]) + self.assertIn("PACKAGE_REGISTRY_TOKEN", run.call_args.args[1]) + self.assertNotIn("secret", run.call_args.args[1]) + modulus.cli.docker_run(context, ["tools/python.py", "-m", "generator"]) + self.assertNotIn("PACKAGE_REGISTRY_TOKEN", run.call_args.args[1]) def test_conan_remote_url_uses_config_in_ci(self): with unittest.mock.patch.dict( os.environ, @@ -905,7 +916,7 @@ class upload_packages_test(unittest.TestCase): with unittest.mock.patch.object(modulus.cli, "run", side_effect=fake_run) as run: status, artifacts = modulus.cli.create_artifact_package(context, context.default_target, update=False, no_remote=True) self.assertEqual(status, 0) - archive = repo / "build" / "packages" / "sample-native.tar.gz" + archive = repo / "build" / "packages" / "sample_native.tar.gz" checksum = archive.with_name(archive.name + ".sha256") self.assertEqual(artifacts, [archive, checksum]) self.assertTrue(archive.is_file()) @@ -922,58 +933,6 @@ class upload_packages_test(unittest.TestCase): cmake_command = run.call_args_list[1].args[1] self.assertIn("-DSAMPLE_WITH_WEB=ON", cmake_command) self.assertIn(f"-DCMAKE_INSTALL_PREFIX={repo / 'build' / 'build_common' / 'artifacts' / 'native' / 'stage' / 'usr'}", cmake_command) - @unittest.mock.patch.object(modulus.cli, "run", return_value=0) - @unittest.mock.patch.object(modulus.cli, "run_capture") - @unittest.mock.patch.object(pathlib.Path, "is_file", return_value=True) - def test_generic_artifact_upload_uses_package_registry_credentials(self, exists, capture, run): - context = self.context() - archive = pathlib.Path("build/packages/sample-native.tar.gz") - with unittest.mock.patch.dict( - os.environ, - { - "PACKAGE_REGISTRY_USER": "dev", - "PACKAGE_REGISTRY_TOKEN": "secret", - }, - clear=True, - ): - for response in ("204", "404"): - capture.return_value = subprocess.CompletedProcess([], 0, stdout=response, stderr="") - self.assertEqual(modulus.cli.upload_generic_artifact(context, archive), 0) - self.assertEqual(capture.call_args.args[1], ["curl", "--silent", "--show-error", "--user", "dev:secret", "--request", "DELETE", "--output", os.devnull, "--write-out", "%{http_code}", "https://example.test/api/packages/acme/generic/sample/latest/sample-native.tar.gz"]) - self.assertEqual(capture.call_args.kwargs["display_command"][4], "dev:") - self.assertEqual( - run.call_args.args[1], - [ - "curl", - "--fail", - "--silent", - "--show-error", - "--user", - "dev:secret", - "--upload-file", - archive, - "https://example.test/api/packages/acme/generic/sample/latest/sample-native.tar.gz", - ], - ) - self.assertEqual( - run.call_args.kwargs["display_command"][5], - "dev:", - ) - @unittest.mock.patch.object(modulus.cli, "package_registry_credentials", return_value=("dev", "secret")) - @unittest.mock.patch.object(pathlib.Path, "is_file", return_value=True) - @unittest.mock.patch.object(modulus.cli, "run_capture") - @unittest.mock.patch.object(modulus.cli, "run") - def test_artifact_upload_stops_if_replacement_fails(self, run, capture, exists, credentials): - for status, response in ((0, "401"), (0, "500"), (7, "000")): - with self.subTest(status=status, response=response): - capture.return_value = subprocess.CompletedProcess([], status, stdout=response, stderr="") - self.assertNotEqual(modulus.cli.upload_generic_artifact(self.context(), pathlib.Path("sample-native.tar.gz")), 0) - run.assert_not_called() - @unittest.mock.patch.object(modulus.cli, "run_capture") - def test_missing_artifact_does_not_touch_registry(self, capture): - with tempfile.TemporaryDirectory() as temporary: - self.assertEqual(modulus.cli.upload_generic_artifact(self.context(), pathlib.Path(temporary) / "missing.tar.gz"), 1) - capture.assert_not_called() @unittest.mock.patch.object(modulus.cli, "in_docker", return_value=True) @unittest.mock.patch.object(modulus.cli, "create_artifact_package", return_value=(0, [])) def test_artifact_command_needs_no_release_argument(self, create, docker): diff --git a/tests/test_export.py b/tests/test_export.py index 70aed83..690f4e9 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -47,11 +47,11 @@ class modulus_export_test(unittest.TestCase): (package / "tools" / "lsp.py").write_text("#!/usr/bin/env python3\n", encoding="utf-8") (package / "tools" / "run.py").write_text("#!/usr/bin/env python3\n", encoding="utf-8") (package / "tools" / "format.py").write_text("#!/usr/bin/env python3\n", encoding="utf-8") - (package / "tools" / "docker" / "format-requirements.txt").write_text("autopep8==2.3.2\n", encoding="utf-8") + (package / "tools" / "docker" / "format_requirements.txt").write_text("autopep8==2.3.2\n", encoding="utf-8") (package / "tools" / "docker" / "Dockerfile").write_text("FROM scratch\n", encoding="utf-8") (package / "include" / "modulus" / "types.hpp").write_text("", encoding="utf-8") (package / "tests" / "test.cpp").write_text("", encoding="utf-8") - (package / "lib" / "cmake" / "modulus" / "modulus-config.cmake").write_text("", encoding="utf-8") + (package / "lib" / "cmake" / "modulus" / "modulusConfig.cmake").write_text("", encoding="utf-8") (package / "share" / "modulus" / "duplicate.py").write_text("", encoding="utf-8") (package / "bin" / "modulus-tool").write_text("", encoding="utf-8") def test_boost_requirement_options_enable_configured_components(self): @@ -205,8 +205,8 @@ class modulus_export_test(unittest.TestCase): self.assertEqual(config["tooling"]["targets"]["x86_64_gcc12_release"]["profile"], "conan/profiles/x86_64_gcc12_release") def test_docker_defaults_use_shared_modulus_marker(self): config = modulus.config.normalize(self.config()) - self.assertEqual(config["tooling"]["docker"]["image"], "modulus-build:dev") - self.assertEqual(config["tooling"]["docker"]["marker"], "/opt/modulus-build-image") + self.assertEqual(config["tooling"]["docker"]["image"], "modulus_build:dev") + self.assertEqual(config["tooling"]["docker"]["marker"], "/opt/modulus_build_image") self.assertNotIn("apt_architectures", config["tooling"]["docker"]) self.assertNotIn("apt_packages", config["tooling"]["docker"]) def test_export_project_copies_runtime_and_exported_tools(self): @@ -233,7 +233,7 @@ class modulus_export_test(unittest.TestCase): self.assertTrue((runtime / "modulus" / "formatting.py").is_file()) self.assertTrue((runtime / "modulus" / "format_layout.py").is_file()) self.assertTrue((repo / "build/build_tools/format.py").is_file()) - self.assertTrue((repo / "build/build_tools/docker/format-requirements.txt").is_file()) + self.assertTrue((repo / "build/build_tools/docker/format_requirements.txt").is_file()) self.assertFalse((runtime / "CMakeLists.txt").exists()) self.assertFalse((runtime / "README.md").exists()) self.assertFalse((runtime / "conanfile.py").exists()) diff --git a/tests/test_filesystem.py b/tests/test_filesystem.py new file mode 100644 index 0000000..fac842c --- /dev/null +++ b/tests/test_filesystem.py @@ -0,0 +1,58 @@ +import pathlib +import tempfile +import unittest +import unittest.mock + +import modulus.filesystem + +class PublicationTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = pathlib.Path(self.temporary.name) + self.source, self.target = self.root / 'new', self.root / 'installed' + for path, text in ((self.source, 'new'), (self.target, 'old')): + path.mkdir() + (path / 'content').write_text(text) + def test_publish_replaces_complete_directory(self): + with modulus.filesystem.publish_directories([(self.source, self.target)]): + self.assertEqual((self.target / 'content').read_text(), 'new') + self.assertFalse(self.source.exists()) + self.assertEqual(list(self.root.iterdir()), [self.target]) + def test_failure_restores_previous_directory(self): + with self.assertRaisesRegex(RuntimeError, 'failed'): + with modulus.filesystem.publish_directories([(self.source, self.target)]): + raise RuntimeError('failed') + self.assertEqual((self.target / 'content').read_text(), 'old') + self.assertFalse(list(self.root.glob('.publish_*'))) + def test_failed_second_replacement_rolls_back_first(self): + second = self.root / 'second' + second.mkdir() + rename = pathlib.Path.rename + def fail(path, destination): + if path == second: + raise OSError('failed') + return rename(path, destination) + with unittest.mock.patch.object(pathlib.Path, 'rename', fail), self.assertRaises(OSError): + with modulus.filesystem.publish_directories([(self.source, self.target), (second, self.root / 'last')]): + pass + self.assertEqual((self.target / 'content').read_text(), 'old') + def test_overlaps_and_symlinks_rejected_before_replacement(self): + link = self.root / 'link' + link.symlink_to(self.target, target_is_directory=True) + for target in (self.source, self.source / 'child', self.source / '..' / 'new', self.root, link, link / 'child'): + with self.subTest(target=target), self.assertRaises(ValueError): + with modulus.filesystem.publish_directories([(self.source, target)]): + pass + self.assertEqual((self.target / 'content').read_text(), 'old') + def test_failed_rollback_retains_the_previous_directory_for_recovery(self): + rename = pathlib.Path.rename + def fail(path, destination): + if path.name == 'previous': + raise OSError('restore failed') + return rename(path, destination) + with unittest.mock.patch.object(pathlib.Path, 'rename', fail), self.assertRaisesRegex(RuntimeError, 'recovery directories retained'): + with modulus.filesystem.publish_directories([(self.source, self.target)]): + raise RuntimeError('publication failed') + backup = next(self.root.glob('.publish_*/previous/content')) + self.assertEqual(backup.read_text(), 'old') diff --git a/tests/test_python.py b/tests/test_python.py new file mode 100644 index 0000000..2d4d16f --- /dev/null +++ b/tests/test_python.py @@ -0,0 +1,59 @@ +import pathlib +import subprocess +import sys +import tempfile +import unittest +import unittest.mock + +import modulus.cli +import modulus.config +import modulus.python + +class PythonTests(unittest.TestCase): + def context(self, root, groups): + return modulus.cli.build_context(root, modulus.config.normalize({'package': {'name': 'example'}, 'tooling': {'python': {'groups': groups}}})) + def test_optional_packages_are_not_needed_by_default(self): + with tempfile.TemporaryDirectory() as temporary, unittest.mock.patch.object(subprocess, 'run') as run: + context = self.context(pathlib.Path(temporary), []) + self.assertEqual(modulus.python.interpreter(context, install=True), pathlib.Path(sys.executable)) + run.assert_not_called() + def test_conan_reuses_the_project_environment_and_preserves_offline_mode(self): + with tempfile.TemporaryDirectory() as temporary: + context = self.context(pathlib.Path(temporary), ['assets']) + environment = modulus.cli.subprocess_environment(context, ['conan', 'install', '.', '--no-remote']) + self.assertEqual(environment['MODULUS_PYTHON_COMMON_DIR'], str(context.common_dir)) + self.assertEqual(environment['MODULUS_PYTHON_OFFLINE'], '1') + environment = modulus.cli.subprocess_environment(context, ['conan', 'install', '.']) + self.assertEqual(environment['MODULUS_PYTHON_OFFLINE'], '0') + def test_missing_dependencies_require_configure_and_never_download_during_build(self): + with tempfile.TemporaryDirectory() as temporary, unittest.mock.patch.object(subprocess, 'run') as run: + context = self.context(pathlib.Path(temporary), ['assets']) + for options in ({}, {'install': True, 'offline': True}): + with self.assertRaisesRegex(RuntimeError, 'configure'): + modulus.python.interpreter(context, **options) + run.assert_not_called() + def test_unknown_or_duplicate_groups_are_rejected(self): + for groups in (['unknown'], ['assets', 'assets'], 'assets'): + with self.assertRaises(modulus.config.config_error): + self.context(pathlib.Path('/tmp'), groups) + def test_matching_environment_is_reused_and_changed_pins_require_configure(self): + with tempfile.TemporaryDirectory() as temporary, unittest.mock.patch.object(subprocess, 'run') as run: + context = self.context(pathlib.Path(temporary), ['assets']) + executable = context.common_dir / 'python/bin/python3' + executable.parent.mkdir(parents=True) + executable.touch() + marker = executable.parents[1] / 'requirements.sha256' + marker.write_text(modulus.python.identity(modulus.python.requirement_files(context))) + self.assertEqual(modulus.python.interpreter(context, offline=True), executable) + with unittest.mock.patch.object(modulus.python, 'identity', return_value='changed'): + with self.assertRaisesRegex(RuntimeError, 'configure'): + modulus.python.interpreter(context) + run.assert_not_called() + def test_runner_imports_exported_package_even_with_bootstrap_in_working_directory(self): + source = pathlib.Path(__file__).resolve().parents[1] + with tempfile.TemporaryDirectory() as temporary: + root = pathlib.Path(temporary) + (root / 'modulus.py').write_text("raise AssertionError('bootstrap shadowed package')") + result = subprocess.run([sys.executable, str(source / 'tools/python_runner.py'), str(source), '-c', 'import modulus.filesystem; print("loaded")'], cwd=root, text=True, capture_output=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), 'loaded') diff --git a/tools/docker/Dockerfile b/tools/docker/Dockerfile index 09503f5..918b6b3 100755 --- a/tools/docker/Dockerfile +++ b/tools/docker/Dockerfile @@ -37,18 +37,18 @@ RUN apt-get update \ ENV CC=/usr/bin/gcc-12 ENV CXX=/usr/bin/g++-12 -RUN python3 -m venv /opt/modulus-python +RUN python3 -m venv /opt/modulus_python -ENV PATH=/opt/modulus-python/bin:${PATH} +ENV PATH=/opt/modulus_python/bin:${PATH} RUN python -m pip install --no-cache-dir \ conan==2.26.2 # The path is shared by source builds and exported consumer tooling. ARG MODULUS_TOOLS=tools -COPY ${MODULUS_TOOLS}/docker/format-requirements.txt /tmp/modulus-format-requirements.txt -RUN python -m pip install --no-cache-dir -r /tmp/modulus-format-requirements.txt +COPY ${MODULUS_TOOLS}/docker/format_requirements.txt /tmp/modulus_format_requirements.txt +RUN python -m pip install --no-cache-dir -r /tmp/modulus_format_requirements.txt WORKDIR /workspace -RUN mkdir -p /opt && touch /opt/modulus-build-image +RUN mkdir -p /opt && touch /opt/modulus_build_image diff --git a/tools/docker/format-requirements.txt b/tools/docker/format_requirements.txt similarity index 100% rename from tools/docker/format-requirements.txt rename to tools/docker/format_requirements.txt diff --git a/tools/publish.py b/tools/publish.py new file mode 100755 index 0000000..3eed1ba --- /dev/null +++ b/tools/publish.py @@ -0,0 +1,4 @@ +#!/usr/bin/env python3 +import modulus_tool + +modulus_tool.run('publish') diff --git a/tools/python.py b/tools/python.py new file mode 100755 index 0000000..6f2f3b6 --- /dev/null +++ b/tools/python.py @@ -0,0 +1,4 @@ +#!/usr/bin/env python3 +import modulus_tool + +modulus_tool.run('python') diff --git a/tools/python_runner.py b/tools/python_runner.py new file mode 100644 index 0000000..89791cc --- /dev/null +++ b/tools/python_runner.py @@ -0,0 +1,23 @@ +"""Run build Python with the exported Modulus package ahead of the bootstrap.""" + +import pathlib +import runpy +import sys + +def main(): + root, *arguments = sys.argv[1:] + sys.path[:0] = [str(pathlib.Path(root).resolve()), str(pathlib.Path.cwd())] + if not arguments: + raise SystemExit('expected -m MODULE, -c CODE, or a Python script') + if arguments[0] == '-m' and len(arguments) >= 2: + sys.argv = arguments[1:] + runpy.run_module(arguments[1], run_name='__main__', alter_sys=True) + elif arguments[0] == '-c' and len(arguments) >= 2: + sys.argv = ['-c', *arguments[2:]] + exec(compile(arguments[1], '', 'exec'), {'__name__': '__main__'}) + else: + sys.argv = arguments + runpy.run_path(arguments[0], run_name='__main__') + +if __name__ == '__main__': + main()