From 1612e686501dcf18a9a591014bed9f635f32dc52 Mon Sep 17 00:00:00 2001 From: khodowany Date: Fri, 25 Sep 2026 11:36:46 -0500 Subject: [PATCH] Changes --- .gitea/workflows/ci.yml | 6 +-- README.md | 6 ++- modulus/cli.py | 46 ++++++++++------------ modulus/config.py | 2 +- tests/test_clangd.py | 2 +- tests/test_cli.py | 48 +++++++++++++++++------ tests/test_export.py | 13 ++++++ tools/conan/profiles/arm_64_gcc12_debug | 8 +++- tools/conan/profiles/arm_64_gcc12_release | 8 +++- tools/docker/Dockerfile | 4 +- 10 files changed, 94 insertions(+), 49 deletions(-) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 2a3f56e..861c96a 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -29,9 +29,9 @@ jobs: ca-certificates \ cmake \ g++-12 \ - g++-aarch64-linux-gnu \ + g++-12-aarch64-linux-gnu \ gcc-12 \ - gcc-aarch64-linux-gnu \ + gcc-12-aarch64-linux-gnu \ git \ ninja-build \ pkg-config \ @@ -44,7 +44,7 @@ jobs: uses: actions/cache@v4 with: path: build/build_common/conan_cache - key: conan-${{ runner.os }}-${{ hashFiles('conanfile.py', 'modulus.json', 'tools/conan/profiles/*') }} + key: conan-${{ runner.os }}-${{ hashFiles('conanfile.py', 'modulus.json', 'tools/conan/profiles/*', '.gitea/workflows/ci.yml') }} restore-keys: conan-${{ runner.os }}- - name: Configure native release run: ./tools/configure.py --target x86_64_gcc12_release --update diff --git a/README.md b/README.md index 5283db1..1b6260c 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,8 @@ MODULUS_DIRECT=1 ./tools/configure.py --target x86_64_gcc12_release Outside that environment, downstream repos use `python3 modulus.py ` to fetch the public `modulus/latest` package from the khodo Conan registry and export the tool wrappers into `build/build_tools/`. +GCC 12 profiles select versioned native and ARM compiler executables. Install `gcc-12-aarch64-linux-gnu` and `g++-12-aarch64-linux-gnu` for ARM builds; the unversioned packages can select a different compiler on newer distributions. ARM profiles include the selected compiler executables in the internal libraries' Conan package IDs so previously mislabeled binaries are not reused. Third-party package IDs are unchanged. Publish Modulus before rerunning the sibling libraries' package workflows so they export the corrected profiles. + `build` automatically configures a target when CMake metadata is missing. `configure --update` refreshes dependencies and, in consumer repos, the exported Python tooling. Failed downloads/exports leave the previous tooling in place. `--no-remote` uses only the project-local cache, including initial bootstrap; it never downloads packages. Each consumer's tracked `modulus.py` is copied unchanged from `modulus/bootstrap.py`. Update that canonical source and synchronize consumer copies together. Projects can select a tracked build recipe using `tooling.docker.dockerfile` and list additional build-context inputs in `tooling.docker.inputs`. Those files, the Dockerfile, and the shared formatting requirements determine the image fingerprint used by builds and clangd. Paths are project-relative. `tooling.docker.socket_commands` optionally permits host Docker socket access for `build` and/or `package`; the default is no socket access. Editor, configure, and test commands never receive the socket. @@ -89,7 +91,7 @@ Server settings are separate from `.clangd`'s parsing/diagnostic settings: "clangd": { "executable": "/usr/bin/clangd-19", "arguments": ["--background-index", "--completion-style=detailed", "--header-insertion=never"], - "query_drivers": ["/usr/bin/gcc-12", "/usr/bin/g++-12", "/usr/bin/aarch64-linux-gnu-gcc", "/usr/bin/aarch64-linux-gnu-g++"], + "query_drivers": ["/usr/bin/gcc-12", "/usr/bin/g++-12", "/usr/bin/aarch64-linux-gnu-gcc-12", "/usr/bin/aarch64-linux-gnu-g++-12"], "header_roots": [] } } @@ -131,6 +133,8 @@ Generic application artifacts default to: https://git.thyme-labs.com/api/packages/khodo/generic ``` +Application archives have stable names, `-.tar.gz`, with matching SHA-256 files. Each build replaces the corresponding local artifact. Generic uploads use the registry's required `latest` slot; application packaging has no release number or version argument. Conan library references and external dependency pins remain unchanged. + Configure/build/test do not require package credentials for public packages. Read operations log out only the configured public remote in the project-local Conan home and strip credential environment variables, including native Conan username/password variables. Authentication failures are noninteractive errors, not password prompts. Package contents and global Conan configuration are untouched. Credentials are validated only for publishing, redacted in command logs, and the remote is logged out in a `finally` cleanup after every upload attempt so CI caches do not retain the session. ## License diff --git a/modulus/cli.py b/modulus/cli.py index 2f0828b..6fc5205 100644 --- a/modulus/cli.py +++ b/modulus/cli.py @@ -841,21 +841,6 @@ def package_artifact_targets(context, names, all_targets): return [context.target(name) for name in configured] return [context.default_target] -def git_version(context): - completed = run_capture( - context, - [ - "git", - "rev-parse", - "--short=12", - "HEAD", - ], - ) - if completed.returncode != 0: - return "dev" - version = completed.stdout.strip() - return version if version else "dev" - def package_artifact_build_root(context, target): return context.common_dir / "artifacts" / target.name @@ -882,7 +867,7 @@ def write_checksum(archive): checksum.write_text(f"{digest.hexdigest()} {archive.name}\n", encoding="utf-8") return checksum -def create_artifact_package(context, target, version, update, no_remote=False): +def create_artifact_package(context, target, update, no_remote=False): root = package_artifact_build_root(context, target) build_dir = root / "build" stage_root = root / "stage" @@ -956,7 +941,7 @@ def create_artifact_package(context, target, version, update, no_remote=False): return status, [] dist_dir = package_artifact_dist_dir(context) dist_dir.mkdir(parents=True, exist_ok=True) - archive = dist_dir / f"{package_name(context)}-{version}-{target.name}.tar.gz" + archive = dist_dir / f"{package_name(context)}-{target.name}.tar.gz" if archive.exists(): archive.unlink() create_archive(stage_root, archive) @@ -965,19 +950,30 @@ def create_artifact_package(context, target, version, update, no_remote=False): print(f"Created {checksum}") return 0, [archive, checksum] -def generic_package_upload_url(context, version, path): +def generic_package_upload_url(context, path): base = str(context.artifact["registry_url"]).rstrip("/") package = urllib.parse.quote(package_name(context), safe="") - version = urllib.parse.quote(version, safe="") filename = urllib.parse.quote(path.name, safe="") - return f"{base}/{package}/{version}/{filename}" + return f"{base}/{package}/latest/{filename}" -def upload_generic_artifact(context, version, path): +def upload_generic_artifact(context, path): + if not path.is_file(): + print(f"missing artifact: {path}", file=sys.stderr) + return 1 credentials = package_registry_credentials() if credentials is None: return 1 user, token = credentials - url = generic_package_upload_url(context, version, path) + url = generic_package_upload_url(context, path) + command = ["curl", "--silent", "--show-error", "--user", f"{user}:{token}", "--request", "DELETE", "--output", os.devnull, "--write-out", "%{http_code}", url] + visible = [*command] + visible[4] = f"{user}:" + deleted = run_capture(context, command, display_command=visible) + if deleted.returncode != 0: + return deleted.returncode + if deleted.stdout.strip() not in {"204", "404"}: + print(f"cannot replace artifact {path.name}: HTTP {deleted.stdout.strip()}", file=sys.stderr) + return 1 return run( context, [ @@ -1024,7 +1020,6 @@ def package_artifacts(context, argv): parser.add_argument("--all", action="store_true", help="Package all supported targets.") parser.add_argument("--update", action="store_true", help="Check Conan remotes for newer dependency revisions.") parser.add_argument("--no-remote", action="store_true", help="Do not query Conan remotes or upload package artifacts.") - parser.add_argument("--version", default=None, help="Artifact version. Defaults to the current git commit.") args = parser.parse_args(argv) context.conan_cache_dir.mkdir(parents=True, exist_ok=True) if should_upload_artifacts(context) and not args.no_remote and package_registry_credentials() is None: @@ -1033,16 +1028,15 @@ def package_artifacts(context, argv): status = configure_conan_remote(context) if status != 0: return status - version = args.version or git_version(context) artifacts = [] for target in package_artifact_targets(context, args.target, args.all): - status, created = create_artifact_package(context, target, version, args.update, no_remote=args.no_remote) + status, created = create_artifact_package(context, target, args.update, no_remote=args.no_remote) if status != 0: return status artifacts.extend(created) if should_upload_artifacts(context) and not args.no_remote: for artifact in artifacts: - status = upload_generic_artifact(context, version, artifact) + status = upload_generic_artifact(context, artifact) if status != 0: return status return 0 diff --git a/modulus/config.py b/modulus/config.py index fe1656b..a8ced6e 100644 --- a/modulus/config.py +++ b/modulus/config.py @@ -107,7 +107,7 @@ clangd_defaults = { "arguments": ["--background-index", "--completion-style=detailed", "--header-insertion=never"], "query_drivers": [ "/usr/bin/gcc-12", "/usr/bin/g++-12", - "/usr/bin/aarch64-linux-gnu-gcc", "/usr/bin/aarch64-linux-gnu-g++", + "/usr/bin/aarch64-linux-gnu-gcc-12", "/usr/bin/aarch64-linux-gnu-g++-12", ], "header_roots": [], } diff --git a/tests/test_clangd.py b/tests/test_clangd.py index 08cc8e1..0af77b5 100644 --- a/tests/test_clangd.py +++ b/tests/test_clangd.py @@ -66,7 +66,7 @@ class clangd_test(unittest.TestCase): self.assertNotIn(unwanted, " ".join(command)) volumes = [command[index + 1] for index, argument in enumerate(command) if argument == "--volume"] self.assertFalse(any(volume.startswith(str(pathlib.Path.home()) + ":") for volume in volumes)) - self.assertIn("--query-driver=/usr/bin/gcc-12,/usr/bin/g++-12,/usr/bin/aarch64-linux-gnu-gcc,/usr/bin/aarch64-linux-gnu-g++", command) + self.assertIn("--query-driver=/usr/bin/gcc-12,/usr/bin/g++-12,/usr/bin/aarch64-linux-gnu-gcc-12,/usr/bin/aarch64-linux-gnu-g++-12", command) def test_startup_failure_does_not_prepare_and_keeps_stdout_clean(self): output, errors = io.StringIO(), io.StringIO() with patch.object(modulus.clangd, "current_image", return_value=None), patch.object(modulus.clangd, "prepare") as prepare, \ diff --git a/tests/test_cli.py b/tests/test_cli.py index fb60975..c85edc3 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -903,15 +903,9 @@ class upload_packages_test(unittest.TestCase): (install_root / "bin" / "sample").write_text("binary\n", encoding="utf-8") return 0 with unittest.mock.patch.object(modulus.cli, "run", side_effect=fake_run) as run: - status, artifacts = modulus.cli.create_artifact_package( - context, - context.default_target, - "abc123", - update=False, - no_remote=True, - ) + status, artifacts = modulus.cli.create_artifact_package(context, context.default_target, update=False, no_remote=True) self.assertEqual(status, 0) - archive = repo / "build" / "packages" / "sample-abc123-native.tar.gz" + archive = repo / "build" / "packages" / "sample-native.tar.gz" checksum = archive.with_name(archive.name + ".sha256") self.assertEqual(artifacts, [archive, checksum]) self.assertTrue(archive.is_file()) @@ -929,9 +923,11 @@ class upload_packages_test(unittest.TestCase): self.assertIn("-DSAMPLE_WITH_WEB=ON", cmake_command) self.assertIn(f"-DCMAKE_INSTALL_PREFIX={repo / 'build' / 'build_common' / 'artifacts' / 'native' / 'stage' / 'usr'}", cmake_command) @unittest.mock.patch.object(modulus.cli, "run", return_value=0) - def test_generic_artifact_upload_uses_package_registry_credentials(self, run): + @unittest.mock.patch.object(modulus.cli, "run_capture") + @unittest.mock.patch.object(pathlib.Path, "is_file", return_value=True) + def test_generic_artifact_upload_uses_package_registry_credentials(self, exists, capture, run): context = self.context() - archive = pathlib.Path("build/packages/sample-abc123-native.tar.gz") + archive = pathlib.Path("build/packages/sample-native.tar.gz") with unittest.mock.patch.dict( os.environ, { @@ -940,8 +936,11 @@ class upload_packages_test(unittest.TestCase): }, clear=True, ): - status = modulus.cli.upload_generic_artifact(context, "abc123", archive) - self.assertEqual(status, 0) + for response in ("204", "404"): + capture.return_value = subprocess.CompletedProcess([], 0, stdout=response, stderr="") + self.assertEqual(modulus.cli.upload_generic_artifact(context, archive), 0) + self.assertEqual(capture.call_args.args[1], ["curl", "--silent", "--show-error", "--user", "dev:secret", "--request", "DELETE", "--output", os.devnull, "--write-out", "%{http_code}", "https://example.test/api/packages/acme/generic/sample/latest/sample-native.tar.gz"]) + self.assertEqual(capture.call_args.kwargs["display_command"][4], "dev:") self.assertEqual( run.call_args.args[1], [ @@ -953,13 +952,36 @@ class upload_packages_test(unittest.TestCase): "dev:secret", "--upload-file", archive, - "https://example.test/api/packages/acme/generic/sample/abc123/sample-abc123-native.tar.gz", + "https://example.test/api/packages/acme/generic/sample/latest/sample-native.tar.gz", ], ) self.assertEqual( run.call_args.kwargs["display_command"][5], "dev:", ) + @unittest.mock.patch.object(modulus.cli, "package_registry_credentials", return_value=("dev", "secret")) + @unittest.mock.patch.object(pathlib.Path, "is_file", return_value=True) + @unittest.mock.patch.object(modulus.cli, "run_capture") + @unittest.mock.patch.object(modulus.cli, "run") + def test_artifact_upload_stops_if_replacement_fails(self, run, capture, exists, credentials): + for status, response in ((0, "401"), (0, "500"), (7, "000")): + with self.subTest(status=status, response=response): + capture.return_value = subprocess.CompletedProcess([], status, stdout=response, stderr="") + self.assertNotEqual(modulus.cli.upload_generic_artifact(self.context(), pathlib.Path("sample-native.tar.gz")), 0) + run.assert_not_called() + @unittest.mock.patch.object(modulus.cli, "run_capture") + def test_missing_artifact_does_not_touch_registry(self, capture): + with tempfile.TemporaryDirectory() as temporary: + self.assertEqual(modulus.cli.upload_generic_artifact(self.context(), pathlib.Path(temporary) / "missing.tar.gz"), 1) + capture.assert_not_called() + @unittest.mock.patch.object(modulus.cli, "in_docker", return_value=True) + @unittest.mock.patch.object(modulus.cli, "create_artifact_package", return_value=(0, [])) + def test_artifact_command_needs_no_release_argument(self, create, docker): + with tempfile.TemporaryDirectory() as temporary: + context = self.context(repo_root=pathlib.Path(temporary)) + context.package["artifact"]["enabled"] = True + self.assertEqual(modulus.cli.package_artifacts(context, ["--no-remote"]), 0) + create.assert_called_once_with(context, context.default_target, False, no_remote=True) if __name__ == "__main__": unittest.main() diff --git a/tests/test_export.py b/tests/test_export.py index 8276569..70aed83 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -82,6 +82,19 @@ class modulus_export_test(unittest.TestCase): self.assertFalse(config["tooling"]["run"]["enabled"]) self.assertFalse(config["tooling"]["lsp"]["enabled"]) self.assertEqual(config["tooling"]["lsp"]["compile_databases"], []) + def test_arm_profiles_pin_compilers_and_separate_legacy_internal_binaries(self): + root = pathlib.Path(__file__).resolve().parents[1] + for build_type in ("debug", "release"): + profile = (root / "tools/conan/profiles" / f"arm_64_gcc12_{build_type}").read_text() + self.assertIn('compiler.version=12', profile) + self.assertIn('tools.build:compiler_executables={"c": "aarch64-linux-gnu-gcc-12", "cpp": "aarch64-linux-gnu-g++-12"}', profile) + for name in ("modulus", "ligo", "noto", "luma", "sonus"): + self.assertIn(f'{name}/*:tools.info.package_id:confs=["tools.build:compiler_executables"]', profile) + for path in (".gitea/workflows/ci.yml", "tools/docker/Dockerfile"): + source = (root / path).read_text() + for compiler in ("gcc", "g++"): + self.assertIn(f"{compiler}-12-aarch64-linux-gnu", source) + self.assertNotIn(f"{compiler}-aarch64-linux-gnu", source) def test_product_artifact_and_run_config_are_validated(self): config = self.config() config["package"]["artifact"] = { diff --git a/tools/conan/profiles/arm_64_gcc12_debug b/tools/conan/profiles/arm_64_gcc12_debug index a1e466d..25f92df 100755 --- a/tools/conan/profiles/arm_64_gcc12_debug +++ b/tools/conan/profiles/arm_64_gcc12_debug @@ -11,4 +11,10 @@ build_type=Debug [conf] tools.cmake.cmaketoolchain:generator=Ninja -tools.build:compiler_executables={"c": "aarch64-linux-gnu-gcc", "cpp": "aarch64-linux-gnu-g++"} +tools.build:compiler_executables={"c": "aarch64-linux-gnu-gcc-12", "cpp": "aarch64-linux-gnu-g++-12"} +# Exclude older internal binaries published with an unversioned cross compiler. +modulus/*:tools.info.package_id:confs=["tools.build:compiler_executables"] +ligo/*:tools.info.package_id:confs=["tools.build:compiler_executables"] +noto/*:tools.info.package_id:confs=["tools.build:compiler_executables"] +luma/*:tools.info.package_id:confs=["tools.build:compiler_executables"] +sonus/*:tools.info.package_id:confs=["tools.build:compiler_executables"] diff --git a/tools/conan/profiles/arm_64_gcc12_release b/tools/conan/profiles/arm_64_gcc12_release index c414913..f885834 100755 --- a/tools/conan/profiles/arm_64_gcc12_release +++ b/tools/conan/profiles/arm_64_gcc12_release @@ -11,4 +11,10 @@ build_type=Release [conf] tools.cmake.cmaketoolchain:generator=Ninja -tools.build:compiler_executables={"c": "aarch64-linux-gnu-gcc", "cpp": "aarch64-linux-gnu-g++"} +tools.build:compiler_executables={"c": "aarch64-linux-gnu-gcc-12", "cpp": "aarch64-linux-gnu-g++-12"} +# Exclude older internal binaries published with an unversioned cross compiler. +modulus/*:tools.info.package_id:confs=["tools.build:compiler_executables"] +ligo/*:tools.info.package_id:confs=["tools.build:compiler_executables"] +noto/*:tools.info.package_id:confs=["tools.build:compiler_executables"] +luma/*:tools.info.package_id:confs=["tools.build:compiler_executables"] +sonus/*:tools.info.package_id:confs=["tools.build:compiler_executables"] diff --git a/tools/docker/Dockerfile b/tools/docker/Dockerfile index 7e0aef4..09503f5 100755 --- a/tools/docker/Dockerfile +++ b/tools/docker/Dockerfile @@ -13,9 +13,9 @@ RUN apt-get update \ cmake \ curl \ g++-12 \ - g++-aarch64-linux-gnu \ + g++-12-aarch64-linux-gnu \ gcc-12 \ - gcc-aarch64-linux-gnu \ + gcc-12-aarch64-linux-gnu \ git \ libdrm-dev \ libdrm-dev:arm64 \