Initial public khodo snapshot

modulus.py

@@ -0,0 +1,367 @@
+#!/usr/bin/env python3
+import json
+import os
+import pathlib
+import shlex
+import subprocess
+import sys
+
+PACKAGE_REF = "modulus/latest"
+CONAN_VERSION = "2.26.2"
+FETCH_IMAGE = "modulus-fetch:dev"
+COMMANDS = (
+  "configure",
+  "build",
+  "test",
+  "package",
+  "lsp",
+)
+COMMON_DIR = pathlib.Path("build") / "build_common"
+TOOLS_DIR = pathlib.Path("build") / "build_tools"
+RUNTIME_DIR = COMMON_DIR / "modulus"
+CONAN_HOME_DIR = COMMON_DIR / "conan_cache"
+DEFAULT_REMOTE_NAME = "khodo"
+DEFAULT_REMOTE_URL = "https://git.thyme-labs.com/api/packages/khodo/conan"
+DEFAULT_DOCKER_MARKER = "/opt/modulus-build-image"
+
+def repo_root():
+  root = pathlib.Path(__file__).resolve().parent
+  if not (root / "modulus.json").is_file():
+    raise SystemExit("modulus.py must be run from a Modulus-managed repository root")
+  return root
+
+def read_json(path):
+  try:
+    data = json.loads(path.read_text(encoding="utf-8"))
+  except (OSError, json.JSONDecodeError) as error:
+    raise SystemExit(f"failed to read {path}: {error}") from error
+  if not isinstance(data, dict):
+    raise SystemExit(f"{path} must contain a JSON object")
+  return data
+
+def remote_config(root):
+  config = read_json(root / "modulus.json")
+  remote = config.get(
+    "tooling",
+    {},
+  ).get("remote", {})
+  if not isinstance(remote, dict):
+    remote = {}
+  return (
+    str(remote.get("name") or DEFAULT_REMOTE_NAME),
+    str(remote.get("url") or DEFAULT_REMOTE_URL),
+  )
+
+def runtime_ready(root):
+  return (
+    (root / TOOLS_DIR / "configure.py").is_file()
+    and (root / TOOLS_DIR / "lsp.py").is_file()
+    and (root / RUNTIME_DIR / "modulus" / "cli.py").is_file()
+    and (root / RUNTIME_DIR / "modulus" / "export.py").is_file()
+  )
+
+def optional_credentials():
+  user = os.environ.get("CONAN_USER")
+  token = os.environ.get("CONAN_TOKEN")
+  if bool(user) != bool(token):
+    raise SystemExit("CONAN_USER and CONAN_TOKEN must either both be set or both be unset")
+  return user, token
+
+def env_flag(name):
+  return os.environ.get(name, "").lower() in {"1", "true", "yes", "on"}
+
+def docker_marker(root):
+  config = read_json(root / "modulus.json")
+  marker = config.get(
+    "tooling",
+    {},
+  ).get("docker", {}).get("marker", DEFAULT_DOCKER_MARKER)
+  return pathlib.Path(str(marker))
+
+def direct_mode(root):
+  return env_flag("MODULUS_DIRECT") or docker_marker(root).exists()
+
+def run(command, input_text=None, capture=False, display_command=None, env=None, check=True):
+  visible = display_command if display_command is not None else command
+  print("+", " ".join(shlex.quote(str(part)) for part in visible))
+  completed = subprocess.run(
+    [str(part) for part in command],
+    input=input_text,
+    stdout=subprocess.PIPE if capture else None,
+    stderr=subprocess.PIPE if capture else None,
+    text=True,
+    check=False,
+    env=env,
+  )
+  if check and completed.returncode != 0:
+    if capture and completed.stdout:
+      print(completed.stdout, end="")
+    if capture and completed.stderr:
+      print(completed.stderr, end="", file=sys.stderr)
+    raise SystemExit(completed.returncode)
+  return completed
+
+def fetch_dockerfile():
+  return f"""FROM debian:bookworm-slim
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update \\
+  && apt-get install -y --no-install-recommends \\
+    ca-certificates \\
+    python3 \\
+    python3-venv \\
+  && rm -rf /var/lib/apt/lists/*
+
+RUN python3 -m venv /opt/modulus-python
+
+ENV PATH=/opt/modulus-python/bin:$PATH
+
+RUN python -m pip install --no-cache-dir conan=={CONAN_VERSION}
+
+WORKDIR /workspace
+"""
+
+def ensure_fetch_image():
+  inspected = subprocess.run(
+    [
+      "docker",
+      "image",
+      "inspect",
+      FETCH_IMAGE,
+    ],
+    stdout=subprocess.DEVNULL,
+    stderr=subprocess.DEVNULL,
+    check=False,
+  )
+  if inspected.returncode == 0:
+    return
+  run(
+    [
+      "docker",
+      "build",
+      "--tag",
+      FETCH_IMAGE,
+      "-",
+    ],
+    input_text=fetch_dockerfile(),
+  )
+
+def in_container():
+  return pathlib.Path("/.dockerenv").exists()
+
+def docker_workspace_args(root):
+  if in_container():
+    container = os.environ.get("HOSTNAME", "").strip()
+    if not container:
+      raise SystemExit("HOSTNAME is required to share the CI workspace with the Modulus fetch container")
+    return [
+      "--volumes-from",
+      container,
+    ]
+  return [
+    "--volume",
+    f"{root}:{root}",
+  ]
+
+def fetch_env(root):
+  env = os.environ.copy()
+  env["CONAN_HOME"] = str(root / CONAN_HOME_DIR)
+  return env
+
+def run_fetch(root, command, capture=False, display_command=None, check=True):
+  if direct_mode(root):
+    return run(command, capture=capture, display_command=display_command, env=fetch_env(root), check=check)
+  ensure_fetch_image()
+  docker_command = [
+    "docker",
+    "run",
+    "--rm",
+    "--interactive",
+    "--user",
+    f"{os.getuid()}:{os.getgid()}",
+    "--env",
+    "HOME=/tmp",
+    "--env",
+    "CONAN_HOME",
+    "--env",
+    "CONAN_USER",
+    "--env",
+    "CONAN_TOKEN",
+    *docker_workspace_args(root),
+    "--workdir",
+    str(root),
+    FETCH_IMAGE,
+    *command,
+  ]
+  return run(docker_command, capture=capture, display_command=display_command, env=fetch_env(root), check=check)
+
+def configure_remote(root, remote_name, remote_url):
+  user, token = optional_credentials()
+  run_fetch(
+    root,
+    [
+      "conan",
+      "remote",
+      "add",
+      "-f",
+      remote_name,
+      remote_url,
+    ],
+  )
+  if not user and not token:
+    return
+  run_fetch(
+    root,
+    [
+      "conan",
+      "remote",
+      "login",
+      remote_name,
+      user,
+      "-p",
+      token,
+    ],
+    display_command=[
+      "conan",
+      "remote",
+      "login",
+      remote_name,
+      user,
+      "-p",
+      "<hidden>",
+    ],
+  )
+
+def package_refs(data):
+  refs = []
+  for location_data in data.values():
+    if not isinstance(location_data, dict):
+      continue
+    for ref, ref_data in location_data.items():
+      if not ref.startswith(PACKAGE_REF) or not isinstance(ref_data, dict):
+        continue
+      revisions = ref_data.get(
+        "revisions",
+        {},
+      )
+      if not isinstance(revisions, dict):
+        continue
+      for recipe_revision, revision_data in revisions.items():
+        if not isinstance(revision_data, dict):
+          continue
+        packages = revision_data.get(
+          "packages",
+          {},
+        )
+        if not isinstance(packages, dict):
+          continue
+        for package_id, package_data in packages.items():
+          package_revisions = package_data.get(
+            "revisions",
+            {},
+          )if isinstance(package_data, dict) else {}
+          if isinstance(package_revisions, dict) and package_revisions:
+            for package_revision in package_revisions:
+              refs.append(f"{PACKAGE_REF}#{recipe_revision}:{package_id}#{package_revision}")
+          else:
+            refs.append(f"{PACKAGE_REF}#{recipe_revision}:{package_id}")
+  return sorted(refs)
+
+def remove_cached_modulus(root):
+  run_fetch(
+    root,
+    [
+      "conan",
+      "remove",
+      PACKAGE_REF + ":*",
+      "-c",
+    ],
+    check=False,
+  )
+
+def download_modulus(root, remote_name):
+  remove_cached_modulus(root)
+  run_fetch(
+    root,
+    [
+      "conan",
+      "download",
+      PACKAGE_REF + ":*",
+      "-r",
+      remote_name,
+    ],
+  )
+  completed = run_fetch(
+    root,
+    [
+      "conan",
+      "list",
+      PACKAGE_REF + ":*",
+      "-c",
+      "-f",
+      "json",
+    ],
+    capture=True,
+  )
+  refs = package_refs(json.loads(completed.stdout))
+  if not refs:
+    raise SystemExit(f"no downloaded {PACKAGE_REF} package found in the Conan cache")
+  package_ref = refs[-1]
+  completed = run_fetch(
+    root,
+    [
+      "conan",
+      "cache",
+      "path",
+      package_ref,
+    ],
+    capture=True,
+  )
+  package_root = pathlib.Path(completed.stdout.strip())
+  if not (package_root / "modulus" / "export.py").is_file() or not (package_root / "tools").is_dir():
+    raise SystemExit(f"downloaded {PACKAGE_REF} package does not contain Modulus export files: {package_root}")
+  return package_root
+
+def materialize_runtime(root):
+  remote_name, remote_url = remote_config(root)
+  configure_remote(root, remote_name, remote_url)
+  package_root = download_modulus(root, remote_name)
+  sys.path.insert(0, str(package_root))
+  import modulus.export
+  modulus.export.sync_project(root, package_root)
+
+def ensure_runtime(root):
+  if runtime_ready(root):
+    return
+  materialize_runtime(root)
+
+def delegate(command, argv):
+  root = repo_root()
+  ensure_runtime(root)
+  sys.path.insert(0, str(root / RUNTIME_DIR))
+  import modulus.cli
+  previous_argv = sys.argv
+  previous_cwd = pathlib.Path.cwd()
+  sys.argv = [
+    str(root / "modulus.py"),
+    *argv,
+  ]
+  os.chdir(root)
+  try:
+    return modulus.cli.main(command)
+  finally:
+    os.chdir(previous_cwd)
+    sys.argv = previous_argv
+
+def main(argv):
+  if not argv or argv[0] in ("-h", "--help"):
+    print("usage: python3 modulus.py " + "|".join(COMMANDS) + " ...")
+    return 0 if argv else 2
+  command = argv[0]
+  if command not in COMMANDS:
+    raise SystemExit(f"unknown Modulus command: {command}")
+  return delegate(command, argv[1:])
+
+if __name__ == "__main__":
+  raise SystemExit(main(sys.argv[1:]))