modulus/modulus/artifacts.py

#!/usr/bin/env python3
import argparse
import base64
import hashlib
import json
import os
import pathlib
import sys
import tarfile
import urllib.error
import urllib.parse
import urllib.request

REGISTRY_SLOT = "latest"
PAGE_LIMIT = 50

def archive_name(package_name, target):
  return f'{package_name}_{target}.tar.gz'

def create_archive(source_dir, archive):
  with tarfile.open(archive, 'w:gz') as output:
    for path in sorted(source_dir.rglob('*')):
      output.add(path, arcname=path.relative_to(source_dir), recursive=False)

def checksum_text(archive):
  with archive.open('rb') as stream:
    digest = hashlib.file_digest(stream, 'sha256').hexdigest()
  return f'{digest}  {archive.name}\n'

def write_checksum(archive):
  checksum = archive.with_name(archive.name + '.sha256')
  checksum.write_text(checksum_text(archive), encoding='utf-8')
  return checksum

def parse_registry_url(registry_url):
  parsed = urllib.parse.urlparse(registry_url.rstrip("/"))
  parts = [urllib.parse.unquote(part) for part in parsed.path.split("/") if part]
  if len(parts) < 4 or parts[-4] != "api" or parts[-3] != "packages" or parts[-1] != "generic":
    raise SystemExit("package artifact registry_url must end with /api/packages/{owner}/generic")
  owner = parts[-2]
  prefix = "/" + "/".join(urllib.parse.quote(part, safe="") for part in parts[:-4]) if parts[:-4] else ""
  return parsed, prefix, owner

def build_url(parsed, path, query=""):
  return urllib.parse.urlunparse((
    parsed.scheme,
    parsed.netloc,
    path,
    "",
    query,
    "",
  ))

def quote_path(*parts):
  return "/".join(urllib.parse.quote(part, safe="") for part in parts)

def version_list_url(parsed, prefix, owner, package_name, page):
  path = f"{prefix}/api/v1/packages/{quote_path(owner, 'generic', package_name)}"
  query = urllib.parse.urlencode({
    "page": page,
    "limit": PAGE_LIMIT,
  })
  return build_url(parsed, path, query)

def version_delete_url(parsed, prefix, owner, package_name, version):
  path = f"{prefix}/api/packages/{quote_path(owner, 'generic', package_name, version)}"
  return build_url(parsed, path)

def file_upload_url(registry_url, package_name, path):
  return "/".join((
    registry_url.rstrip("/"),
    urllib.parse.quote(package_name, safe=""),
    REGISTRY_SLOT,
    urllib.parse.quote(path.name, safe=""),
  ))

def authorization(user, token):
  encoded = base64.b64encode(f"{user}:{token}".encode("utf-8")).decode("ascii")
  return f"Basic {encoded}"

def request(method, url, auth, data=None, headers=None, missing_ok=False):
  request_headers = {
    "Authorization": auth,
    **(headers or {}),
  }
  request = urllib.request.Request(url, data=data, method=method, headers=request_headers)
  try:
    return urllib.request.urlopen(request, timeout=120)
  except urllib.error.HTTPError as error:
    if missing_ok and error.code == 404:
      return None
    message = f"{method} {url} failed with HTTP {error.code}"
    raise SystemExit(message) from error
  except urllib.error.URLError as error:
    raise SystemExit(f"{method} {url} failed: {error.reason}") from error

def list_versions(parsed, prefix, owner, package_name, auth):
  versions = []
  page = 1
  while True:
    response = request("GET", version_list_url(parsed, prefix, owner, package_name, page), auth, missing_ok=True)
    if response is None:
      return []
    with response:
      data = json.loads(response.read().decode("utf-8"))
    if not isinstance(data, list):
      raise SystemExit("Gitea package version list response was not an array")
    if not data:
      break
    for item in data:
      if isinstance(item, dict) and isinstance(item.get("version"), str):
        versions.append(item["version"])
    if len(data) < PAGE_LIMIT:
      break
    page += 1
  return sorted(set(versions))

def delete_version(parsed, prefix, owner, package_name, version, auth):
  url = version_delete_url(parsed, prefix, owner, package_name, version)
  response = request("DELETE", url, auth, missing_ok=True)
  if response is None:
    print(f"Package version already absent: {package_name}/{version}")
    return
  with response:
    if response.status != 204:
      raise SystemExit(f"DELETE {url} returned HTTP {response.status}")
  print(f"Deleted package version: {package_name}/{version}")

def discover_artifacts(dist_dir, package_name, targets):
  artifacts = []
  for target in sorted(set(targets)):
    archive = dist_dir / archive_name(package_name, target)
    artifacts.extend((archive, archive.with_name(archive.name + ".sha256")))
  missing = [path for path in artifacts if not path.is_file()]
  if missing:
    found = sorted(path.name for path in dist_dir.glob(f"{package_name}*.tar.gz*") if path.is_file())
    raise SystemExit("missing artifact files: " + ", ".join(str(path) for path in missing)
      + "\nFound: " + (", ".join(found) or "none")
      + "\nPublish the current Modulus tooling, run configure --update, and recreate the package so the builder and publisher use the same filenames.")
  for archive, checksum in zip(artifacts[::2], artifacts[1::2]):
    if archive.is_symlink() or checksum.is_symlink():
      raise SystemExit('artifact files must not be symlinks')
    if checksum.read_text(encoding='utf-8') != checksum_text(archive):
      raise SystemExit(f'artifact checksum mismatch: {archive}')
  return artifacts

def upload_file(registry_url, package_name, path, auth):
  url = file_upload_url(registry_url, package_name, path)
  headers = {
    "Content-Length": str(path.stat().st_size),
    "Content-Type": "application/octet-stream",
  }
  with path.open("rb") as input_file:
    response = request("PUT", url, auth, data=input_file, headers=headers)
    with response:
      if response.status != 201:
        raise SystemExit(f"PUT {url} returned HTTP {response.status}")
  print(f"Uploaded package file: {package_name}/{path.name}")

def parse_args(argv):
  parser = argparse.ArgumentParser(description="Publish the configured rolling Gitea generic package.")
  parser.add_argument("--dist-dir", type=pathlib.Path)
  return parser.parse_args(argv)

def main(context, argv):
  args = parse_args(argv)
  if not context.artifact['enabled']:
    raise SystemExit('artifact publishing is not enabled in modulus.json')
  package_name, registry_url = context.package['name'], context.artifact['registry_url']
  targets = context.artifact['targets'] or [context.default_target_name]
  artifacts = discover_artifacts(args.dist_dir or context.build_root / 'packages', package_name, targets)
  user = os_environ("PACKAGE_REGISTRY_USER")
  token = os_environ("PACKAGE_REGISTRY_TOKEN")
  auth = authorization(user, token)
  parsed, prefix, owner = parse_registry_url(registry_url)
  existing_versions = list_versions(parsed, prefix, owner, package_name, auth)
  # Gitea rejects duplicate file names within a generic package version, so
  # replacing the rolling release requires deleting the old version first.
  if REGISTRY_SLOT in existing_versions:
    delete_version(parsed, prefix, owner, package_name, REGISTRY_SLOT, auth)
  for artifact in artifacts:
    upload_file(registry_url, package_name, artifact, auth)
  for version in existing_versions:
    if version != REGISTRY_SLOT:
      delete_version(parsed, prefix, owner, package_name, version, auth)
  return 0

def os_environ(name):
  value = os.environ.get(name)
  if not value:
    raise SystemExit(f"{name} is required")
  return value