Clangd

tests/nvim_clangd.lua

@@ -0,0 +1,71 @@
+-- Explicit test loading leaves Neovim's persistent exrc trust database unchanged.
+if vim.env.MODULUS_NVIM_USER_CONFIG == "1" then
+  dofile(vim.fn.stdpath("config") .. "/init.lua")
+  require("lazy").load({ plugins = { "nvim-lspconfig" } })
+else
+  vim.lsp.config("clangd", { cmd = { "clangd" }, filetypes = { "c", "cpp" }, capabilities = { offsetEncoding = { "utf-16" } } })
+end
+vim.o.exrc = false
+local root = assert(vim.env.MODULUS_LSP_REPO)
+local attached = false
+local previous = vim.lsp.config.clangd.on_attach
+local on_attach = function(client, buffer)
+  attached = true
+  if previous then previous(client, buffer) end
+end
+vim.lsp.config("clangd", { on_attach = on_attach, capabilities = { modulus_test = true } })
+local original_cmd = vim.lsp.config.clangd.cmd
+dofile(root .. "/.nvim.lua")
+assert(vim.lsp.config.clangd.on_attach == on_attach)
+assert(vim.lsp.config.clangd.capabilities.modulus_test)
+local rpc_start = vim.lsp.rpc.start
+local fallback
+vim.lsp.rpc.start = function(command) fallback = command end
+vim.lsp.config.clangd.cmd({}, { root_dir = "/tmp/not-a-modulus-project" })
+vim.lsp.rpc.start = rpc_start
+assert(vim.deep_equal(fallback, original_cmd), "non-Modulus server command changed")
+
+vim.schedule(function()
+  local ok, error = pcall(function()
+    vim.cmd.edit(root .. "/" .. assert(vim.env.MODULUS_LSP_SOURCE))
+    assert(vim.wait(60000, function()
+      local clients = vim.lsp.get_clients({ bufnr = 0, name = "clangd" })
+      return #clients == 1 and clients[1].initialized and attached
+    end, 50), "container clangd did not attach")
+    local clients = vim.lsp.get_clients({ bufnr = 0, name = "clangd" })
+    assert(#clients == 1, "duplicate clangd clients")
+    local client = clients[1]
+    assert(client.config._modulus_container, "not the container server")
+    assert(client.config.root_dir == root, "wrong root")
+    local header = vim.env.MODULUS_LSP_HEADER or "cstdint"
+    local changed = false
+    local notification = vim.api.nvim_create_autocmd("LspNotify", {
+      callback = function(event)
+        if event.data.method == "textDocument/didChange" then changed = true end
+      end,
+    })
+    vim.api.nvim_buf_set_lines(0, 0, 0, false, { "#include <" .. header .. ">" })
+    assert(vim.wait(5000, function() return changed end, 10), "buffer change was not sent")
+    vim.api.nvim_del_autocmd(notification)
+    local result = assert(client:request_sync("textDocument/definition", {
+      textDocument = { uri = vim.uri_from_bufnr(0) }, position = { line = 0, character = 12 },
+    }, 60000, 0))
+    assert(result.result and #result.result > 0, vim.inspect(result))
+    local errors = vim.diagnostic.get(0, { severity = vim.diagnostic.severity.ERROR })
+    assert(#errors == 0, vim.inspect(errors))
+    local location = result.result[1]
+    local target = vim.uri_to_fname(location.uri or location.targetUri)
+    assert(target:find("/modulus/clangd/", 1, true), target)
+    vim.lsp.util.show_document(location, client.offset_encoding, { focus = true })
+    assert(vim.bo.readonly and not vim.bo.modifiable, "cached header is writable")
+    assert(#vim.lsp.get_clients({ name = "clangd" }) == 1, "navigation spawned another clangd")
+    print("MODULUS_NVIM_OK " .. root .. " " .. target)
+    client:stop()
+    vim.wait(20000, function() return client:is_stopped() end, 50)
+  end)
+  if not ok then
+    print("MODULUS_NVIM_FAILURE " .. tostring(error))
+    vim.cmd("cquit 1")
+  end
+  vim.cmd("qa!")
+end)

tests/test_clangd.py

@@ -0,0 +1,161 @@
+import contextlib
+import io
+import json
+import os
+import pathlib
+import subprocess
+import tarfile
+import tempfile
+import unittest
+from unittest.mock import MagicMock, patch
+
+import modulus.clangd as clangd
+import modulus.cli as cli
+import modulus.config as config
+import modulus.export as export
+
+class clangd_test(unittest.TestCase):
+  def setUp(self):
+    temporary = tempfile.TemporaryDirectory()
+    self.addCleanup(temporary.cleanup)
+    self.root = pathlib.Path(temporary.name)
+    self.context = cli.build_context(self.root, config.normalize({"package": {"name": "sample"}}))
+    self.context.dockerfile.parent.mkdir(parents=True)
+    self.context.dockerfile.write_text("FROM sample\n")
+    self.image = "sha256:" + "1" * 64
+
+  def test_defaults_and_custom_settings(self):
+    self.assertEqual(clangd.settings(self.context)["executable"], "/usr/bin/clangd-19")
+    self.assertIn("/usr/bin/g++-12", clangd.settings(self.context)["query_drivers"])
+    custom = {"executable": "/opt/bin/clangd", "arguments": [], "query_drivers": ["/opt/bin/g++"], "header_roots": ["/opt/sdk"]}
+    result = config.normalize({"package": {"name": "sample"}, "tooling": {"lsp": {"clangd": custom}}})
+    self.assertEqual(result["tooling"]["lsp"]["clangd"], custom)
+
+  def test_config_rejects_unsafe_or_unknown_settings(self):
+    for invalid in ([], {"unknown": True}, {"executable": "clangd"}, {"arguments": "--background-index"},
+                    {"query_drivers": ["/usr/bin/*"]}, {"header_roots": ["/"]},
+                    {"header_roots": ["/usr/../etc"]}, {"query_drivers": ["/bin/g++,/bin/gcc"]},
+                    {"arguments": ["--query-driver=*"]}, {"arguments": ["--path-mappings=/=/"]}):
+      with self.subTest(invalid=invalid), self.assertRaises(config.config_error):
+        config.normalize({"package": {"name": "sample"}, "tooling": {"lsp": {"clangd": invalid}}})
+
+  def test_image_fingerprint_and_architecture(self):
+    image = {"Id": self.image, "Architecture": "amd64", "Config": {"Labels": {clangd.image_label: clangd.recipe_digest(self.context)}}}
+    with patch.object(subprocess, "run", return_value=subprocess.CompletedProcess([], 0, json.dumps([image]))):
+      self.assertEqual(clangd.current_image(self.context), self.image)
+      self.assertIsNone(clangd.current_image(self.context, "arm64"))
+      self.context.dockerfile.write_text("FROM changed\n")
+      self.assertIsNone(clangd.current_image(self.context))
+
+  def test_cache_is_shared_by_image_and_header_settings_not_repo(self):
+    cache = clangd.cache_directory(self.context, self.image)
+    other = cli.build_context(self.root / "other", self.context.config)
+    self.assertEqual(cache, clangd.cache_directory(other, self.image))
+    self.context.lsp_settings["clangd"]["arguments"].append("--log=error")
+    self.assertEqual(cache, clangd.cache_directory(self.context, self.image))
+    self.context.lsp_settings["clangd"]["header_roots"].append("/opt/sdk")
+    self.assertNotEqual(cache, clangd.cache_directory(self.context, self.image))
+    self.assertNotEqual(cache, clangd.cache_directory(other, "sha256:" + "2" * 64))
+
+  def test_stdio_launch_has_no_tty_socket_credentials_or_host_home(self):
+    command = clangd.launch_command(self.context, [], self.image, self.root / "cache", ["/usr/include"], "test-server")
+    self.assertIn("--interactive", command)
+    self.assertIn("--read-only", command)
+    self.assertIn(f"{self.root}:{self.root}:ro", command)
+    self.assertIn(f"{self.context.build_root}:{self.context.build_root}:rw", command)
+    self.assertIn("--network", command)
+    self.assertNotIn("-t", command)
+    for unwanted in ("--tty", "--volumes-from", "CONAN_TOKEN", "/var/run/docker.sock", str(pathlib.Path.home()) + ":"):
+      self.assertNotIn(unwanted, " ".join(command))
+    self.assertIn("--query-driver=/usr/bin/gcc-12,/usr/bin/g++-12,/usr/bin/aarch64-linux-gnu-gcc,/usr/bin/aarch64-linux-gnu-g++", command)
+
+  def test_startup_failure_does_not_prepare_and_keeps_stdout_clean(self):
+    output, errors = io.StringIO(), io.StringIO()
+    with patch.object(clangd, "current_image", return_value=None), patch.object(clangd, "prepare") as prepare, \
+         patch.object(cli, "ensure_build_image") as build, contextlib.redirect_stdout(output), contextlib.redirect_stderr(errors):
+      self.assertEqual(clangd.launch(self.context, []), 1)
+    prepare.assert_not_called()
+    build.assert_not_called()
+    self.assertEqual(output.getvalue(), "")
+    self.assertIn("configure", errors.getvalue())
+
+  def test_path_mapping_uses_client_to_server_direction_and_specificity(self):
+    self.assertEqual(clangd.path_mappings(pathlib.Path("/cache"), ["/usr/include", "/opt/sdk"]),
+                     "--path-mappings=/cache/rootfs/usr/include=/usr/include,/cache/rootfs/opt/sdk=/opt/sdk")
+
+  def archive(self, name, kind=tarfile.REGTYPE):
+    stream = io.BytesIO()
+    with tarfile.open(fileobj=stream, mode="w") as archive:
+      member = tarfile.TarInfo(name)
+      member.type = kind
+      member.linkname = "/etc/passwd" if kind == tarfile.SYMTYPE else ""
+      archive.addfile(member)
+    stream.seek(0)
+    return stream
+
+  def test_header_extraction_rejects_escape_and_links(self):
+    for name, kind in (("../bad", tarfile.REGTYPE), ("/usr/include/bad", tarfile.REGTYPE),
+                       ("usr/include/../bad", tarfile.REGTYPE), ("etc/passwd", tarfile.REGTYPE),
+                       ("usr/include/link", tarfile.SYMTYPE)):
+      with self.subTest(name=name), self.assertRaises(ValueError):
+        clangd.extract_headers(self.archive(name, kind), self.root, ["/usr/include"])
+    clangd.extract_headers(self.archive("usr/include/header.h"), self.root, ["/usr/include"])
+    self.assertEqual((self.root / "usr/include/header.h").stat().st_mode & 0o777, 0o444)
+
+  def test_completed_cache_never_invokes_container_again(self):
+    cache = self.root / "cache"
+    (cache / "rootfs/usr/include").mkdir(parents=True)
+    (cache / "manifest.json").write_text(json.dumps({"version": clangd.cache_version, "roots": ["/usr/include"]}))
+    with patch.object(clangd, "cache_directory", return_value=cache), patch.object(clangd, "current_image", return_value=self.image), \
+         patch.object(subprocess, "run") as run:
+      self.assertEqual(clangd.prepare(self.context), 0)
+      run.assert_not_called()
+
+  def test_failed_extraction_never_publishes_partial_cache(self):
+    cache = self.root / "image/cache"
+    process = MagicMock()
+    process.__enter__.return_value = process
+    process.stdout = self.archive("../escape")
+    with patch.object(clangd, "cache_directory", return_value=cache), patch.object(clangd, "current_image", return_value=self.image), \
+         patch.object(subprocess, "run", return_value=subprocess.CompletedProcess([], 0, '["/usr/include"]')), \
+         patch.object(subprocess, "Popen", return_value=process), contextlib.redirect_stderr(io.StringIO()):
+      self.assertEqual(clangd.prepare(self.context), 1)
+    self.assertFalse(cache.exists())
+    self.assertEqual(list(cache.parent.glob(".headers-*")), [])
+
+  def test_cleanup_always_waits_for_child_even_if_docker_removal_times_out(self):
+    process = MagicMock()
+    process.wait.side_effect = [0, subprocess.TimeoutExpired("docker", 5), 0]
+    with patch.object(subprocess, "Popen", return_value=process), \
+         patch.object(subprocess, "run", side_effect=subprocess.TimeoutExpired("docker rm", 15)), \
+         contextlib.redirect_stderr(io.StringIO()):
+      self.assertEqual(clangd.run_server(["docker"], "test-server"), 0)
+    process.kill.assert_called_once()
+    self.assertEqual(process.wait.call_count, 3)
+
+  def test_nvim_generation_is_stable_opt_out_and_respects_ownership(self):
+    export.sync_nvim_file(self.root, {})
+    path = self.root / ".nvim.lua"
+    before = path.stat().st_mtime_ns
+    export.sync_nvim_file(self.root, {})
+    self.assertEqual(before, path.stat().st_mtime_ns)
+    export.sync_nvim_file(self.root, {"enabled": False})
+    self.assertFalse(path.exists())
+    path.write_text("-- user settings\n")
+    with contextlib.redirect_stderr(io.StringIO()):
+      export.sync_nvim_file(self.root, {})
+      export.sync_nvim_file(self.root, {"enabled": False})
+    self.assertEqual(path.read_text(), "-- user settings\n")
+
+  def test_lsp_prepares_after_successful_database_refresh_only(self):
+    with patch.object(cli, "in_docker", return_value=False), patch.object(cli, "docker_run", return_value=0) as run, \
+         patch.object(clangd, "prepare", return_value=0) as prepare:
+      self.assertEqual(cli.lsp(self.context, []), 0)
+      prepare.assert_called_once_with(self.context)
+      prepare.reset_mock()
+      run.return_value = 1
+      self.assertEqual(cli.lsp(self.context, []), 1)
+      prepare.assert_not_called()
+
+if __name__ == "__main__":
+  unittest.main()

tests/test_clangd_integration.py

@@ -0,0 +1,164 @@
+"""Opt-in real Docker/LSP checks: MODULUS_LSP_TEST_ROOTS=/repo:/other/repo."""
+import json
+import os
+import pathlib
+import queue
+import shutil
+import subprocess
+import tempfile
+import threading
+import time
+import unittest
+import urllib.parse
+
+class lsp_session:
+  def __init__(self, root):
+    command = ["python3", "tools/clangd.py"] if (root / "modulus/cli.py").is_file() else ["python3", "modulus.py", "clangd"]
+    self.log = tempfile.TemporaryFile(mode="w+b")
+    self.process = subprocess.Popen(command, cwd=root, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=self.log)
+    self.messages = queue.Queue()
+    self.reader = threading.Thread(target=self.read_messages, daemon=True)
+    self.reader.start()
+    self.next_id = 0
+    self.pending = []
+
+  def read_messages(self):
+    try:
+      while True:
+        line = self.process.stdout.readline()
+        if not line:
+          raise EOFError("LSP stream closed")
+        headers = {}
+        while line != b"\r\n":
+          name, value = line.decode().strip().split(":", 1)
+          headers[name.lower()] = value.strip()
+          line = self.process.stdout.readline()
+          if not line:
+            raise EOFError("incomplete LSP header")
+        self.messages.put(json.loads(self.process.stdout.read(int(headers["content-length"]))))
+    except Exception as error:
+      self.messages.put(error)
+
+  def send(self, message):
+    body = json.dumps({"jsonrpc": "2.0", **message}).encode()
+    self.process.stdin.write(f"Content-Length: {len(body)}\r\n\r\n".encode() + body)
+    self.process.stdin.flush()
+
+  def wait(self, predicate, timeout=90):
+    deadline = time.monotonic() + timeout
+    while True:
+      for index, message in enumerate(self.pending):
+        if predicate(message):
+          return self.pending.pop(index)
+      try:
+        message = self.messages.get(timeout=max(0, deadline - time.monotonic()))
+      except queue.Empty:
+        raise AssertionError("LSP response timed out") from None
+      if isinstance(message, Exception):
+        self.log.seek(0)
+        raise AssertionError(f"{message}\n{self.log.read().decode(errors='replace')}")
+      self.pending.append(message)
+
+  def request(self, method, params):
+    self.next_id += 1
+    self.send({"id": self.next_id, "method": method, "params": params})
+    response = self.wait(lambda message: message.get("id") == self.next_id)
+    if "error" in response:
+      raise AssertionError(response["error"])
+    return response.get("result")
+
+  def initialize(self, root):
+    result = self.request("initialize", {"processId": os.getpid(), "rootUri": root.as_uri(),
+      "capabilities": {"textDocument": {"publishDiagnostics": {"versionSupport": True}}}})
+    self.send({"method": "initialized", "params": {}})
+    return result
+
+  def open(self, path, text):
+    uri = path.as_uri()
+    self.send({"method": "textDocument/didOpen", "params": {"textDocument": {
+      "uri": uri, "languageId": "cpp", "version": 1, "text": text}}})
+    return self.wait(lambda message: message.get("method") == "textDocument/publishDiagnostics"
+                     and message["params"]["uri"] == uri)["params"]["diagnostics"]
+
+  def close(self):
+    if self.process.poll() is None:
+      self.process.terminate()
+      self.process.wait(timeout=25)
+    self.process.stdin.close()
+    self.reader.join(timeout=5)
+    self.process.stdout.close()
+    self.log.close()
+
+@unittest.skipUnless(os.environ.get("MODULUS_LSP_TEST_ROOTS"), "requires prepared Docker images and databases")
+class container_clangd_test(unittest.TestCase):
+  def roots(self):
+    return [pathlib.Path(root).resolve() for root in os.environ["MODULUS_LSP_TEST_ROOTS"].split(os.pathsep)]
+
+  @unittest.skipUnless(shutil.which("nvim"), "Neovim is not installed")
+  def test_neovim_attachment_and_readonly_navigation(self):
+    script = pathlib.Path(__file__).with_name("nvim_clangd.lua")
+    for root in self.roots():
+      database = json.loads((root / "build/build_x86_debug/compile_commands.json").read_text())
+      source = next(pathlib.Path(entry["file"]) for entry in database
+                    if pathlib.Path(entry["file"]).is_relative_to(root / "src"))
+      sources = [(source, "cstdint")]
+      if (root / "src/vamp_pico/main.cpp").exists():
+        sources.append((root / "src/vamp_pico/main.cpp", "pico/stdlib.h"))
+      for source, header in sources:
+        with self.subTest(repo=root.name, source=str(source)):
+          env = {**os.environ, "MODULUS_LSP_REPO": str(root), "MODULUS_LSP_SOURCE": str(source.relative_to(root)),
+                 "MODULUS_LSP_HEADER": header}
+          result = subprocess.run(["nvim", "--headless", "-i", "NONE", "-u", str(script)], cwd=root,
+                                  env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=180)
+          self.assertEqual(result.returncode, 0, result.stdout)
+          self.assertIn("MODULUS_NVIM_OK", result.stdout)
+
+  def test_native_and_pico_diagnostics_and_local_header_definitions(self):
+    for root in self.roots():
+      with self.subTest(repo=root.name):
+        session = lsp_session(root)
+        try:
+          initialized = session.initialize(root)
+          self.assertIn("19.", initialized["serverInfo"]["version"])
+          database = json.loads((root / "build/build_x86_debug/compile_commands.json").read_text())
+          source = next(pathlib.Path(entry["file"]) for entry in database
+                        if pathlib.Path(entry["file"]).is_relative_to(root / "src"))
+          sources = [(source, "cstdint")]
+          if (root / "src/vamp_pico").is_dir():
+            sources.extend((path, "pico/stdlib.h") for path in sorted((root / "src/vamp_pico").glob("*.cpp")))
+          for source, header in sources:
+            with self.subTest(source=str(source)):
+              text = f"#include <{header}>\n" + source.read_text()
+              diagnostics = session.open(source, text)
+              self.assertEqual([item for item in diagnostics if item.get("severity") == 1], [])
+              definitions = session.request("textDocument/definition", {
+                "textDocument": {"uri": source.as_uri()}, "position": {"line": 0, "character": 12}})
+              self.assertTrue(definitions)
+              uri = definitions[0].get("uri", definitions[0].get("targetUri"))
+              path = pathlib.Path(urllib.parse.unquote(urllib.parse.urlparse(uri).path))
+              self.assertIn("/modulus/clangd/", str(path))
+              self.assertTrue(path.is_file(), str(path))
+              self.assertEqual(path.stat().st_mode & 0o222, 0)
+          session.request("shutdown", None)
+          session.send({"method": "exit"})
+          session.process.stdin.close()
+          self.assertEqual(session.process.wait(timeout=25), 0)
+        finally:
+          session.close()
+
+  def test_eof_and_termination_remove_container(self):
+    def containers():
+      return set(subprocess.check_output(["docker", "ps", "-q", "--filter", "name=modulus-clangd-"], text=True).split())
+    before = containers()
+    for terminate in (False, True):
+      session = lsp_session(self.roots()[0])
+      try:
+        session.initialize(self.roots()[0])
+        if terminate:
+          session.process.terminate()
+        else:
+          session.process.stdin.close()
+        session.process.wait(timeout=25)
+        self.assertEqual(containers() - before, set())
+      finally:
+        session.close()

tests/test_cli.py

@@ -189,7 +189,7 @@ class upload_packages_test(unittest.TestCase):
       previous_cwd = pathlib.Path.cwd()
       os.chdir(repo)
       try:
-        for command in ("image", "docs", "shell", "clangd", "docker_support"):
+        for command in ("image", "docs", "shell", "docker_support"):
           with self.assertRaisesRegex(SystemExit, "unknown modulus command"):
             modulus.cli.main(command)
       finally:
@@ -332,6 +332,8 @@ class upload_packages_test(unittest.TestCase):
         context.dockerfile,
         "--tag",
         "sample-build:dev",
+        "--label",
+        f"{modulus.cli.modulus.clangd.image_label}={modulus.cli.modulus.clangd.recipe_digest(context)}",
         ".",
       ],
     )

tests/test_export.py

@@ -263,7 +263,7 @@ class modulus_export_test(unittest.TestCase):
     }
     config["tooling"] = {"editor": editor}
     normalized = modulus.config.normalize(config)
-    self.assertEqual(normalized["tooling"]["editor"], editor)
+    self.assertEqual(normalized["tooling"]["editor"], {**editor, "nvim": {"enabled": True}})
     with tempfile.TemporaryDirectory() as directory:
       root = pathlib.Path(directory)
       modulus.export.sync_editor_files(root, normalized)