Cleanup

.gitea/workflows/ci.yml

@@ -47,7 +47,7 @@ jobs:
           key: conan-${{ runner.os }}-${{ hashFiles('conanfile.py', 'modulus.json') }}
           restore-keys: conan-${{ runner.os }}-
       - name: Configure native release
-        run: python3 modulus.py configure --target x86_64_gcc12_release
+        run: python3 modulus.py configure --target x86_64_gcc12_release --update
       - name: Check code style
         run: |
           python3 -m pip install --no-cache-dir -r build/build_tools/docker/format-requirements.txt
@@ -63,4 +63,4 @@ jobs:
           CONAN_TOKEN: ${{ secrets.CONAN_TOKEN }}
           NOTO_CONAN_UPLOAD: "1"
           MODULUS_ALLOW_PACKAGE_REPLACE: "1"
-        run: python3 modulus.py package --all --replace
+        run: python3 modulus.py package --all --replace --update

conanfile.py

@@ -28,8 +28,11 @@ class noto_conan(conan.ConanFile):
     "fPIC": True,
     "with_tests": False,
   }
-  exports = "modulus.json"
+  exports = "modulus.json", "!build", "!.git", "!test_package/build"
   exports_sources = (
+    "!build",
+    "!.git",
+    "!test_package/build",
     "CMakeLists.txt",
     "cmake/*.in",
     "include/*.hpp",

include/noto/detail/errors.hpp

@@ -15,7 +15,7 @@ constexpr std::int32_t error_invalid_argument{1};
 constexpr std::int32_t error_not_found{2};
 constexpr std::int32_t error_unavailable{3};
 constexpr std::int32_t error_timeout{4};
-constexpr std::int32_t error_cancelled{5};
+constexpr std::int32_t error_canceled{5};
 constexpr std::int32_t error_unauthenticated{6};
 constexpr std::int32_t error_permission_denied{7};
 constexpr std::int32_t error_already_exists{8};

modulus.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+# Canonical consumer bootstrap. Copy this file unchanged to a project's modulus.py.
 import json
 import os
 import pathlib
@@ -17,6 +18,7 @@ COMMANDS = (
   "lsp",
   "clangd",
   "format",
+  "run",
 )
 COMMON_DIR = pathlib.Path("build") / "build_common"
 TOOLS_DIR = pathlib.Path("build") / "build_tools"
@@ -66,13 +68,6 @@ def runtime_ready(root):
     and (root / RUNTIME_DIR / "modulus" / "format_layout.py").is_file()
   )
 
-def optional_credentials():
-  user = os.environ.get("CONAN_USER")
-  token = os.environ.get("CONAN_TOKEN")
-  if bool(user) != bool(token):
-    raise SystemExit("CONAN_USER and CONAN_TOKEN must either both be set or both be unset")
-  return user, token
-
 def env_flag(name):
   return os.environ.get(name, "").lower() in {"1", "true", "yes", "on"}
 
@@ -128,7 +123,7 @@ RUN python -m pip install --no-cache-dir conan=={CONAN_VERSION}
 WORKDIR /workspace
 """
 
-def ensure_fetch_image():
+def ensure_fetch_image(offline=False):
   inspected = subprocess.run(
     [
       "docker",
@@ -140,12 +135,15 @@ def ensure_fetch_image():
     stderr=subprocess.DEVNULL,
     check=False,
   )
-  if inspected.returncode == 0:
+  if inspected.returncode == 0 and (offline or not docker_platform_args()):
     return
+  if offline:
+    raise SystemExit('offline bootstrap requires the existing Modulus fetch image or MODULUS_DIRECT=1 with Conan installed')
   run(
     [
       "docker",
       "build",
+      *docker_platform_args(),
       "--tag",
       FETCH_IMAGE,
       "-",
@@ -156,6 +154,10 @@ def ensure_fetch_image():
 def in_container():
   return pathlib.Path("/.dockerenv").exists()
 
+def docker_platform_args():
+  platform = os.environ.get("MODULUS_DOCKER_PLATFORM", "").strip()
+  return ["--platform", platform] if platform else []
+
 def docker_workspace_args(root):
   if in_container():
     container = os.environ.get("HOSTNAME", "").strip()
@@ -172,18 +174,24 @@ def docker_workspace_args(root):
 
 def fetch_env(root):
   env = os.environ.copy()
+  for name in list(env):
+    if name in {"CONAN_USER", "CONAN_TOKEN"} or name.startswith(("CONAN_LOGIN_USERNAME", "CONAN_PASSWORD")):
+      del env[name]
   env["CONAN_HOME"] = str(root / CONAN_HOME_DIR)
+  env["CONAN_NON_INTERACTIVE"] = "1"
   return env
 
-def run_fetch(root, command, capture=False, display_command=None, check=True):
+def run_fetch(root, command, capture=False, display_command=None, check=True, offline=False):
   if direct_mode(root):
     return run(command, capture=capture, display_command=display_command, env=fetch_env(root), check=check)
-  ensure_fetch_image()
+  ensure_fetch_image(offline=offline)
   docker_command = [
     "docker",
     "run",
     "--rm",
     "--interactive",
+    *(["--pull=never", "--network=none"] if offline else []),
+    *docker_platform_args(),
     "--user",
     f"{os.getuid()}:{os.getgid()}",
     "--env",
@@ -191,9 +199,7 @@ def run_fetch(root, command, capture=False, display_command=None, check=True):
     "--env",
     "CONAN_HOME",
     "--env",
-    "CONAN_USER",
+    "CONAN_NON_INTERACTIVE",
-    "--env",
-    "CONAN_TOKEN",
     *docker_workspace_args(root),
     "--workdir",
     str(root),
@@ -203,7 +209,6 @@ def run_fetch(root, command, capture=False, display_command=None, check=True):
   return run(docker_command, capture=capture, display_command=display_command, env=fetch_env(root), check=check)
 
 def configure_remote(root, remote_name, remote_url):
-  user, token = optional_credentials()
   run_fetch(
     root,
     [
@@ -215,29 +220,7 @@ def configure_remote(root, remote_name, remote_url):
       remote_url,
     ],
   )
-  if not user and not token:
+  run_fetch(root, ["conan", "remote", "logout", remote_name])
-    return
-  run_fetch(
-    root,
-    [
-      "conan",
-      "remote",
-      "login",
-      remote_name,
-      user,
-      "-p",
-      token,
-    ],
-    display_command=[
-      "conan",
-      "remote",
-      "login",
-      remote_name,
-      user,
-      "-p",
-      "<hidden>",
-    ],
-  )
 
 def package_refs(data):
   refs = []
@@ -274,46 +257,26 @@ def package_refs(data):
             refs.append(f"{PACKAGE_REF}#{recipe_revision}:{package_id}")
   return sorted(refs)
 
-def remove_cached_modulus(root):
+def find_modulus(root, remote_name=None):
-  run_fetch(
-    root,
-    [
-      "conan",
-      "remove",
-      PACKAGE_REF + ":*",
-      "-c",
-    ],
-    check=False,
-  )
-
-def download_modulus(root, remote_name):
-  remove_cached_modulus(root)
-  run_fetch(
-    root,
-    [
-      "conan",
-      "download",
-      PACKAGE_REF + ":*",
-      "-r",
-      remote_name,
-    ],
-  )
   completed = run_fetch(
     root,
     [
       "conan",
       "list",
-      PACKAGE_REF + ":*",
+      PACKAGE_REF + "#latest:*#latest",
-      "-c",
+      *(["-r", remote_name] if remote_name else ["-c"]),
       "-f",
       "json",
     ],
     capture=True,
+    offline=remote_name is None,
   )
   refs = package_refs(json.loads(completed.stdout))
   if not refs:
-    raise SystemExit(f"no downloaded {PACKAGE_REF} package found in the Conan cache")
+    raise SystemExit(f"no {PACKAGE_REF} package available in {remote_name or 'the local Conan cache'}")
   package_ref = refs[-1]
+  if remote_name:
+    run_fetch(root, ["conan", "download", package_ref, "-r", remote_name])
   completed = run_fetch(
     root,
     [
@@ -323,24 +286,25 @@ def download_modulus(root, remote_name):
       package_ref,
     ],
     capture=True,
+    offline=remote_name is None,
   )
   package_root = pathlib.Path(completed.stdout.strip())
   if not (package_root / "modulus" / "export.py").is_file() or not (package_root / "tools").is_dir():
     raise SystemExit(f"downloaded {PACKAGE_REF} package does not contain Modulus export files: {package_root}")
   return package_root
 
-def materialize_runtime(root):
+def materialize_runtime(root, no_remote=False):
   remote_name, remote_url = remote_config(root)
+  if not no_remote:
     configure_remote(root, remote_name, remote_url)
-  package_root = download_modulus(root, remote_name)
+  package_root = find_modulus(root, None if no_remote else remote_name)
   sys.path.insert(0, str(package_root))
   import modulus.export
   modulus.export.sync_project(root, package_root)
 
-def ensure_runtime(root):
+def ensure_runtime(root, update=False, no_remote=False):
-  if runtime_ready(root):
+  if update or not runtime_ready(root):
-    return
+    materialize_runtime(root, no_remote=no_remote)
-  materialize_runtime(root)
 
 def delegate(command, argv):
   root = repo_root()
@@ -349,7 +313,7 @@ def delegate(command, argv):
       print(f"Modulus {command} tooling is missing; run ./modulus.py configure first", file=sys.stderr)
       return 1
   else:
-    ensure_runtime(root)
+    ensure_runtime(root, update=command == "configure" and "--update" in argv, no_remote="--no-remote" in argv)
   sys.path.insert(0, str(root / RUNTIME_DIR))
   import modulus.cli
   previous_argv = sys.argv

src/detail/core.cpp

@@ -767,6 +767,45 @@ auto reset_statement_after_execute_failure(sqlite3_stmt *statement, sqlite3 *dat
   return reset ? status : reset;
 }
 
+auto reset_and_bind(sqlite3_stmt *statement, sqlite3 *database, std::vector<noto::sqlite_value> const &params) -> std::expected<void, modulus::error> {
+  auto code = sqlite3_reset(statement);
+  auto status = noto::result_from(code, database, "reset statement");
+  if (status) {
+    code = sqlite3_clear_bindings(statement);
+    status = noto::result_from(code, database, "clear statement bindings");
+  }
+  if (status) {
+    status = bind_values(statement, params, database);
+  }
+  return status;
+}
+
+auto collect_result(sqlite3_stmt *statement, sqlite3 *database, noto::execute_options const &options) -> std::expected<noto::query_result, modulus::error> {
+  auto result = noto::query_result{};
+  auto result_bytes = std::size_t{};
+  result.columns = column_names(statement);
+  auto status = track_result_columns(result.columns, result_bytes, options);
+  if (!status) {
+    return failure_from<noto::query_result>(reset_statement_after_execute_failure(statement, database, status));
+  }
+  while (true) {
+    auto code = sqlite3_step(statement);
+    if (code == SQLITE_ROW) {
+      status = append_result_row(result, current_row(statement), result_bytes, options);
+      if (!status) {
+        return failure_from<noto::query_result>(reset_statement_after_execute_failure(statement, database, status));
+      }
+      continue;
+    }
+    if (code == SQLITE_DONE) {
+      result.changes = sqlite3_changes64(database);
+      result.last_insert_rowid = sqlite3_last_insert_rowid(database);
+      return result;
+    }
+    return failure_from<noto::query_result>(noto::result_from(code, database, "execute statement"));
+  }
+}
+
 auto trailing_sql_is_empty(std::char_t const *tail) -> std::bool_t {
   if (tail == nullptr) {
     return true;
@@ -937,29 +976,7 @@ auto execute_prepared(sqlite3 *database, std::string const &sql, std::vector<not
   if (!bound) {
     return failure_from<noto::query_result>(bound);
   }
-  auto result = noto::query_result{};
+  return collect_result(statement.get(), database, options);
-  auto result_bytes = std::size_t{};
-  result.columns = column_names(statement.get());
-  auto status = track_result_columns(result.columns, result_bytes, options);
-  if (!status) {
-    return failure_from<noto::query_result>(status);
-  }
-  while (true) {
-    auto code = sqlite3_step(statement.get());
-    if (code == SQLITE_ROW) {
-      status = append_result_row(result, current_row(statement.get()), result_bytes, options);
-      if (!status) {
-        return failure_from<noto::query_result>(status);
-      }
-      continue;
-    }
-    if (code == SQLITE_DONE) {
-      result.changes = sqlite3_changes64(database);
-      result.last_insert_rowid = sqlite3_last_insert_rowid(database);
-      return result;
-    }
-    return failure_from<noto::query_result>(noto::result_from(code, database, "execute statement"));
-  }
 }
 
 auto hex_literal(std::byte const *data, std::size_t size) -> std::string {
@@ -1896,15 +1913,7 @@ auto noto::detail::statement_core::async_reset(std::vector<noto::sqlite_value> p
   if (!ready) {
     co_return ready;
   }
-  auto code = sqlite3_reset(statement);
+  auto status = reset_and_bind(statement, owner->database, params);
-  auto status = noto::result_from(code, owner->database, "reset statement");
-  if (status) {
-    code = sqlite3_clear_bindings(statement);
-    status = noto::result_from(code, owner->database, "clear statement bindings");
-  }
-  if (status) {
-    status = bind_values(statement, params, owner->database);
-  }
   co_return finish_operation(*owner, status);
 }
 
@@ -1948,49 +1957,19 @@ auto noto::detail::statement_core::async_execute(std::vector<noto::sqlite_value>
   if (!ready) {
     co_return std::unexpected(ready.error());
   }
-  auto code = sqlite3_reset(statement);
+  auto status = reset_and_bind(statement, owner->database, params);
-  auto status = noto::result_from(code, owner->database, "reset statement");
-  if (status) {
-    code = sqlite3_clear_bindings(statement);
-    status = noto::result_from(code, owner->database, "clear statement bindings");
-  }
-  if (status) {
-    status = bind_values(statement, params, owner->database);
-  }
   if (!status) {
     status = finish_operation(*owner, status);
     co_return std::unexpected(status.error());
   }
-  auto result = noto::query_result{};
+  auto result = collect_result(statement, owner->database, options);
-  auto result_bytes = std::size_t{};
+  if (!result) {
-  result.columns = column_names(statement);
+    status = finish_operation(*owner, std::unexpected(result.error()));
-  status = track_result_columns(result.columns, result_bytes, options);
-  if (!status) {
-    status = reset_statement_after_execute_failure(statement, owner->database, status);
-    status = finish_operation(*owner, status);
     co_return std::unexpected(status.error());
   }
-  while (true) {
-    code = sqlite3_step(statement);
-    if (code == SQLITE_ROW) {
-      status = append_result_row(result, current_row(statement), result_bytes, options);
-      if (!status) {
-        status = reset_statement_after_execute_failure(statement, owner->database, status);
-        status = finish_operation(*owner, status);
-        co_return std::unexpected(status.error());
-      }
-      continue;
-    }
-    if (code == SQLITE_DONE) {
-      result.changes = sqlite3_changes64(owner->database);
-      result.last_insert_rowid = sqlite3_last_insert_rowid(owner->database);
   static_cast<void>(finish_operation(*owner, {}));
   co_return result;
 }
-    status = finish_operation(*owner, noto::result_from(code, owner->database, "execute statement"));
-    co_return std::unexpected(status.error());
-  }
-}
 
 auto noto::detail::statement_core::async_finalize() -> boost::asio::awaitable<std::expected<void, modulus::error>> {
   auto owner = connection;

src/errors.cpp

@@ -30,7 +30,7 @@ auto code_from(std::int32_t code) -> std::int32_t {
   case SQLITE_LOCKED:
     return noto::detail::error_unavailable;
   case SQLITE_INTERRUPT:
-    return noto::detail::error_cancelled;
+    return noto::detail::error_canceled;
   case SQLITE_AUTH:
   case SQLITE_PERM:
   case SQLITE_READONLY:

tests/test.cpp

@@ -679,6 +679,31 @@ BOOST_AUTO_TEST_CASE(noto_migrations_apply_validate_and_rollback) {
   BOOST_TEST(static_cast<std::bool_t>(missing->rows.empty()));
 }
 
+BOOST_AUTO_TEST_CASE(noto_direct_and_prepared_queries_share_limits_and_binding_rules) {
+  auto io = boost::asio::io_context{};
+  auto connection = noto::connection{io.get_executor(), noto::memory_connection_config()};
+  BOOST_REQUIRE(run(io, connection.async_connect()));
+  auto sql = std::string{"SELECT ? AS value UNION ALL SELECT 'second'"};
+  auto statement = run(io, connection.async_prepare(sql));
+  BOOST_REQUIRE(statement);
+  for (auto limit : {std::size_t{0}, std::size_t{1}, std::size_t{8}, std::size_t{100}}) {
+    auto options = noto::execute_options{};
+    options.max_result_bytes = limit;
+    auto direct = run(io, connection.async_execute(sql, {"first"}, options));
+    auto reusable = run(io, statement->async_execute({"first"}, options));
+    BOOST_REQUIRE(direct.has_value() == reusable.has_value());
+    if (!direct) {
+      BOOST_TEST(direct.error().code() == reusable.error().code());
+    }
+    auto recovered = run(io, statement->async_execute({"replacement"}));
+    BOOST_REQUIRE(recovered);
+    BOOST_TEST(recovered->rows.size() == 2U);
+    BOOST_TEST(std::get<std::string>(recovered->rows[0][0].value) == "replacement");
+  }
+  BOOST_REQUIRE(run(io, statement->async_finalize()));
+  BOOST_REQUIRE(run(io, connection.async_close()));
+}
+
 BOOST_AUTO_TEST_CASE(noto_backup_export_integrity_and_pragmas_work) {
   auto io = boost::asio::io_context{};
   auto source_path = temp_database_path("source");