diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 2b275d9..66f6a5a 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -47,7 +47,7 @@ jobs: key: conan-${{ runner.os }}-${{ hashFiles('conanfile.py', 'modulus.json') }} restore-keys: conan-${{ runner.os }}- - name: Configure native release - run: python3 modulus.py configure --target x86_64_gcc12_release + run: python3 modulus.py configure --target x86_64_gcc12_release --update - name: Check code style run: | python3 -m pip install --no-cache-dir -r build/build_tools/docker/format-requirements.txt @@ -63,4 +63,4 @@ jobs: CONAN_TOKEN: ${{ secrets.CONAN_TOKEN }} NOTO_CONAN_UPLOAD: "1" MODULUS_ALLOW_PACKAGE_REPLACE: "1" - run: python3 modulus.py package --all --replace + run: python3 modulus.py package --all --replace --update diff --git a/conanfile.py b/conanfile.py index b448377..09393ba 100644 --- a/conanfile.py +++ b/conanfile.py @@ -28,8 +28,11 @@ class noto_conan(conan.ConanFile): "fPIC": True, "with_tests": False, } - exports = "modulus.json" + exports = "modulus.json", "!build", "!.git", "!test_package/build" exports_sources = ( + "!build", + "!.git", + "!test_package/build", "CMakeLists.txt", "cmake/*.in", "include/*.hpp", diff --git a/include/noto/detail/errors.hpp b/include/noto/detail/errors.hpp index d758123..081018f 100644 --- a/include/noto/detail/errors.hpp +++ b/include/noto/detail/errors.hpp @@ -15,7 +15,7 @@ constexpr std::int32_t error_invalid_argument{1}; constexpr std::int32_t error_not_found{2}; constexpr std::int32_t error_unavailable{3}; constexpr std::int32_t error_timeout{4}; -constexpr std::int32_t error_cancelled{5}; +constexpr std::int32_t error_canceled{5}; constexpr std::int32_t error_unauthenticated{6}; constexpr std::int32_t error_permission_denied{7}; constexpr std::int32_t error_already_exists{8}; diff --git a/modulus.py b/modulus.py index 1b69392..856cec7 100755 --- a/modulus.py +++ b/modulus.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# Canonical consumer bootstrap. Copy this file unchanged to a project's modulus.py. import json import os import pathlib @@ -17,6 +18,7 @@ COMMANDS = ( "lsp", "clangd", "format", + "run", ) COMMON_DIR = pathlib.Path("build") / "build_common" TOOLS_DIR = pathlib.Path("build") / "build_tools" @@ -66,13 +68,6 @@ def runtime_ready(root): and (root / RUNTIME_DIR / "modulus" / "format_layout.py").is_file() ) -def optional_credentials(): - user = os.environ.get("CONAN_USER") - token = os.environ.get("CONAN_TOKEN") - if bool(user) != bool(token): - raise SystemExit("CONAN_USER and CONAN_TOKEN must either both be set or both be unset") - return user, token - def env_flag(name): return os.environ.get(name, "").lower() in {"1", "true", "yes", "on"} @@ -128,7 +123,7 @@ RUN python -m pip install --no-cache-dir conan=={CONAN_VERSION} WORKDIR /workspace """ -def ensure_fetch_image(): +def ensure_fetch_image(offline=False): inspected = subprocess.run( [ "docker", @@ -140,12 +135,15 @@ def ensure_fetch_image(): stderr=subprocess.DEVNULL, check=False, ) - if inspected.returncode == 0: + if inspected.returncode == 0 and (offline or not docker_platform_args()): return + if offline: + raise SystemExit('offline bootstrap requires the existing Modulus fetch image or MODULUS_DIRECT=1 with Conan installed') run( [ "docker", "build", + *docker_platform_args(), "--tag", FETCH_IMAGE, "-", @@ -156,6 +154,10 @@ def ensure_fetch_image(): def in_container(): return pathlib.Path("/.dockerenv").exists() +def docker_platform_args(): + platform = os.environ.get("MODULUS_DOCKER_PLATFORM", "").strip() + return ["--platform", platform] if platform else [] + def docker_workspace_args(root): if in_container(): container = os.environ.get("HOSTNAME", "").strip() @@ -172,18 +174,24 @@ def docker_workspace_args(root): def fetch_env(root): env = os.environ.copy() + for name in list(env): + if name in {"CONAN_USER", "CONAN_TOKEN"} or name.startswith(("CONAN_LOGIN_USERNAME", "CONAN_PASSWORD")): + del env[name] env["CONAN_HOME"] = str(root / CONAN_HOME_DIR) + env["CONAN_NON_INTERACTIVE"] = "1" return env -def run_fetch(root, command, capture=False, display_command=None, check=True): +def run_fetch(root, command, capture=False, display_command=None, check=True, offline=False): if direct_mode(root): return run(command, capture=capture, display_command=display_command, env=fetch_env(root), check=check) - ensure_fetch_image() + ensure_fetch_image(offline=offline) docker_command = [ "docker", "run", "--rm", "--interactive", + *(["--pull=never", "--network=none"] if offline else []), + *docker_platform_args(), "--user", f"{os.getuid()}:{os.getgid()}", "--env", @@ -191,9 +199,7 @@ def run_fetch(root, command, capture=False, display_command=None, check=True): "--env", "CONAN_HOME", "--env", - "CONAN_USER", - "--env", - "CONAN_TOKEN", + "CONAN_NON_INTERACTIVE", *docker_workspace_args(root), "--workdir", str(root), @@ -203,7 +209,6 @@ def run_fetch(root, command, capture=False, display_command=None, check=True): return run(docker_command, capture=capture, display_command=display_command, env=fetch_env(root), check=check) def configure_remote(root, remote_name, remote_url): - user, token = optional_credentials() run_fetch( root, [ @@ -215,29 +220,7 @@ def configure_remote(root, remote_name, remote_url): remote_url, ], ) - if not user and not token: - return - run_fetch( - root, - [ - "conan", - "remote", - "login", - remote_name, - user, - "-p", - token, - ], - display_command=[ - "conan", - "remote", - "login", - remote_name, - user, - "-p", - "", - ], - ) + run_fetch(root, ["conan", "remote", "logout", remote_name]) def package_refs(data): refs = [] @@ -274,46 +257,26 @@ def package_refs(data): refs.append(f"{PACKAGE_REF}#{recipe_revision}:{package_id}") return sorted(refs) -def remove_cached_modulus(root): - run_fetch( - root, - [ - "conan", - "remove", - PACKAGE_REF + ":*", - "-c", - ], - check=False, - ) - -def download_modulus(root, remote_name): - remove_cached_modulus(root) - run_fetch( - root, - [ - "conan", - "download", - PACKAGE_REF + ":*", - "-r", - remote_name, - ], - ) +def find_modulus(root, remote_name=None): completed = run_fetch( root, [ "conan", "list", - PACKAGE_REF + ":*", - "-c", + PACKAGE_REF + "#latest:*#latest", + *(["-r", remote_name] if remote_name else ["-c"]), "-f", "json", ], capture=True, + offline=remote_name is None, ) refs = package_refs(json.loads(completed.stdout)) if not refs: - raise SystemExit(f"no downloaded {PACKAGE_REF} package found in the Conan cache") + raise SystemExit(f"no {PACKAGE_REF} package available in {remote_name or 'the local Conan cache'}") package_ref = refs[-1] + if remote_name: + run_fetch(root, ["conan", "download", package_ref, "-r", remote_name]) completed = run_fetch( root, [ @@ -323,24 +286,25 @@ def download_modulus(root, remote_name): package_ref, ], capture=True, + offline=remote_name is None, ) package_root = pathlib.Path(completed.stdout.strip()) if not (package_root / "modulus" / "export.py").is_file() or not (package_root / "tools").is_dir(): raise SystemExit(f"downloaded {PACKAGE_REF} package does not contain Modulus export files: {package_root}") return package_root -def materialize_runtime(root): +def materialize_runtime(root, no_remote=False): remote_name, remote_url = remote_config(root) - configure_remote(root, remote_name, remote_url) - package_root = download_modulus(root, remote_name) + if not no_remote: + configure_remote(root, remote_name, remote_url) + package_root = find_modulus(root, None if no_remote else remote_name) sys.path.insert(0, str(package_root)) import modulus.export modulus.export.sync_project(root, package_root) -def ensure_runtime(root): - if runtime_ready(root): - return - materialize_runtime(root) +def ensure_runtime(root, update=False, no_remote=False): + if update or not runtime_ready(root): + materialize_runtime(root, no_remote=no_remote) def delegate(command, argv): root = repo_root() @@ -349,7 +313,7 @@ def delegate(command, argv): print(f"Modulus {command} tooling is missing; run ./modulus.py configure first", file=sys.stderr) return 1 else: - ensure_runtime(root) + ensure_runtime(root, update=command == "configure" and "--update" in argv, no_remote="--no-remote" in argv) sys.path.insert(0, str(root / RUNTIME_DIR)) import modulus.cli previous_argv = sys.argv diff --git a/src/detail/core.cpp b/src/detail/core.cpp index 88a387a..496a3e4 100644 --- a/src/detail/core.cpp +++ b/src/detail/core.cpp @@ -767,6 +767,45 @@ auto reset_statement_after_execute_failure(sqlite3_stmt *statement, sqlite3 *dat return reset ? status : reset; } +auto reset_and_bind(sqlite3_stmt *statement, sqlite3 *database, std::vector const ¶ms) -> std::expected { + auto code = sqlite3_reset(statement); + auto status = noto::result_from(code, database, "reset statement"); + if (status) { + code = sqlite3_clear_bindings(statement); + status = noto::result_from(code, database, "clear statement bindings"); + } + if (status) { + status = bind_values(statement, params, database); + } + return status; +} + +auto collect_result(sqlite3_stmt *statement, sqlite3 *database, noto::execute_options const &options) -> std::expected { + auto result = noto::query_result{}; + auto result_bytes = std::size_t{}; + result.columns = column_names(statement); + auto status = track_result_columns(result.columns, result_bytes, options); + if (!status) { + return failure_from(reset_statement_after_execute_failure(statement, database, status)); + } + while (true) { + auto code = sqlite3_step(statement); + if (code == SQLITE_ROW) { + status = append_result_row(result, current_row(statement), result_bytes, options); + if (!status) { + return failure_from(reset_statement_after_execute_failure(statement, database, status)); + } + continue; + } + if (code == SQLITE_DONE) { + result.changes = sqlite3_changes64(database); + result.last_insert_rowid = sqlite3_last_insert_rowid(database); + return result; + } + return failure_from(noto::result_from(code, database, "execute statement")); + } +} + auto trailing_sql_is_empty(std::char_t const *tail) -> std::bool_t { if (tail == nullptr) { return true; @@ -937,29 +976,7 @@ auto execute_prepared(sqlite3 *database, std::string const &sql, std::vector(bound); } - auto result = noto::query_result{}; - auto result_bytes = std::size_t{}; - result.columns = column_names(statement.get()); - auto status = track_result_columns(result.columns, result_bytes, options); - if (!status) { - return failure_from(status); - } - while (true) { - auto code = sqlite3_step(statement.get()); - if (code == SQLITE_ROW) { - status = append_result_row(result, current_row(statement.get()), result_bytes, options); - if (!status) { - return failure_from(status); - } - continue; - } - if (code == SQLITE_DONE) { - result.changes = sqlite3_changes64(database); - result.last_insert_rowid = sqlite3_last_insert_rowid(database); - return result; - } - return failure_from(noto::result_from(code, database, "execute statement")); - } + return collect_result(statement.get(), database, options); } auto hex_literal(std::byte const *data, std::size_t size) -> std::string { @@ -1896,15 +1913,7 @@ auto noto::detail::statement_core::async_reset(std::vector p if (!ready) { co_return ready; } - auto code = sqlite3_reset(statement); - auto status = noto::result_from(code, owner->database, "reset statement"); - if (status) { - code = sqlite3_clear_bindings(statement); - status = noto::result_from(code, owner->database, "clear statement bindings"); - } - if (status) { - status = bind_values(statement, params, owner->database); - } + auto status = reset_and_bind(statement, owner->database, params); co_return finish_operation(*owner, status); } @@ -1948,48 +1957,18 @@ auto noto::detail::statement_core::async_execute(std::vector if (!ready) { co_return std::unexpected(ready.error()); } - auto code = sqlite3_reset(statement); - auto status = noto::result_from(code, owner->database, "reset statement"); - if (status) { - code = sqlite3_clear_bindings(statement); - status = noto::result_from(code, owner->database, "clear statement bindings"); - } - if (status) { - status = bind_values(statement, params, owner->database); - } + auto status = reset_and_bind(statement, owner->database, params); if (!status) { status = finish_operation(*owner, status); co_return std::unexpected(status.error()); } - auto result = noto::query_result{}; - auto result_bytes = std::size_t{}; - result.columns = column_names(statement); - status = track_result_columns(result.columns, result_bytes, options); - if (!status) { - status = reset_statement_after_execute_failure(statement, owner->database, status); - status = finish_operation(*owner, status); - co_return std::unexpected(status.error()); - } - while (true) { - code = sqlite3_step(statement); - if (code == SQLITE_ROW) { - status = append_result_row(result, current_row(statement), result_bytes, options); - if (!status) { - status = reset_statement_after_execute_failure(statement, owner->database, status); - status = finish_operation(*owner, status); - co_return std::unexpected(status.error()); - } - continue; - } - if (code == SQLITE_DONE) { - result.changes = sqlite3_changes64(owner->database); - result.last_insert_rowid = sqlite3_last_insert_rowid(owner->database); - static_cast(finish_operation(*owner, {})); - co_return result; - } - status = finish_operation(*owner, noto::result_from(code, owner->database, "execute statement")); + auto result = collect_result(statement, owner->database, options); + if (!result) { + status = finish_operation(*owner, std::unexpected(result.error())); co_return std::unexpected(status.error()); } + static_cast(finish_operation(*owner, {})); + co_return result; } auto noto::detail::statement_core::async_finalize() -> boost::asio::awaitable> { diff --git a/src/errors.cpp b/src/errors.cpp index 8bdbfb4..27505ac 100644 --- a/src/errors.cpp +++ b/src/errors.cpp @@ -30,7 +30,7 @@ auto code_from(std::int32_t code) -> std::int32_t { case SQLITE_LOCKED: return noto::detail::error_unavailable; case SQLITE_INTERRUPT: - return noto::detail::error_cancelled; + return noto::detail::error_canceled; case SQLITE_AUTH: case SQLITE_PERM: case SQLITE_READONLY: diff --git a/tests/test.cpp b/tests/test.cpp index 09d0e04..188be1a 100644 --- a/tests/test.cpp +++ b/tests/test.cpp @@ -679,6 +679,31 @@ BOOST_AUTO_TEST_CASE(noto_migrations_apply_validate_and_rollback) { BOOST_TEST(static_cast(missing->rows.empty())); } +BOOST_AUTO_TEST_CASE(noto_direct_and_prepared_queries_share_limits_and_binding_rules) { + auto io = boost::asio::io_context{}; + auto connection = noto::connection{io.get_executor(), noto::memory_connection_config()}; + BOOST_REQUIRE(run(io, connection.async_connect())); + auto sql = std::string{"SELECT ? AS value UNION ALL SELECT 'second'"}; + auto statement = run(io, connection.async_prepare(sql)); + BOOST_REQUIRE(statement); + for (auto limit : {std::size_t{0}, std::size_t{1}, std::size_t{8}, std::size_t{100}}) { + auto options = noto::execute_options{}; + options.max_result_bytes = limit; + auto direct = run(io, connection.async_execute(sql, {"first"}, options)); + auto reusable = run(io, statement->async_execute({"first"}, options)); + BOOST_REQUIRE(direct.has_value() == reusable.has_value()); + if (!direct) { + BOOST_TEST(direct.error().code() == reusable.error().code()); + } + auto recovered = run(io, statement->async_execute({"replacement"})); + BOOST_REQUIRE(recovered); + BOOST_TEST(recovered->rows.size() == 2U); + BOOST_TEST(std::get(recovered->rows[0][0].value) == "replacement"); + } + BOOST_REQUIRE(run(io, statement->async_finalize())); + BOOST_REQUIRE(run(io, connection.async_close())); +} + BOOST_AUTO_TEST_CASE(noto_backup_export_integrity_and_pragmas_work) { auto io = boost::asio::io_context{}; auto source_path = temp_database_path("source");