product-vamp/tests/test_catalog_runtime.cpp

#include <boost/test/unit_test.hpp>

#include <cstddef>
#include <cstdint>
#include <cstdlib>
#include <expected>
#include <filesystem>
#include <fstream>
#include <future>
#include <memory>
#include <optional>
#include <string>
#include <string_view>
#include <system_error>
#include <utility>
#include <variant>
#include <vector>

#include <boost/asio/co_spawn.hpp>
#include <boost/asio/io_context.hpp>
#include <boost/asio/use_future.hpp>
#include <boost/json.hpp>

#include <noto.hpp>

#include <vamp_zero/database.hpp>
#include <vamp_zero/errors.hpp>
#include <vamp_zero/library.hpp>
#include <vamp_zero/navigation.hpp>
#include <vamp_zero/control.hpp>

#include "catalog_fixture.hpp"

namespace {

template <typename T>
concept retains_score_json = requires(T value) { value.source_json; };

static_assert(!retains_score_json<vamp_zero::library_part>);
static_assert(!retains_score_json<vamp_zero::catalog>);

struct catalog_directory {
  std::filesystem::path path;
  catalog_directory() {
    auto pattern = (std::filesystem::temp_directory_path() / "vamp_catalog-XXXXXX").string();
    BOOST_REQUIRE(::mkdtemp(pattern.data()) != nullptr);
    path = pattern;
  }
  ~catalog_directory() {
    auto ignored = std::error_code{};
    std::filesystem::remove_all(path, ignored);
  }
};

template <typename Function, typename... Arguments>
auto run_catalog_operation(Function function, Arguments... arguments) {
  auto io = boost::asio::io_context{};
  auto future = boost::asio::co_spawn(io, function(io.get_executor(), std::move(arguments)...), boost::asio::use_future);
  io.run();
  return future.get();
}

auto catalog_query(boost::asio::any_io_executor executor, vamp_zero::database_config config, std::string sql, std::vector<noto::sqlite_value> params)
  -> boost::asio::awaitable<std::expected<noto::query_result, modulus::error>> {
  auto connection = noto::connection{executor, noto::file_connection_config(config.path, "vamp_catalog-test")};
  auto connected = co_await connection.async_connect();
  if (!connected) {
    co_return std::unexpected(connected.error());
  }
  auto result = co_await connection.async_execute(std::move(sql), std::move(params));
  auto closed = co_await connection.async_close();
  if (!result) {
    co_return std::unexpected(result.error());
  }
  if (!closed) {
    co_return std::unexpected(closed.error());
  }
  co_return result;
}

auto catalog_bundle(std::string_view title, std::string_view key = "C", std::string_view part = "Lead") -> std::string {
  auto value = boost::json::parse(R"json({
    "music":{"metadata":{"title":"Fixture","composer":"Composer","key_signature":"C","time_signature":"4/4"},
      "staves":[{"name":"Lead","measures":[{"voices":[]},{"voices":[]},{"voices":[]}]}]},
    "presentation":{"width":976,"height":484,"title_image":"0000000000000000000000000000000000000000000000000000000000000000","pages":[
      {"path":"fixture.pages/1.png","sha256":"0000000000000000000000000000000000000000000000000000000000000000"},
      {"path":"fixture.pages/2.png","sha256":"1111111111111111111111111111111111111111111111111111111111111111"}]}
  })json");
  auto &metadata = value.at("music").at("metadata").as_object();
  metadata["title"] = title;
  metadata["key_signature"] = key;
  value.at("music").at("staves").as_array().front().as_object()["name"] = part;
  vamp_test::add_timing(value);
  return boost::json::serialize(value);
}

struct catalog_database {
  catalog_directory directory;
  std::filesystem::path assets{directory.path / "assets"};
  vamp_zero::database_config config{.path = (directory.path / "catalog.db").string()};
  catalog_database() {
    std::filesystem::create_directories(assets / "songs");
  }
  auto query(std::string sql, std::vector<noto::sqlite_value> params = {}) const -> noto::query_result {
    auto result = run_catalog_operation(catalog_query, config, std::move(sql), std::move(params));
    BOOST_REQUIRE_MESSAGE(result, (result ? "" : result.error().describe()));
    return std::move(*result);
  }
  auto rows(std::string const &table, std::string const &where = "") const -> std::vector<std::string> {
    auto result = query("SELECT json_array(id, title, composer, key_signature, source_path, source_json, "
                        "measure_count, total_pages) FROM " +
                        table + where + " ORDER BY id");
    auto rows = std::vector<std::string>{};
    for (auto const &row : result.rows) {
      rows.push_back(std::get<std::string>(row[0].value));
    }
    return rows;
  }
  auto write(std::filesystem::path const &relative, std::string const &text) const -> void {
    auto path = assets / relative;
    std::filesystem::create_directories(path.parent_path());
    auto output = std::ofstream{path, std::ios::binary | std::ios::trunc};
    output << text;
    output.close();
    BOOST_REQUIRE(output.good());
  }
  auto build() const -> std::bool_t {
    return vamp_test::build_catalog(assets, config.path);
  }
  auto performance() const -> std::expected<vamp_zero::catalog, modulus::error> {
    return run_catalog_operation(vamp_zero::async_load_catalog, config, assets.string());
  }
  auto score(std::string id) const -> std::expected<vamp_zero::score_document, modulus::error> {
    return run_catalog_operation(vamp_zero::async_load_score, config, vamp_zero::score_reference{std::move(id)}, assets.string());
  }
};

}

BOOST_AUTO_TEST_CASE(catalog_snapshots_heartbeat_and_navigation_share_immutable_metadata) {
  auto performance = vamp_zero::group_ordered_parts({
    {.id = "other", .title = "A Tune", .composer = "Composer", .part = "Lead", .total_pages = 1},
    {.id = "chosen", .title = "B Tune", .composer = "Composer", .part = "Lead", .total_pages = 2},
  });
  auto catalog = std::make_shared<vamp_zero::catalog const>(std::move(performance));
  auto original = vamp_zero::app_state{
    .catalog = catalog,
    .mode = vamp_zero::app_mode::PERFORMANCE_PART_MENU,
    .selected_song_index = 2,
    .selected_part_index = 1,
  };
  auto control = vamp_zero::control_state{{}, original};
  auto observed = std::vector<vamp_zero::app_state>{};
  control.set_observer([&](vamp_zero::app_state state) { observed.push_back(std::move(state)); });
  auto before = control.snapshot();
  BOOST_TEST(before.catalog.get() == catalog.get());
  BOOST_TEST(vamp_zero::selected_part(before) == &catalog->parts[1]);
  auto heartbeat = control.heartbeat();
  BOOST_TEST(heartbeat.catalog.get() == catalog.get());
  BOOST_TEST(heartbeat.input_revision == before.input_revision);
  BOOST_TEST(heartbeat.pico_connected);
  auto playing = control.apply(vamp_protocol::page_command::SELECT);
  BOOST_TEST(vamp_zero::to_string(playing.mode) == "performance_score");
  BOOST_TEST(vamp_zero::active_part(playing) == &catalog->parts[1]);
  BOOST_TEST(control.apply(vamp_protocol::page_command::NEXT).page_index == 1U);
  BOOST_TEST(control.apply(vamp_protocol::page_command::PREVIOUS).page_index == 0U);
  control.apply(vamp_protocol::page_command::SELECT);
  BOOST_TEST(control.snapshot().catalog.get() == catalog.get());
  BOOST_TEST(!before.pico_connected);
  BOOST_TEST(before.input_revision == 0U);
  BOOST_TEST(before.active_part_id.empty());
  BOOST_TEST(original.catalog.get() == catalog.get());
  BOOST_REQUIRE(!observed.empty());
  for (auto const &snapshot : observed) {
    BOOST_TEST(snapshot.catalog.get() == catalog.get());
  }
}

BOOST_AUTO_TEST_CASE(catalog_performance_groups_use_title_and_composer_with_part_indices) {
  auto library = vamp_zero::group_ordered_parts({
    {.id = "other-title", .title = "Other Title", .composer = "Ada", .part = "Lead"},
    {.id = "guitar", .title = "Shared Title", .composer = "Ada", .part = "Guitar"},
    {.id = "vocal", .title = "Shared Title", .composer = "Ada", .part = "Vocal"},
    {.id = "other-composer", .title = "Shared Title", .composer = "Bea", .part = "Lead"},
  });
  BOOST_REQUIRE_EQUAL(library.parts.size(), 4U);
  BOOST_REQUIRE_EQUAL(library.groups.size(), 3U);
  BOOST_TEST(library.groups[0].title == "Other Title");
  BOOST_TEST(library.groups[1].title == "Shared Title");
  BOOST_TEST(library.groups[1].composer == "Ada");
  BOOST_TEST(library.groups[2].composer == "Bea");
  BOOST_REQUIRE_EQUAL(library.groups[1].parts.size(), 2U);
  auto seen = std::vector<std::uint32_t>(library.parts.size());
  for (auto const &group : library.groups) {
    for (auto index : group.parts) {
      BOOST_REQUIRE_LT(index, library.parts.size());
      auto const &part = library.parts[index];
      BOOST_TEST(part.title == group.title);
      BOOST_TEST(part.composer == group.composer);
      ++seen[index];
    }
  }
  for (auto count : seen) {
    BOOST_TEST(count == 1U);
  }
  BOOST_TEST(library.parts[library.groups[1].parts[0]].id == "guitar");
  BOOST_TEST(library.parts[library.groups[1].parts[1]].id == "vocal");
}

BOOST_AUTO_TEST_CASE(catalog_individual_score_lookup_is_parameterized_and_reports_missing_or_invalid) {
  auto db = catalog_database{};
  auto song = catalog_bundle("Song", "F");
  db.write("songs/shared.json", song);
  BOOST_REQUIRE(db.build());
  std::filesystem::remove_all(db.assets);
  auto document = db.score("shared");
  BOOST_REQUIRE(document);
  BOOST_TEST(document->id == "shared");
  BOOST_TEST(document->title == "Song");
  BOOST_TEST(document->key_signature == "F");
  BOOST_TEST(document->source_json == song);
  BOOST_TEST(document->source_path == (db.assets / "songs/shared.json").string());
  BOOST_TEST(document->measure_count == 3U);
  BOOST_REQUIRE_EQUAL(document->pages.size(), 2U);
  BOOST_TEST(document->pages[1].path == "fixture.pages/2.png");
  for (auto id : {"missing", "shared' OR 1=1 --"}) {
    auto missing = db.score(id);
    BOOST_REQUIRE(!missing);
    BOOST_TEST(missing.error().code() == static_cast<std::int32_t>(vamp_zero::error_code::NOT_FOUND));
  }
  db.query("UPDATE songs SET source_json = '{}' WHERE id = 'shared'");
  auto invalid = db.score("shared");
  BOOST_REQUIRE(!invalid);
  BOOST_TEST(invalid.error().code() == static_cast<std::int32_t>(vamp_zero::error_code::INVALID_ARGUMENT));
}

BOOST_AUTO_TEST_CASE(catalog_query_orders_and_groups_by_exact_song_identity) {
  auto db = catalog_database{};
  for (auto const &id : {"upper-bea", "upper-vocal", "upper-acoustic", "lower", "upper-acoustic-second"}) {
    db.write(std::filesystem::path{"songs"} / (std::string{id} + ".json"), catalog_bundle("Same"));
  }
  BOOST_REQUIRE(db.build());
  db.query("UPDATE songs SET composer = 'Ada', part = 'Acoustic'");
  db.query("UPDATE songs SET composer = 'Bea' WHERE id = 'upper-bea'");
  db.query("UPDATE songs SET part = 'Vocal' WHERE id = 'upper-vocal'");
  db.query("UPDATE songs SET title = 'same' WHERE id = 'lower'");
  auto result = db.performance();
  BOOST_REQUIRE(result);
  BOOST_REQUIRE_EQUAL(result->groups.size(), 3U);
  auto ids = std::vector<std::string>{};
  for (auto const &part : result->parts) {
    ids.push_back(part.id);
  }
  auto expected = std::vector<std::string>{"upper-acoustic", "upper-acoustic-second", "upper-vocal", "upper-bea", "lower"};
  BOOST_TEST(ids == expected, boost::test_tools::per_element());
  BOOST_TEST(result->groups[0].composer == "Ada");
  BOOST_TEST(result->groups[0].parts.size() == 3U);
  BOOST_TEST(result->groups[1].composer == "Bea");
  BOOST_TEST(result->groups[2].title == "same");
}

BOOST_AUTO_TEST_CASE(catalog_sequential_score_loads_keep_executor_stack_bounded) {
  auto db = catalog_database{};
  db.write("songs/repeated.json", catalog_bundle("Repeated"));
  BOOST_REQUIRE(db.build());
  auto repeated_loads = [](boost::asio::any_io_executor executor, vamp_zero::database_config config, std::string assets)
    -> boost::asio::awaitable<std::expected<std::uint32_t, modulus::error>> {
    auto reference = vamp_zero::score_reference{.id = "repeated"};
    for (auto count = std::uint32_t{}; count < 4096U; ++count) {
      auto score = co_await vamp_zero::async_load_score(executor, config, reference, assets);
      if (!score) {
        co_return std::unexpected(score.error());
      }
    }
    co_return 4096U;
  };
  auto result = run_catalog_operation(repeated_loads, db.config, db.assets.string());
  BOOST_REQUIRE_MESSAGE(result, (result ? "" : result.error().describe()));
  BOOST_TEST(*result == 4096U);
}

BOOST_AUTO_TEST_CASE(catalog_reload_uses_cached_metadata_without_parsing_score_json) {
  auto db = catalog_database{};
  BOOST_REQUIRE(db.build());
  // Complete cached metadata must not require parsing or retaining the score body.
  db.query("INSERT INTO songs(id, title, composer, source_json, part, source_path, "
           "key_signature, measure_count, total_pages, title_image) "
           "VALUES('tune', 'Metadata Song', 'Composer', '{\"presentation\":{\"playback_pages\":[{},{},{}]}}', 'Lead', 'songs/tune.json', 'F', 3, 2, '0000000000000000000000000000000000000000000000000000000000000000')");
  auto songs = db.rows("songs");
  for (auto attempt = std::uint32_t{}; attempt < 2U; ++attempt) {
    auto performance = db.performance();
    BOOST_REQUIRE(performance);
    BOOST_REQUIRE_EQUAL(performance->parts.size(), 1U);
    BOOST_TEST(performance->parts[0].title == "Metadata Song");
    BOOST_TEST(performance->parts[0].key_signature == "F");
    BOOST_TEST(performance->parts[0].measure_count == 3U);
    BOOST_TEST(performance->parts[0].total_pages == 2U);
    BOOST_TEST(performance->parts[0].playback_pages == 3U);
  }
  BOOST_TEST(db.rows("songs") == songs, boost::test_tools::per_element());
  auto song = db.score("tune");
  BOOST_REQUIRE(!song);
  BOOST_TEST(song.error().code() == static_cast<std::int32_t>(vamp_zero::error_code::INVALID_ARGUMENT));
}

BOOST_AUTO_TEST_CASE(catalog_validation_rejects_missing_incompatible_and_corrupt_files_without_repair) {
  auto db = catalog_database{};
  auto validate = [&] { return run_catalog_operation(vamp_zero::async_validate_database, db.config, true); };
  BOOST_TEST(!validate());
  BOOST_TEST(!std::filesystem::exists(db.config.path));
  BOOST_REQUIRE(db.build());
  BOOST_REQUIRE(validate());
  auto songs = db.performance();
  BOOST_REQUIRE(songs);
  BOOST_TEST(songs->parts.empty());
  for (auto sql : {"ALTER TABLE songs RENAME COLUMN part TO obsolete_part", "DROP TABLE songs"}) {
    db.query(sql);
    auto snapshot_sql = "SELECT json_group_array(sql) FROM (SELECT sql FROM sqlite_schema ORDER BY name)";
    auto before = std::get<std::string>(db.query(snapshot_sql).rows[0][0].value);
    auto result = validate();
    BOOST_REQUIRE(!result);
    BOOST_TEST(result.error().message().find("reinstall") != std::string_view::npos);
    BOOST_TEST(std::get<std::string>(db.query(snapshot_sql).rows[0][0].value) == before);
    BOOST_REQUIRE(db.build());
  }
  db.query("PRAGMA application_id = 0");
  BOOST_TEST(!validate());
  auto output = std::ofstream{db.config.path, std::ios::trunc};
  output << "not a database";
  output.close();
  BOOST_TEST(!validate());
  BOOST_TEST(!std::filesystem::exists(db.config.path + "-wal"));
  BOOST_TEST(!std::filesystem::exists(db.config.path + "-journal"));
}

BOOST_AUTO_TEST_CASE(catalog_validation_checks_column_types_constraints_and_table_kind) {
  auto db = catalog_database{};
  auto validate = [&] { return run_catalog_operation(vamp_zero::async_validate_database, db.config, false); };
  for (auto change : {std::pair{"title TEXT NOT NULL", "title BLOB NOT NULL"}, {"title TEXT NOT NULL", "title TEXT"}, {"PRIMARY KEY", ""}, {" STRICT", ""}}) {
    BOOST_REQUIRE(db.build());
    auto sql = std::get<std::string>(db.query("SELECT sql FROM sqlite_schema WHERE name = 'songs'").rows[0][0].value);
    auto position = sql.find(change.first);
    BOOST_REQUIRE(position != std::string::npos);
    sql.replace(position, std::string_view{change.first}.size(), change.second);
    db.query("DROP TABLE songs");
    db.query(sql);
    auto timestamp = std::filesystem::last_write_time(db.config.path);
    BOOST_TEST(!validate());
    BOOST_TEST((std::filesystem::last_write_time(db.config.path) == timestamp));
  }
  BOOST_REQUIRE(db.build());
  db.query("ALTER TABLE songs RENAME TO underlying_songs");
  db.query("CREATE VIEW songs AS SELECT * FROM underlying_songs");
  BOOST_TEST(!validate());
}

BOOST_AUTO_TEST_CASE(catalog_reads_readonly_database_without_json_and_resolves_relocated_assets) {
  auto db = catalog_database{};
  db.write("songs/tune.json", catalog_bundle("Relocated"));
  BOOST_REQUIRE(db.build());
  std::filesystem::remove(db.assets / "songs/tune.json");
  auto moved = db.directory.path / "relocated";
  std::filesystem::rename(db.assets, moved);
  db.assets = moved;
  auto timestamp = std::filesystem::last_write_time(db.config.path);
  std::filesystem::permissions(db.config.path, std::filesystem::perms::owner_read);
  BOOST_REQUIRE(run_catalog_operation(vamp_zero::async_validate_database, db.config, true));
  auto songs = db.performance();
  auto score = db.score("tune");
  BOOST_REQUIRE(songs);
  BOOST_REQUIRE(score);
  BOOST_TEST(score->source_path == (moved / "songs/tune.json").string());
  BOOST_TEST(songs->parts[0].source_path == score->source_path);
  BOOST_TEST((std::filesystem::last_write_time(db.config.path) == timestamp));
  BOOST_TEST(!std::filesystem::exists(db.config.path + "-wal"));
  BOOST_TEST(!std::filesystem::exists(db.config.path + "-shm"));
  BOOST_TEST(!std::filesystem::exists(db.config.path + "-journal"));
}

BOOST_AUTO_TEST_CASE(catalog_rejects_paths_outside_the_asset_root) {
  auto db = catalog_database{};
  db.write("songs/tune.json", catalog_bundle("Fixture"));
  BOOST_REQUIRE(db.build());
  for (auto path : {"/tmp/tune.json", "../tune.json", "songs/../../tune.json", "songs/./tune.json", "songs\\tune.json", ""}) {
    if (std::string_view{path}.empty()) {
      continue;
    }
    db.query("UPDATE songs SET source_path = ?1", {noto::sqlite_value{std::string{path}}});
    BOOST_TEST(!db.performance());
    BOOST_TEST(!db.score("tune"));
  }
  std::filesystem::create_directory_symlink(db.directory.path, db.assets / "escape");
  db.query("UPDATE songs SET source_path = 'escape/tune.json'");
  BOOST_TEST(!db.performance());
  BOOST_TEST(!db.score("tune"));
}