khodo/product-vamp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Shared Catalog

Browse Source
This commit is contained in:
khodowany
2026-09-17 00:44:19 -05:00
parent aa1273b07d
commit 5941a4cd49
44 changed files with 3613 additions and 733 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/vamp_zero/database.cpp
+155 -37
View File
@@ -4,12 +4,15 @@
#include <filesystem>
#include <string>
#include <string_view>
#include <map>
#include <utility>
#include <variant>
#include <vector>
#include <boost/asio/any_io_executor.hpp>
#include <boost/asio/awaitable.hpp>
#include <boost/asio/post.hpp>
#include <boost/asio/use_awaitable.hpp>
#include <noto.hpp>
@@ -189,6 +192,18 @@ CREATE TABLE IF NOT EXISTS practice_exercises (
"NOCASE, lesson_group_id COLLATE NOCASE, title COLLATE NOCASE)",
"CREATE INDEX IF NOT EXISTS idx_practice_exercises_catalog_managed ON practice_exercises(catalog_managed)",
};
for (auto table : {"songs", "practice_exercises"}) {
for (auto column : {"measure_count", "total_pages", "authored_key_signature"}) {
if (!ensured) {
break;
}
auto type = std::string_view{column} == "authored_key_signature" ? " TEXT" : " INTEGER CHECK(" + std::string{column} + " > 0)";
ensured = co_await ensure_column(connection, table, column, "ALTER TABLE " + std::string{table} + " ADD COLUMN " + column + type);
}
if (ensured) {
ensured = co_await execute_schema_statement(connection, "CREATE TRIGGER IF NOT EXISTS invalidate_" + std::string{table} + "_metadata AFTER UPDATE OF source_json ON " + table + " WHEN NEW.source_json != OLD.source_json BEGIN UPDATE " + table + " SET measure_count = NULL, total_pages = NULL, authored_key_signature = NULL WHERE id = NEW.id; END");
}
}
for (auto &statement : index_statements) {
if (!ensured) {
break;
@@ -240,6 +255,39 @@ auto now_ms() -> std::int64_t {
.count();
}
auto backfill_metadata(noto::connection &connection, std::string const &table) -> boost::asio::awaitable<std::expected<void, modulus::error>> {
for (;;) {
auto row = co_await connection.async_execute("SELECT id, source_path, source_json FROM " + table + " WHERE measure_count IS NULL OR total_pages IS NULL OR authored_key_signature IS NULL LIMIT 1");
if (!row) {
co_return std::unexpected(row.error());
}
if (row->rows.empty()) {
co_return std::expected<void, modulus::error>{};
}
auto const &values = row->rows.front();
auto document = vamp_zero::parse_score_document(text_value(values[2]), text_value(values[1]));
if (!document) {
co_return std::unexpected(document.error());
}
auto params = std::vector<noto::sqlite_value>{
noto::sqlite_value{static_cast<std::int64_t>(document->measure_count)},
noto::sqlite_value{static_cast<std::int64_t>(document->pages.size())},
noto::sqlite_value{document->authored_key_signature},
noto::sqlite_value{text_value(values[0])},
noto::sqlite_value{text_value(values[2])},
};
auto updated = co_await connection.async_execute("UPDATE " + table + " SET measure_count = ?1, total_pages = ?2, authored_key_signature = ?3 WHERE id = ?4 AND source_json = ?5", std::move(params));
if (!updated) {
co_return std::unexpected(updated.error());
}
}
}
auto count_value(noto::sqlite_value const &value) -> std::size_t {
auto count = std::get_if<std::int64_t>(&value.value);
return count == nullptr || *count <= 0 ? 0U : static_cast<std::size_t>(*count);
}
auto execute_delete_stale_catalog_rows(noto::connection &connection, std::string_view table, std::vector<std::string> const &ids) -> boost::asio::awaitable<std::expected<void, modulus::error>> {
auto sql = std::string{"DELETE FROM "};
sql += table;
@@ -321,11 +369,11 @@ auto vamp_zero::async_ensure_database(boost::asio::any_io_executor executor, vam
}
auto vamp_zero::async_import_catalogs(boost::asio::any_io_executor executor, vamp_zero::database_config config, std::string assets_path) -> boost::asio::awaitable<std::expected<void, modulus::error>> {
auto performance = vamp_zero::load_performance_library(assets_path);
auto performance = vamp_zero::score_paths(assets_path, vamp_zero::score_category::PERFORMANCE);
if (!performance) {
co_return std::unexpected(performance.error());
}
auto practice = vamp_zero::load_practice_library(assets_path);
auto practice = vamp_zero::score_paths(assets_path, vamp_zero::score_category::PRACTICE);
if (!practice) {
co_return std::unexpected(practice.error());
}
@@ -341,11 +389,17 @@ auto vamp_zero::async_import_catalogs(boost::asio::any_io_executor executor, vam
auto imported = std::expected<void, modulus::error>{};
auto timestamp = now_ms();
auto song_ids = std::vector<std::string>{};
song_ids.reserve(performance->parts.size());
for (auto const &part : performance->parts) {
song_ids.reserve(performance->size());
for (auto const &path : *performance) {
if (!imported) {
break;
}
auto document = vamp_zero::read_score_document(path);
if (!document) {
imported = std::unexpected(document.error());
break;
}
auto const &part = *document;
song_ids.push_back(part.id);
auto params = std::vector<noto::sqlite_value>{
noto::sqlite_value{part.id},
@@ -357,11 +411,14 @@ auto vamp_zero::async_import_catalogs(boost::asio::any_io_executor executor, vam
noto::sqlite_value{part.part},
noto::sqlite_value{part.source_path},
noto::sqlite_value{part.key_signature},
noto::sqlite_value{static_cast<std::int64_t>(part.measure_count)},
noto::sqlite_value{static_cast<std::int64_t>(part.pages.size())},
noto::sqlite_value{part.authored_key_signature},
};
auto inserted = co_await connection.async_execute(
R"sql(
INSERT INTO songs(id, title, composer, source_json, created_at_ms, updated_at_ms, part, source_path, key_signature, catalog_managed)
VALUES(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, 1)
INSERT INTO songs(id, title, composer, source_json, created_at_ms, updated_at_ms, part, source_path, key_signature, catalog_managed, measure_count, total_pages, authored_key_signature)
VALUES(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, 1, ?10, ?11, ?12)
ON CONFLICT(id) DO UPDATE SET
title = excluded.title,
composer = excluded.composer,
@@ -370,6 +427,9 @@ ON CONFLICT(id) DO UPDATE SET
source_path = excluded.source_path,
key_signature = excluded.key_signature,
catalog_managed = 1,
measure_count = excluded.measure_count,
total_pages = excluded.total_pages,
authored_key_signature = excluded.authored_key_signature,
updated_at_ms = CASE
WHEN songs.title != excluded.title
OR songs.composer != excluded.composer
@@ -388,10 +448,27 @@ WHERE songs.catalog_managed = 1
}
}
auto practice_ids = std::vector<std::string>{};
for (auto const &exercise : practice->exercises) {
auto authored_keys = std::map<std::string, std::string>{};
for (auto const &path : *practice) {
if (!imported) {
break;
}
auto document = vamp_zero::read_score_document(path);
if (!document) {
imported = std::unexpected(document.error());
break;
}
auto metadata = vamp_zero::practice_metadata(*document, assets_path);
if (!metadata) {
imported = std::unexpected(metadata.error());
break;
}
auto const &exercise = *metadata;
auto [origin, added] = authored_keys.emplace(exercise.id, exercise.authored_key_signature);
if (!added && origin->second != exercise.authored_key_signature) {
imported = std::unexpected(vamp_zero::make_error(vamp_zero::error_code::INVALID_ARGUMENT, "practice variants disagree on authored key: " + exercise.id));
break;
}
for (auto const &variant : exercise.variants) {
practice_ids.push_back(variant.id);
auto params = std::vector<noto::sqlite_value>{
@@ -403,15 +480,18 @@ WHERE songs.catalog_managed = 1
noto::sqlite_value{exercise.title},
noto::sqlite_value{exercise.composer},
noto::sqlite_value{variant.key_signature},
noto::sqlite_value{variant.source_json},
noto::sqlite_value{document->source_json},
noto::sqlite_value{variant.source_path},
noto::sqlite_value{timestamp},
noto::sqlite_value{timestamp},
noto::sqlite_value{static_cast<std::int64_t>(variant.measure_count)},
noto::sqlite_value{static_cast<std::int64_t>(variant.total_pages)},
noto::sqlite_value{exercise.authored_key_signature},
};
auto inserted = co_await connection.async_execute(
R"sql(
INSERT INTO practice_exercises(id, practice_type_id, practice_type_title, lesson_group_id, lesson_group_title, title, composer, key_signature, source_json, source_path, created_at_ms, updated_at_ms, catalog_managed)
VALUES(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, 1)
INSERT INTO practice_exercises(id, practice_type_id, practice_type_title, lesson_group_id, lesson_group_title, title, composer, key_signature, source_json, source_path, created_at_ms, updated_at_ms, catalog_managed, measure_count, total_pages, authored_key_signature)
VALUES(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, 1, ?13, ?14, ?15)
ON CONFLICT(id) DO UPDATE SET
practice_type_id = excluded.practice_type_id,
practice_type_title = excluded.practice_type_title,
@@ -423,6 +503,9 @@ ON CONFLICT(id) DO UPDATE SET
source_json = excluded.source_json,
source_path = excluded.source_path,
catalog_managed = 1,
measure_count = excluded.measure_count,
total_pages = excluded.total_pages,
authored_key_signature = excluded.authored_key_signature,
updated_at_ms = CASE
WHEN practice_exercises.practice_type_id != excluded.practice_type_id
OR practice_exercises.practice_type_title != excluded.practice_type_title
@@ -482,8 +565,13 @@ auto vamp_zero::async_load_performance_library(boost::asio::any_io_executor exec
if (!connected) {
co_return std::unexpected(connected.error());
}
auto filled = co_await backfill_metadata(connection, "songs");
if (!filled) {
static_cast<void>(co_await connection.async_close());
co_return std::unexpected(filled.error());
}
auto selected =
co_await connection.async_prepare("SELECT id, title, composer, part, source_path, source_json, key_signature FROM "
co_await connection.async_prepare("SELECT id, title, composer, part, source_path, measure_count, key_signature, total_pages FROM "
"songs ORDER BY title COLLATE NOCASE, part COLLATE NOCASE");
if (!selected) {
auto close = co_await connection.async_close();
@@ -504,13 +592,8 @@ auto vamp_zero::async_load_performance_library(boost::asio::any_io_executor exec
break;
}
auto const &row = **next;
if (row.size() < 7U) {
continue;
}
auto source_json = text_value(row[5]);
auto parsed = vamp_zero::parse_score_document(source_json, text_value(row[4]));
if (!parsed) {
status = std::unexpected(parsed.error());
if (row.size() != 8U || count_value(row[5]) == 0U || count_value(row[7]) == 0U) {
status = std::unexpected(vamp_zero::make_error(vamp_zero::error_code::PERSISTENCE, "invalid performance metadata"));
break;
}
parts.push_back({
@@ -520,9 +603,8 @@ auto vamp_zero::async_load_performance_library(boost::asio::any_io_executor exec
.key_signature = text_value(row[6]),
.part = text_value(row[3]),
.source_path = text_value(row[4]),
.source_json = std::move(source_json),
.measure_count = parsed->measure_count,
.total_pages = vamp_zero::score_page_count(*parsed),
.measure_count = count_value(row[5]),
.total_pages = count_value(row[7]),
});
}
auto closed = co_await close_catalogue_query(connection, *selected, std::move(status));
@@ -539,8 +621,13 @@ auto vamp_zero::async_load_practice_library(boost::asio::any_io_executor executo
if (!connected) {
co_return std::unexpected(connected.error());
}
auto filled = co_await backfill_metadata(connection, "practice_exercises");
if (!filled) {
static_cast<void>(co_await connection.async_close());
co_return std::unexpected(filled.error());
}
auto selected = co_await connection.async_prepare("SELECT id, practice_type_id, practice_type_title, lesson_group_id, lesson_group_title, title, composer, "
"key_signature, source_path, source_json FROM practice_exercises ORDER BY practice_type_id COLLATE NOCASE, "
"key_signature, source_path, measure_count, total_pages, authored_key_signature FROM practice_exercises ORDER BY practice_type_id COLLATE NOCASE, "
"lesson_group_id COLLATE NOCASE, source_path COLLATE NOCASE, title COLLATE NOCASE, id COLLATE NOCASE");
if (!selected) {
auto close = co_await connection.async_close();
@@ -561,13 +648,8 @@ auto vamp_zero::async_load_practice_library(boost::asio::any_io_executor executo
break;
}
auto const &row = **next;
if (row.size() < 10U) {
continue;
}
auto source_json = text_value(row[9]);
auto parsed = vamp_zero::parse_score_document(source_json, text_value(row[8]));
if (!parsed) {
status = std::unexpected(parsed.error());
if (row.size() != 12U || count_value(row[9]) == 0U || count_value(row[10]) == 0U) {
status = std::unexpected(vamp_zero::make_error(vamp_zero::error_code::PERSISTENCE, "invalid practice metadata"));
break;
}
auto type_id = text_value(row[1]);
@@ -578,9 +660,8 @@ auto vamp_zero::async_load_practice_library(boost::asio::any_io_executor executo
.id = text_value(row[0]),
.key_signature = key_signature,
.source_path = source_path,
.source_json = source_json,
.measure_count = parsed->measure_count,
.total_pages = vamp_zero::score_page_count(*parsed),
.measure_count = count_value(row[9]),
.total_pages = count_value(row[10]),
};
exercises.push_back({
.id = logical_practice_exercise_id(type_id, lesson_id, source_path, text_value(row[5])),
@@ -590,11 +671,7 @@ auto vamp_zero::async_load_practice_library(boost::asio::any_io_executor executo
.lesson_group_title = text_value(row[4]),
.title = text_value(row[5]),
.composer = text_value(row[6]),
.key_signature = variant.key_signature,
.source_path = variant.source_path,
.source_json = variant.source_json,
.measure_count = variant.measure_count,
.total_pages = variant.total_pages,
.authored_key_signature = text_value(row[11]),
.variants = {std::move(variant)},
});
}
@@ -604,3 +681,44 @@ auto vamp_zero::async_load_practice_library(boost::asio::any_io_executor executo
}
co_return vamp_zero::group_practice_exercises(std::move(exercises));
}
auto vamp_zero::async_load_score(boost::asio::any_io_executor executor, vamp_zero::database_config config, vamp_zero::score_reference reference) -> boost::asio::awaitable<std::expected<vamp_zero::score_document, modulus::error>> {
// Unwind inline database completions between sequential loads of many scores.
co_await boost::asio::post(executor, boost::asio::use_awaitable);
auto connection = noto::connection{executor, connection_config(config)};
auto connected = co_await connection.async_connect();
if (!connected) {
co_return std::unexpected(connected.error());
}
auto table = reference.category == vamp_zero::score_category::PERFORMANCE ? "songs" : "practice_exercises";
auto selected = co_await connection.async_prepare("SELECT source_json, source_path FROM " + std::string{table} + " WHERE id = ?1");
if (!selected) {
static_cast<void>(co_await connection.async_close());
co_return std::unexpected(selected.error());
}
auto result = std::expected<vamp_zero::score_document, modulus::error>{std::unexpected(vamp_zero::make_error(vamp_zero::error_code::NOT_FOUND, "score not found: " + reference.id))};
auto params = std::vector<noto::sqlite_value>{noto::sqlite_value{reference.id}};
auto bound = co_await selected->async_reset(std::move(params));
if (!bound) {
result = std::unexpected(bound.error());
} else {
auto row = co_await selected->async_step();
if (!row) {
result = std::unexpected(row.error());
} else if (*row) {
result = vamp_zero::parse_score_document(text_value((**row)[0]), text_value((**row)[1]));
if (result) {
result->id = reference.id;
}
}
}
auto status = std::expected<void, modulus::error>{};
if (!result) {
status = std::unexpected(result.error());
}
auto closed = co_await close_catalogue_query(connection, *selected, std::move(status));
if (!closed) {
co_return std::unexpected(closed.error());
}
co_return result;
}