From b58c08150853b8055093dc116d407ffd543f8ec8 Mon Sep 17 00:00:00 2001 From: jacqueline Date: Mon, 30 Oct 2023 15:47:38 +1100 Subject: add locale-aware colation to db indexes --- src/database/database.cpp | 18 ++++++++++++------ src/database/include/database.hpp | 8 ++++++-- src/database/include/index.hpp | 3 ++- src/database/index.cpp | 10 ++++++---- 4 files changed, 26 insertions(+), 13 deletions(-) (limited to 'src/database') diff --git a/src/database/database.cpp b/src/database/database.cpp index e05fa26a..9b978b8c 100644 --- a/src/database/database.cpp +++ b/src/database/database.cpp @@ -17,6 +17,7 @@ #include #include +#include "collation.hpp" #include "esp_log.h" #include "ff.h" #include "freertos/projdefs.h" @@ -49,7 +50,7 @@ static SingletonEnv sEnv; static const char kDbPath[] = "/.tangara-db"; static const char kKeyDbVersion[] = "schema_version"; -static const uint8_t kCurrentDbVersion = 2; +static const uint8_t kCurrentDbVersion = 3; static const char kKeyTrackId[] = "next_track_id"; @@ -73,7 +74,9 @@ static auto CreateNewDatabase(leveldb::Options& options) -> leveldb::DB* { return db; } -auto Database::Open(IFileGatherer& gatherer, ITagParser& parser) +auto Database::Open(IFileGatherer& gatherer, + ITagParser& parser, + locale::ICollator& collator) -> cpp::result { // TODO(jacqueline): Why isn't compare_and_exchange_* available? if (sIsDbOpen.exchange(true)) { @@ -127,7 +130,8 @@ auto Database::Open(IFileGatherer& gatherer, ITagParser& parser) } ESP_LOGI(kTag, "Database opened successfully"); - return new Database(db, cache.release(), gatherer, parser, worker); + return new Database(db, cache.release(), gatherer, parser, collator, + worker); }) .get(); } @@ -142,12 +146,14 @@ Database::Database(leveldb::DB* db, leveldb::Cache* cache, IFileGatherer& file_gatherer, ITagParser& tag_parser, + locale::ICollator& collator, std::shared_ptr worker) : db_(db), cache_(cache), worker_task_(worker), file_gatherer_(file_gatherer), - tag_parser_(tag_parser) {} + tag_parser_(tag_parser), + collator_(collator) {} Database::~Database() { // Delete db_ first so that any outstanding background work finishes before @@ -539,7 +545,7 @@ auto Database::dbGetHash(const uint64_t& hash) -> std::optional { auto Database::dbCreateIndexesForTrack(const Track& track) -> void { for (const IndexInfo& index : GetIndexes()) { leveldb::WriteBatch writes; - auto entries = Index(index, track); + auto entries = Index(collator_, index, track); for (const auto& it : entries) { writes.Put(EncodeIndexKey(it.first), {it.second.data(), it.second.size()}); @@ -555,7 +561,7 @@ auto Database::dbRemoveIndexes(std::shared_ptr data) -> void { } Track track{data, tags}; for (const IndexInfo& index : GetIndexes()) { - auto entries = Index(index, track); + auto entries = Index(collator_, index, track); for (auto it = entries.rbegin(); it != entries.rend(); it++) { auto key = EncodeIndexKey(it->first); auto status = db_->Delete(leveldb::WriteOptions{}, key); diff --git a/src/database/include/database.hpp b/src/database/include/database.hpp index cdf69db0..5eb3a8e9 100644 --- a/src/database/include/database.hpp +++ b/src/database/include/database.hpp @@ -16,6 +16,7 @@ #include #include +#include "collation.hpp" #include "file_gatherer.hpp" #include "index.hpp" #include "leveldb/cache.h" @@ -92,9 +93,10 @@ class Database { ALREADY_OPEN, FAILED_TO_OPEN, }; - static auto Open(IFileGatherer& file_gatherer, ITagParser& tag_parser) + static auto Open(IFileGatherer& file_gatherer, + ITagParser& tag_parser, + locale::ICollator& collator) -> cpp::result; - static auto Open() -> cpp::result; static auto Destroy() -> void; @@ -136,11 +138,13 @@ class Database { // Not owned. IFileGatherer& file_gatherer_; ITagParser& tag_parser_; + locale::ICollator& collator_; Database(leveldb::DB* db, leveldb::Cache* cache, IFileGatherer& file_gatherer, ITagParser& tag_parser, + locale::ICollator& collator, std::shared_ptr worker); auto dbMintNewTrackId() -> TrackId; diff --git a/src/database/include/index.hpp b/src/database/include/index.hpp index 13de952d..15b21ee8 100644 --- a/src/database/include/index.hpp +++ b/src/database/include/index.hpp @@ -13,6 +13,7 @@ #include #include +#include "collation.hpp" #include "leveldb/db.h" #include "leveldb/slice.h" @@ -60,7 +61,7 @@ struct IndexKey { std::optional track; }; -auto Index(const IndexInfo&, const Track&) +auto Index(locale::ICollator&, const IndexInfo&, const Track&) -> std::vector>; auto ExpandHeader(const IndexKey::Header&, diff --git a/src/database/index.cpp b/src/database/index.cpp index 84ea050a..7d556192 100644 --- a/src/database/index.cpp +++ b/src/database/index.cpp @@ -7,8 +7,11 @@ #include "index.hpp" #include +#include #include +#include "collation.hpp" +#include "esp_log.h" #include "komihash.h" #include "leveldb/write_batch.h" @@ -59,7 +62,7 @@ static auto missing_component_text(const Track& track, Tag tag) } } -auto Index(const IndexInfo& info, const Track& t) +auto Index(locale::ICollator& collator, const IndexInfo& info, const Track& t) -> std::vector> { std::vector> out; IndexKey key{ @@ -72,15 +75,14 @@ auto Index(const IndexInfo& info, const Track& t) .track = {}, }; - auto& col = std::use_facet>(std::locale()); - for (std::uint8_t i = 0; i < info.components.size(); i++) { // Fill in the text for this depth. auto text = t.tags().at(info.components.at(i)); std::pmr::string value; if (text) { std::pmr::string orig = *text; - key.item = col.transform(&orig[0], &orig[0] + orig.size()); + auto xfrm = collator.Transform({orig.data(), orig.size()}); + key.item = {xfrm.data(), xfrm.size()}; value = *text; } else { key.item = {}; -- cgit v1.2.3