diff options
Diffstat (limited to 'src/database')
| -rw-r--r-- | src/database/database.cpp | 1028 | ||||
| -rw-r--r-- | src/database/file_gatherer.cpp | 10 | ||||
| -rw-r--r-- | src/database/include/database.hpp | 229 | ||||
| -rw-r--r-- | src/database/include/file_gatherer.hpp | 8 | ||||
| -rw-r--r-- | src/database/include/tag_parser.hpp | 14 | ||||
| -rw-r--r-- | src/database/include/track.hpp | 2 | ||||
| -rw-r--r-- | src/database/tag_parser.cpp | 24 |
7 files changed, 440 insertions, 875 deletions
diff --git a/src/database/database.cpp b/src/database/database.cpp index e646154e..1adfec87 100644 --- a/src/database/database.cpp +++ b/src/database/database.cpp @@ -16,6 +16,7 @@ #include <memory> #include <optional> #include <sstream> +#include <variant> #include "collation.hpp" #include "cppbor.h" @@ -200,11 +201,11 @@ Database::~Database() { sIsDbOpen.store(false); } -auto Database::Put(const std::string& key, const std::string& val) -> void { +auto Database::put(const std::string& key, const std::string& val) -> void { db_->Put(leveldb::WriteOptions{}, kKeyCustom + key, val); } -auto Database::Get(const std::string& key) -> std::optional<std::string> { +auto Database::get(const std::string& key) -> std::optional<std::string> { std::string val; auto res = db_->Get(leveldb::ReadOptions{}, kKeyCustom + key, &val); if (!res.ok()) { @@ -213,320 +214,212 @@ auto Database::Get(const std::string& key) -> std::optional<std::string> { return val; } -auto Database::Update() -> std::future<void> { +auto Database::getTrackPath(TrackId id) -> std::optional<std::string> { + auto track_data = dbGetTrackData(id); + if (!track_data) { + return {}; + } + return std::string{track_data->filepath.data(), track_data->filepath.size()}; +} + +auto Database::getTrack(TrackId id) -> std::shared_ptr<Track> { + std::shared_ptr<TrackData> data = dbGetTrackData(id); + if (!data || data->is_tombstoned) { + return {}; + } + std::shared_ptr<TrackTags> tags = tag_parser_.ReadAndParseTags( + {data->filepath.data(), data->filepath.size()}); + if (!tags) { + return {}; + } + return std::make_shared<Track>(data, tags); +} + +auto Database::getIndexes() -> std::vector<IndexInfo> { + // TODO(jacqueline): This probably needs to be async? When we have runtime + // configurable indexes, they will need to come from somewhere. + return { + kAllTracks, + kAllAlbums, + kAlbumsByArtist, + kTracksByGenre, + }; +} + +auto Database::updateIndexes() -> void { events::Ui().Dispatch(event::UpdateStarted{}); - return worker_task_->Dispatch<void>([&]() -> void { - leveldb::ReadOptions read_options; - read_options.fill_cache = false; - - std::pair<uint16_t, uint16_t> newest_track{0, 0}; - - // Stage 1: verify all existing tracks are still valid. - ESP_LOGI(kTag, "verifying existing tracks"); - { - uint64_t num_processed = 0; - std::unique_ptr<leveldb::Iterator> it{db_->NewIterator(read_options)}; - std::string prefix = EncodeDataPrefix(); - for (it->Seek(prefix); it->Valid() && it->key().starts_with(prefix); - it->Next()) { - num_processed++; - events::Ui().Dispatch(event::UpdateProgress{ - .stage = event::UpdateProgress::Stage::kVerifyingExistingTracks, - .val = num_processed, - }); - - std::shared_ptr<TrackData> track = ParseDataValue(it->value()); - if (!track) { - // The value was malformed. Drop this record. - ESP_LOGW(kTag, "dropping malformed metadata"); - db_->Delete(leveldb::WriteOptions(), it->key()); - continue; - } - - if (track->is_tombstoned) { - ESP_LOGW(kTag, "skipping tombstoned %lx", track->id); - continue; - } - - FRESULT res; - FILINFO info; - { - auto lock = drivers::acquire_spi(); - res = f_stat(track->filepath.c_str(), &info); - } - - std::pair<uint16_t, uint16_t> modified_at{0, 0}; - if (res == FR_OK) { - modified_at = {info.fdate, info.ftime}; - } - if (modified_at == track->modified_at) { - newest_track = std::max(modified_at, newest_track); - continue; - } else { - track->modified_at = modified_at; - } - - std::shared_ptr<TrackTags> tags = - tag_parser_.ReadAndParseTags(track->filepath); - if (!tags || tags->encoding() == Container::kUnsupported) { - // We couldn't read the tags for this track. Either they were - // malformed, or perhaps the file is missing. Either way, tombstone - // this record. - ESP_LOGW(kTag, "entombing missing #%lx", track->id); - dbRemoveIndexes(track); - track->is_tombstoned = true; - dbPutTrackData(*track); - continue; - } - - // At this point, we know that the track still exists in its original - // location. All that's left to do is update any metadata about it. - - uint64_t new_hash = tags->Hash(); - if (new_hash != track->tags_hash) { - // This track's tags have changed. Since the filepath is exactly the - // same, we assume this is a legitimate correction. Update the - // database. - ESP_LOGI(kTag, "updating hash (%llx -> %llx)", track->tags_hash, - new_hash); - dbRemoveIndexes(track); - - track->tags_hash = new_hash; - dbIngestTagHashes(*tags, track->individual_tag_hashes); - dbPutTrackData(*track); - dbPutHash(new_hash, track->id); - } - } - } + leveldb::ReadOptions read_options; + read_options.fill_cache = false; - ESP_LOGI(kTag, "newest unmodified was at %u,%u", newest_track.first, - newest_track.second); + std::pair<uint16_t, uint16_t> newest_track{0, 0}; - // Stage 2: search for newly added files. - ESP_LOGI(kTag, "scanning for new tracks"); + // Stage 1: verify all existing tracks are still valid. + ESP_LOGI(kTag, "verifying existing tracks"); + { uint64_t num_processed = 0; - file_gatherer_.FindFiles("", [&](const std::pmr::string& path, - const FILINFO& info) { + std::unique_ptr<leveldb::Iterator> it{db_->NewIterator(read_options)}; + std::string prefix = EncodeDataPrefix(); + for (it->Seek(prefix); it->Valid() && it->key().starts_with(prefix); + it->Next()) { num_processed++; events::Ui().Dispatch(event::UpdateProgress{ - .stage = event::UpdateProgress::Stage::kScanningForNewTracks, + .stage = event::UpdateProgress::Stage::kVerifyingExistingTracks, .val = num_processed, }); - std::pair<uint16_t, uint16_t> modified{info.fdate, info.ftime}; - if (modified < newest_track) { - return; + std::shared_ptr<TrackData> track = ParseDataValue(it->value()); + if (!track) { + // The value was malformed. Drop this record. + ESP_LOGW(kTag, "dropping malformed metadata"); + db_->Delete(leveldb::WriteOptions(), it->key()); + continue; } - std::shared_ptr<TrackTags> tags = tag_parser_.ReadAndParseTags(path); - if (!tags || tags->encoding() == Container::kUnsupported) { - // No parseable tags; skip this fiile. - return; + if (track->is_tombstoned) { + ESP_LOGW(kTag, "skipping tombstoned %lx", track->id); + continue; } - // Check for any existing record with the same hash. - uint64_t hash = tags->Hash(); - std::string key = EncodeHashKey(hash); - std::optional<TrackId> existing_hash; - std::string raw_entry; - if (db_->Get(leveldb::ReadOptions(), key, &raw_entry).ok()) { - existing_hash = ParseHashValue(raw_entry); + FRESULT res; + FILINFO info; + { + auto lock = drivers::acquire_spi(); + res = f_stat(track->filepath.c_str(), &info); } - if (!existing_hash) { - // We've never met this track before! Or we have, but the entry is - // malformed. Either way, record this as a new track. - TrackId id = dbMintNewTrackId(); - ESP_LOGI(kTag, "recording new 0x%lx", id); - - auto data = std::make_shared<TrackData>(); - data->id = id; - data->filepath = path; - data->tags_hash = hash; - data->modified_at = modified; - dbIngestTagHashes(*tags, data->individual_tag_hashes); - - dbPutTrackData(*data); - dbPutHash(hash, id); - auto t = std::make_shared<Track>(data, tags); - dbCreateIndexesForTrack(*t); - return; + std::pair<uint16_t, uint16_t> modified_at{0, 0}; + if (res == FR_OK) { + modified_at = {info.fdate, info.ftime}; + } + if (modified_at == track->modified_at) { + newest_track = std::max(modified_at, newest_track); + continue; + } else { + track->modified_at = modified_at; } - std::shared_ptr<TrackData> existing_data = dbGetTrackData(*existing_hash); - if (!existing_data) { - // We found a hash that matches, but there's no data record? Weird. - auto new_data = std::make_shared<TrackData>(); - new_data->id = dbMintNewTrackId(); - new_data->filepath = path; - new_data->tags_hash = hash; - new_data->modified_at = modified; - dbIngestTagHashes(*tags, new_data->individual_tag_hashes); - dbPutTrackData(*new_data); - auto t = std::make_shared<Track>(new_data, tags); - dbCreateIndexesForTrack(*t); - return; + std::shared_ptr<TrackTags> tags = tag_parser_.ReadAndParseTags( + {track->filepath.data(), track->filepath.size()}); + if (!tags || tags->encoding() == Container::kUnsupported) { + // We couldn't read the tags for this track. Either they were + // malformed, or perhaps the file is missing. Either way, tombstone + // this record. + ESP_LOGW(kTag, "entombing missing #%lx", track->id); + dbRemoveIndexes(track); + track->is_tombstoned = true; + dbPutTrackData(*track); + continue; } - if (existing_data->is_tombstoned) { - ESP_LOGI(kTag, "exhuming track %lu", existing_data->id); - existing_data->is_tombstoned = false; - existing_data->modified_at = modified; - dbPutTrackData(*existing_data); - auto t = std::make_shared<Track>(existing_data, tags); - dbCreateIndexesForTrack(*t); - } else if (existing_data->filepath != path) { - ESP_LOGW(kTag, "tag hash collision for %s and %s", - existing_data->filepath.c_str(), path.c_str()); - ESP_LOGI(kTag, "hash components: %s, %s, %s", - tags->at(Tag::kTitle).value_or("no title").c_str(), - tags->at(Tag::kArtist).value_or("no artist").c_str(), - tags->at(Tag::kAlbum).value_or("no album").c_str()); + // At this point, we know that the track still exists in its original + // location. All that's left to do is update any metadata about it. + + uint64_t new_hash = tags->Hash(); + if (new_hash != track->tags_hash) { + // This track's tags have changed. Since the filepath is exactly the + // same, we assume this is a legitimate correction. Update the + // database. + ESP_LOGI(kTag, "updating hash (%llx -> %llx)", track->tags_hash, + new_hash); + dbRemoveIndexes(track); + + track->tags_hash = new_hash; + dbIngestTagHashes(*tags, track->individual_tag_hashes); + dbPutTrackData(*track); + dbPutHash(new_hash, track->id); } - }); - events::Ui().Dispatch(event::UpdateFinished{}); - }); -} + } + } -auto Database::GetTrackPath(TrackId id) - -> std::future<std::optional<std::pmr::string>> { - return worker_task_->Dispatch<std::optional<std::pmr::string>>( - [=, this]() -> std::optional<std::pmr::string> { - auto track_data = dbGetTrackData(id); - if (track_data) { - return track_data->filepath; - } - return {}; - }); -} + ESP_LOGI(kTag, "newest unmodified was at %u,%u", newest_track.first, + newest_track.second); -auto Database::GetTrack(TrackId id) -> std::future<std::shared_ptr<Track>> { - return worker_task_->Dispatch<std::shared_ptr<Track>>( - [=, this]() -> std::shared_ptr<Track> { - std::shared_ptr<TrackData> data = dbGetTrackData(id); - if (!data || data->is_tombstoned) { - return {}; - } - std::shared_ptr<TrackTags> tags = - tag_parser_.ReadAndParseTags(data->filepath); - if (!tags) { - return {}; - } - return std::make_shared<Track>(data, tags); - }); -} + // Stage 2: search for newly added files. + ESP_LOGI(kTag, "scanning for new tracks"); + uint64_t num_processed = 0; + file_gatherer_.FindFiles("", [&](const std::string& path, + const FILINFO& info) { + num_processed++; + events::Ui().Dispatch(event::UpdateProgress{ + .stage = event::UpdateProgress::Stage::kScanningForNewTracks, + .val = num_processed, + }); -auto Database::GetBulkTracks(std::vector<TrackId> ids) - -> std::future<std::vector<std::shared_ptr<Track>>> { - return worker_task_->Dispatch<std::vector<std::shared_ptr<Track>>>( - [=, this]() -> std::vector<std::shared_ptr<Track>> { - std::map<TrackId, std::shared_ptr<Track>> id_to_track{}; - - // Sort the list of ids so that we can retrieve them all in a single - // iteration through the database, without re-seeking. - std::vector<TrackId> sorted_ids = ids; - std::sort(sorted_ids.begin(), sorted_ids.end()); - - std::unique_ptr<leveldb::Iterator> it{ - db_->NewIterator(leveldb::ReadOptions{})}; - for (const TrackId& id : sorted_ids) { - std::string key = EncodeDataKey(id); - it->Seek(key); - if (!it->Valid() || it->key() != key) { - // This id wasn't found at all. Skip it. - continue; - } - std::shared_ptr<Track> track = - ParseRecord<Track>(it->key(), it->value()); - if (track) { - id_to_track.insert({id, track}); - } - } - - // We've fetched all of the ids in the request, so now just put them - // back into the order they were asked for in. - std::vector<std::shared_ptr<Track>> results; - for (const TrackId& id : ids) { - if (id_to_track.contains(id)) { - results.push_back(id_to_track.at(id)); - } else { - // This lookup failed. - results.push_back({}); - } - } - return results; - }); -} + std::pair<uint16_t, uint16_t> modified{info.fdate, info.ftime}; + if (modified < newest_track) { + return; + } -auto Database::GetIndexes() -> std::vector<IndexInfo> { - // TODO(jacqueline): This probably needs to be async? When we have runtime - // configurable indexes, they will need to come from somewhere. - return { - kAllTracks, - kAllAlbums, - kAlbumsByArtist, - kTracksByGenre, - }; -} + std::shared_ptr<TrackTags> tags = tag_parser_.ReadAndParseTags(path); + if (!tags || tags->encoding() == Container::kUnsupported) { + // No parseable tags; skip this fiile. + return; + } -auto Database::GetTracksByIndex(IndexId index, std::size_t page_size) - -> std::future<Result<IndexRecord>*> { - return worker_task_->Dispatch<Result<IndexRecord>*>( - [=, this]() -> Result<IndexRecord>* { - IndexKey::Header header{ - .id = index, - .depth = 0, - .components_hash = 0, - }; - std::string prefix = EncodeIndexPrefix(header); - Continuation c{.prefix = {prefix.data(), prefix.size()}, - .start_key = {prefix.data(), prefix.size()}, - .forward = true, - .was_prev_forward = true, - .page_size = page_size}; - return dbGetPage<IndexRecord>(c); - }); -} + // Check for any existing record with the same hash. + uint64_t hash = tags->Hash(); + std::string key = EncodeHashKey(hash); + std::optional<TrackId> existing_hash; + std::string raw_entry; + if (db_->Get(leveldb::ReadOptions(), key, &raw_entry).ok()) { + existing_hash = ParseHashValue(raw_entry); + } -auto Database::GetTracks(std::size_t page_size) -> std::future<Result<Track>*> { - return worker_task_->Dispatch<Result<Track>*>([=, this]() -> Result<Track>* { - std::string prefix = EncodeDataPrefix(); - Continuation c{.prefix = {prefix.data(), prefix.size()}, - .start_key = {prefix.data(), prefix.size()}, - .forward = true, - .was_prev_forward = true, - .page_size = page_size}; - return dbGetPage<Track>(c); - }); -} + if (!existing_hash) { + // We've never met this track before! Or we have, but the entry is + // malformed. Either way, record this as a new track. + TrackId id = dbMintNewTrackId(); + ESP_LOGI(kTag, "recording new 0x%lx", id); + + auto data = std::make_shared<TrackData>(); + data->id = id; + data->filepath = path; + data->tags_hash = hash; + data->modified_at = modified; + dbIngestTagHashes(*tags, data->individual_tag_hashes); + + dbPutTrackData(*data); + dbPutHash(hash, id); + auto t = std::make_shared<Track>(data, tags); + dbCreateIndexesForTrack(*t); + return; + } -auto Database::GetDump(std::size_t page_size) - -> std::future<Result<std::pmr::string>*> { - return worker_task_->Dispatch<Result<std::pmr::string>*>( - [=, this]() -> Result<std::pmr::string>* { - Continuation c{.prefix = "", - .start_key = "", - .forward = true, - .was_prev_forward = true, - .page_size = page_size}; - return dbGetPage<std::pmr::string>(c); - }); -} + std::shared_ptr<TrackData> existing_data = dbGetTrackData(*existing_hash); + if (!existing_data) { + // We found a hash that matches, but there's no data record? Weird. + auto new_data = std::make_shared<TrackData>(); + new_data->id = dbMintNewTrackId(); + new_data->filepath = path; + new_data->tags_hash = hash; + new_data->modified_at = modified; + dbIngestTagHashes(*tags, new_data->individual_tag_hashes); + dbPutTrackData(*new_data); + auto t = std::make_shared<Track>(new_data, tags); + dbCreateIndexesForTrack(*t); + return; + } -template <typename T> -auto Database::GetPage(Continuation* c) -> std::future<Result<T>*> { - Continuation copy = *c; - return worker_task_->Dispatch<Result<T>*>( - [=, this]() -> Result<T>* { return dbGetPage<T>(copy); }); + if (existing_data->is_tombstoned) { + ESP_LOGI(kTag, "exhuming track %lu", existing_data->id); + existing_data->is_tombstoned = false; + existing_data->modified_at = modified; + dbPutTrackData(*existing_data); + auto t = std::make_shared<Track>(existing_data, tags); + dbCreateIndexesForTrack(*t); + } else if (existing_data->filepath != + std::pmr::string{path.data(), path.size()}) { + ESP_LOGW(kTag, "tag hash collision for %s and %s", + existing_data->filepath.c_str(), path.c_str()); + ESP_LOGI(kTag, "hash components: %s, %s, %s", + tags->at(Tag::kTitle).value_or("no title").c_str(), + tags->at(Tag::kArtist).value_or("no artist").c_str(), + tags->at(Tag::kAlbum).value_or("no album").c_str()); + } + }); + events::Ui().Dispatch(event::UpdateFinished{}); } -template auto Database::GetPage<Track>(Continuation* c) - -> std::future<Result<Track>*>; -template auto Database::GetPage<IndexRecord>(Continuation* c) - -> std::future<Result<IndexRecord>*>; -template auto Database::GetPage<std::pmr::string>(Continuation* c) - -> std::future<Result<std::pmr::string>*>; - auto Database::dbMintNewTrackId() -> TrackId { TrackId next_id = 1; std::string val; @@ -592,7 +485,7 @@ auto Database::dbGetHash(const uint64_t& hash) -> std::optional<TrackId> { } auto Database::dbCreateIndexesForTrack(const Track& track) -> void { - for (const IndexInfo& index : GetIndexes()) { + for (const IndexInfo& index : getIndexes()) { leveldb::WriteBatch writes; auto entries = Index(collator_, index, track); for (const auto& it : entries) { @@ -609,7 +502,7 @@ auto Database::dbRemoveIndexes(std::shared_ptr<TrackData> data) -> void { return; } Track track{data, tags}; - for (const IndexInfo& index : GetIndexes()) { + for (const IndexInfo& index : getIndexes()) { auto entries = Index(collator_, index, track); for (auto it = entries.rbegin(); it != entries.rend(); it++) { auto key = EncodeIndexKey(it->first); @@ -666,512 +559,209 @@ auto Database::dbRecoverTagsFromHashes( return out; } -template <typename T> -auto Database::dbGetPage(const Continuation& c) -> Result<T>* { - // Work out our starting point. Sometimes this will already done. - std::unique_ptr<leveldb::Iterator> it{ - db_->NewIterator(leveldb::ReadOptions{})}; - it->Seek({c.start_key.data(), c.start_key.size()}); - - // Fix off-by-one if we just changed direction. - if (c.forward != c.was_prev_forward) { - if (c.forward) { - it->Next(); - } else { +auto seekToOffset(leveldb::Iterator* it, int offset) { + while (it->Valid() && offset != 0) { + if (offset < 0) { it->Prev(); - } - } - - // Grab results. - std::optional<std::pmr::string> first_key; - std::vector<std::shared_ptr<T>> records; - while (records.size() < c.page_size && it->Valid()) { - if (!it->key().starts_with({c.prefix.data(), c.prefix.size()})) { - break; - } - if (!first_key) { - first_key = it->key().ToString(); - } - std::shared_ptr<T> parsed = ParseRecord<T>(it->key(), it->value()); - if (parsed) { - records.push_back(parsed); - } - if (c.forward) { - it->Next(); + offset++; } else { - it->Prev(); + it->Next(); + offset--; } } +} - if (!it->Valid() || - !it->key().starts_with({c.prefix.data(), c.prefix.size()})) { - it.reset(); - } - - // Put results into canonical order if we were iterating backwards. - if (!c.forward) { - std::reverse(records.begin(), records.end()); - } +auto Database::getRecord(const SearchKey& c) + -> std::optional<std::pair<std::pmr::string, Record>> { + std::unique_ptr<leveldb::Iterator> it{ + db_->NewIterator(leveldb::ReadOptions{})}; - // Work out the new continuations. - std::optional<Continuation> next_page; - if (c.forward) { - if (it != nullptr) { - // We were going forward, and now we want the next page. - std::pmr::string key{it->key().data(), it->key().size(), - &memory::kSpiRamResource}; - next_page = Continuation{ - .prefix = c.prefix, - .start_key = key, - .forward = true, - .was_prev_forward = true, - .page_size = c.page_size, - }; - } - // No iterator means we ran out of results in this direction. - } else { - // We were going backwards, and now we want the next page. This is a - // reversal, to set the start key to the first record we saw and mark that - // it's off by one. - next_page = Continuation{ - .prefix = c.prefix, - .start_key = *first_key, - .forward = true, - .was_prev_forward = false, - .page_size = c.page_size, - }; + it->Seek(c.startKey()); + seekToOffset(it.get(), c.offset); + if (!it->Valid() || !it->key().starts_with(std::string_view{c.prefix})) { + return {}; } - std::optional<Continuation> prev_page; - if (c.forward) { - // We were going forwards, and now we want the previous page. Set the - // search key to the first result we saw, and mark that it's off by one. - prev_page = Continuation{ - .prefix = c.prefix, - .start_key = *first_key, - .forward = false, - .was_prev_forward = true, - .page_size = c.page_size, - }; - } else { - if (it != nullptr) { - // We were going backwards, and we still want to go backwards. - std::pmr::string key{it->key().data(), it->key().size(), - &memory::kSpiRamResource}; - prev_page = Continuation{ - .prefix = c.prefix, - .start_key = key, - .forward = false, - .was_prev_forward = false, - .page_size = c.page_size, - }; - } - // No iterator means we ran out of results in this direction. + std::optional<IndexKey> key = ParseIndexKey(it->key()); + if (!key) { + ESP_LOGW(kTag, "parsing index key failed"); + return {}; } - return new Result<T>(std::move(records), next_page, prev_page); + return std::make_pair(std::pmr::string{it->key().data(), it->key().size(), + &memory::kSpiRamResource}, + Record{*key, it->value()}); } -auto Database::dbCount(const Continuation& c) -> size_t { +auto Database::countRecords(const SearchKey& c) -> size_t { std::unique_ptr<leveldb::Iterator> it{ db_->NewIterator(leveldb::ReadOptions{})}; - size_t count = 0; - for (it->Seek({c.start_key.data(), c.start_key.size()}); - it->Valid() && it->key().starts_with({c.prefix.data(), c.prefix.size()}); - it->Next()) { - count++; - } - return count; -} -template auto Database::dbGetPage<Track>(const Continuation& c) - -> Result<Track>*; -template auto Database::dbGetPage<std::pmr::string>(const Continuation& c) - -> Result<std::pmr::string>*; - -template <> -auto Database::ParseRecord<IndexRecord>(const leveldb::Slice& key, - const leveldb::Slice& val) - -> std::shared_ptr<IndexRecord> { - std::optional<IndexKey> data = ParseIndexKey(key); - if (!data) { + it->Seek(c.startKey()); + seekToOffset(it.get(), c.offset); + if (!it->Valid() || !it->key().starts_with(std::string_view{c.prefix})) { return {}; } - std::optional<std::pmr::string> title; - if (!val.empty()) { - title = val.ToString(); + size_t count = 0; + while (it->Valid() && it->key().starts_with(std::string_view{c.prefix})) { + it->Next(); + count++; } - return std::make_shared<IndexRecord>(*data, title, data->track); + return count; } -template <> -auto Database::ParseRecord<Track>(const leveldb::Slice& key, - const leveldb::Slice& val) - -> std::shared_ptr<Track> { - std::shared_ptr<TrackData> data = ParseDataValue(val); - if (!data || data->is_tombstoned) { - return {}; - } - std::shared_ptr<TrackTags> tags = - tag_parser_.ReadAndParseTags(data->filepath); - if (!tags) { - return {}; +auto SearchKey::startKey() const -> std::string_view { + if (key) { + return *key; } - return std::make_shared<Track>(data, tags); + return prefix; } -template <> -auto Database::ParseRecord<std::pmr::string>(const leveldb::Slice& key, - const leveldb::Slice& val) - -> std::shared_ptr<std::pmr::string> { - std::ostringstream stream; - stream << "key: "; - if (key.size() < 3 || key.data()[1] != '\0') { - stream << key.ToString().c_str(); +Record::Record(const IndexKey& key, const leveldb::Slice& t) + : text_(t.data(), t.size(), &memory::kSpiRamResource) { + if (key.track) { + contents_ = *key.track; } else { - for (size_t i = 0; i < key.size(); i++) { - if (i == 0) { - stream << key.data()[i]; - } else if (i == 1) { - stream << " / 0x"; - } else { - stream << std::hex << std::setfill('0') << std::setw(2) - << static_cast<int>(key.data()[i]); - } - } - } - if (!val.empty()) { - stream << "\tval: 0x"; - for (int i = 0; i < val.size(); i++) { - stream << std::hex << std::setfill('0') << std::setw(2) - << static_cast<int>(val.data()[i]); - } + contents_ = ExpandHeader(key.header, key.item); } - std::pmr::string res{stream.str(), &memory::kSpiRamResource}; - return std::make_shared<std::pmr::string>(res); } -IndexRecord::IndexRecord(const IndexKey& key, - std::optional<std::pmr::string> title, - std::optional<TrackId> track) - : key_(key), override_text_(title), track_(track) {} - -auto IndexRecord::text() const -> std::optional<std::pmr::string> { - if (override_text_) { - return override_text_; - } - return key_.item; +auto Record::text() const -> std::string_view { + return text_; } -auto IndexRecord::track() const -> std::optional<TrackId> { - return track_; +auto Record::contents() const + -> const std::variant<TrackId, IndexKey::Header>& { + return contents_; } -auto IndexRecord::Expand(std::size_t page_size) const - -> std::optional<Continuation> { - if (track_) { - return {}; - } - std::string new_prefix = EncodeIndexPrefix(ExpandHeader()); - return Continuation{ - .prefix = {new_prefix.data(), new_prefix.size()}, - .start_key = {new_prefix.data(), new_prefix.size()}, - .forward = true, - .was_prev_forward = true, - .page_size = page_size, - }; -} +Iterator::Iterator(std::shared_ptr<Database> db, IndexId idx) + : Iterator(db, + IndexKey::Header{ + .id = idx, + .depth = 0, + .components_hash = 0, + }) {} -auto IndexRecord::ExpandHeader() const -> IndexKey::Header { - return ::database::ExpandHeader(key_.header, key_.item); +Iterator::Iterator(std::shared_ptr<Database> db, const IndexKey::Header& header) + : db_(db), key_{}, current_() { + std::string prefix = EncodeIndexPrefix(header); + key_ = { + .prefix = {prefix.data(), prefix.size(), &memory::kSpiRamResource}, + .key = {}, + .offset = 0, + }; + iterate(key_); } -Iterator::Iterator(std::weak_ptr<Database> db, const IndexInfo& idx) - : db_(db), pos_mutex_(), current_pos_(), prev_pos_() { - std::string prefix = EncodeIndexPrefix( - IndexKey::Header{.id = idx.id, .depth = 0, .components_hash = 0}); - current_pos_ = Continuation{.prefix = {prefix.data(), prefix.size()}, - .start_key = {prefix.data(), prefix.size()}, - .forward = true, - .was_prev_forward = true, - .page_size = 1}; +auto Iterator::value() const -> const std::optional<Record>& { + return current_; } -auto Iterator::Parse(std::weak_ptr<Database> db, const cppbor::Array& encoded) - -> std::optional<Iterator> { - // Ensure the input looks reasonable. - if (encoded.size() != 3) { - return {}; - } - - if (encoded[0]->type() != cppbor::TSTR) { - return {}; - } - const std::string& prefix = encoded[0]->asTstr()->value(); - - std::optional<Continuation> current_pos{}; - if (encoded[1]->type() == cppbor::TSTR) { - const std::string& key = encoded[1]->asTstr()->value(); - current_pos = Continuation{ - .prefix = {prefix.data(), prefix.size()}, - .start_key = {key.data(), key.size()}, - .forward = true, - .was_prev_forward = true, - .page_size = 1, - }; - } - - std::optional<Continuation> prev_pos{}; - if (encoded[2]->type() == cppbor::TSTR) { - const std::string& key = encoded[2]->asTstr()->value(); - current_pos = Continuation{ - .prefix = {prefix.data(), prefix.size()}, - .start_key = {key.data(), key.size()}, - .forward = false, - .was_prev_forward = false, - .page_size = 1, - }; - } - - return Iterator{db, std::move(current_pos), std::move(prev_pos)}; +auto Iterator::next() -> void { + SearchKey new_key = key_; + new_key.offset = 1; + iterate(new_key); } -Iterator::Iterator(std::weak_ptr<Database> db, const Continuation& c) - : db_(db), pos_mutex_(), current_pos_(c), prev_pos_() {} - -Iterator::Iterator(const Iterator& other) - : db_(other.db_), - pos_mutex_(), - current_pos_(other.current_pos_), - prev_pos_(other.prev_pos_) {} - -Iterator::Iterator(std::weak_ptr<Database> db, - std::optional<Continuation>&& cur, - std::optional<Continuation>&& prev) - : db_(db), current_pos_(cur), prev_pos_(prev) {} - -Iterator& Iterator::operator=(const Iterator& other) { - current_pos_ = other.current_pos_; - prev_pos_ = other.prev_pos_; - return *this; +auto Iterator::prev() -> void { + SearchKey new_key = key_; + new_key.offset = -1; + iterate(new_key); } -auto Iterator::Next(Callback cb) -> void { +auto Iterator::iterate(const SearchKey& key) -> void { auto db = db_.lock(); if (!db) { - InvokeNull(cb); + ESP_LOGW(kTag, "iterate with dead db"); return; } - db->worker_task_->Dispatch<void>([=]() { - std::lock_guard lock{pos_mutex_}; - if (!current_pos_) { - InvokeNull(cb); - return; - } - std::unique_ptr<Result<IndexRecord>> res{ - db->dbGetPage<IndexRecord>(*current_pos_)}; - prev_pos_ = current_pos_; - current_pos_ = res->next_page(); - if (!res || res->values().empty() || !res->values()[0]) { - ESP_LOGI(kTag, "dropping empty result"); - InvokeNull(cb); - return; - } - std::invoke(cb, *res->values()[0]); - }); -} - -auto Iterator::NextSync() -> std::optional<IndexRecord> { - auto db = db_.lock(); - if (!db) { - return {}; - } - std::lock_guard lock{pos_mutex_}; - if (!current_pos_) { - return {}; - } - std::unique_ptr<Result<IndexRecord>> res{ - db->dbGetPage<IndexRecord>(*current_pos_)}; - prev_pos_ = current_pos_; - current_pos_ = res->next_page(); - if (!res || res->values().empty() || !res->values()[0]) { - ESP_LOGI(kTag, "dropping empty result"); - return {}; - } - return *res->values()[0]; -} - -auto Iterator::PeekSync() -> std::optional<IndexRecord> { - auto db = db_.lock(); - if (!db) { - return {}; - } - auto pos = current_pos_; - if (!pos) { - return {}; - } - std::unique_ptr<Result<IndexRecord>> res{db->dbGetPage<IndexRecord>(*pos)}; - if (!res || res->values().empty() || !res->values()[0]) { - return {}; - } - return *res->values()[0]; -} - -auto Iterator::Prev(Callback cb) -> void { - auto db = db_.lock(); - if (!db) { - InvokeNull(cb); - return; + auto res = db->getRecord(key); + if (res) { + key_ = { + .prefix = key_.prefix, + .key = res->first, + .offset = 0, + }; + current_ = res->second; + } else { + key_ = key; + current_.reset(); } - db->worker_task_->Dispatch<void>([=]() { - std::lock_guard lock{pos_mutex_}; - if (!prev_pos_) { - InvokeNull(cb); - return; - } - std::unique_ptr<Result<IndexRecord>> res{ - db->dbGetPage<IndexRecord>(*current_pos_)}; - current_pos_ = prev_pos_; - prev_pos_ = res->prev_page(); - std::invoke(cb, *res->values()[0]); - }); } -auto Iterator::Size() const -> size_t { +auto Iterator::count() const -> size_t { auto db = db_.lock(); if (!db) { - return {}; - } - std::optional<Continuation> pos = current_pos_; - if (!pos) { + ESP_LOGW(kTag, "count with dead db"); return 0; } - return db->dbCount(*pos); + return db->countRecords(key_); } -auto Iterator::InvokeNull(Callback cb) -> void { - std::invoke(cb, std::optional<IndexRecord>{}); +TrackIterator::TrackIterator(const Iterator& it) : db_(it.db_), levels_() { + levels_.push_back(it); + next(false); } -auto Iterator::cbor() const -> cppbor::Array&& { - cppbor::Array res; - - std::pmr::string prefix; - if (current_pos_) { - prefix = current_pos_->prefix; - } else if (prev_pos_) { - prefix = prev_pos_->prefix; - } else { - ESP_LOGW(kTag, "iterator has no prefix"); - return std::move(res); - } - - if (current_pos_) { - res.add(cppbor::Tstr(current_pos_->start_key)); - } else { - res.add(cppbor::Null()); - } - - if (prev_pos_) { - res.add(cppbor::Tstr(prev_pos_->start_key)); - } else { - res.add(cppbor::Null()); - } - - return std::move(res); +auto TrackIterator::next() -> void { + next(true); } -auto TrackIterator::Parse(std::weak_ptr<Database> db, - const cppbor::Array& encoded) - -> std::optional<TrackIterator> { - TrackIterator ret{db}; +auto TrackIterator::next(bool advance) -> void { + while (!levels_.empty()) { + if (advance) { + levels_.back().next(); + } - for (const auto& item : encoded) { - if (item->type() == cppbor::ARRAY) { - auto it = Iterator::Parse(db, *item->asArray()); - if (it) { - ret.levels_.push_back(std::move(*it)); - } else { - return {}; + auto& cur = levels_.back().value(); + if (!cur) { + // The current top iterator is out of tracks. Pop it, and move the parent + // to the next item. + levels_.pop_back(); + advance = true; + } else if (std::holds_alternative<IndexKey::Header>(cur->contents())) { + // This record is a branch. Push a new iterator. + auto key = std::get<IndexKey::Header>(cur->contents()); + auto db = db_.lock(); + if (!db) { + return; } + levels_.emplace_back(db, key); + // Don't skip the first value of the new level. + advance = false; + } else if (std::holds_alternative<TrackId>(cur->contents())) { + // New record is a leaf. + break; } } - - return ret; } -TrackIterator::TrackIterator(const Iterator& it) : db_(it.db_), levels_() { - if (it.current_pos_) { - levels_.push_back(it); +auto TrackIterator::value() const -> std::optional<TrackId> { + if (levels_.empty()) { + return {}; } - NextLeaf(); -} - -TrackIterator::TrackIterator(const TrackIterator& other) - : db_(other.db_), levels_(other.levels_) {} - -TrackIterator::TrackIterator(std::weak_ptr<Database> db) : db_(db), levels_() {} - -TrackIterator& TrackIterator::operator=(TrackIterator&& other) { - levels_ = std::move(other.levels_); - return *this; -} - -auto TrackIterator::Next() -> std::optional<TrackId> { - std::optional<TrackId> next{}; - while (!next && !levels_.empty()) { - auto next_record = levels_.back().NextSync(); - if (!next_record) { - levels_.pop_back(); - NextLeaf(); - continue; - } - // May still be nullopt_t; hence the loop. - next = next_record->track(); + auto cur = levels_.back().value(); + if (!cur) { + return {}; } - return next; + if (std::holds_alternative<TrackId>(cur->contents())) { + return std::get<TrackId>(cur->contents()); + } + return {}; } -auto TrackIterator::Size() const -> size_t { +auto TrackIterator::count() const -> size_t { size_t size = 0; TrackIterator copy{*this}; while (!copy.levels_.empty()) { - size += copy.levels_.back().Size(); + size += copy.levels_.back().count(); copy.levels_.pop_back(); - copy.NextLeaf(); + copy.next(); } return size; } -auto TrackIterator::NextLeaf() -> void { - while (!levels_.empty()) { - ESP_LOGI(kTag, "check next candidate"); - Iterator& candidate = levels_.back(); - auto next = candidate.PeekSync(); - if (!next) { - ESP_LOGI(kTag, "candidate is empty"); - levels_.pop_back(); - continue; - } - if (!next->track()) { - ESP_LOGI(kTag, "candidate is a branch"); - candidate.NextSync(); - levels_.push_back(Iterator{db_, next->Expand(1).value()}); - continue; - } - ESP_LOGI(kTag, "candidate is a leaf"); - break; - } -} - -auto TrackIterator::cbor() const -> cppbor::Array&& { - cppbor::Array res; - for (const auto& i : levels_) { - res.add(i.cbor()); - } - return std::move(res); -} - } // namespace database diff --git a/src/database/file_gatherer.cpp b/src/database/file_gatherer.cpp index 0809ee0d..f07a1b4d 100644 --- a/src/database/file_gatherer.cpp +++ b/src/database/file_gatherer.cpp @@ -21,13 +21,13 @@ namespace database { static_assert(sizeof(TCHAR) == sizeof(char), "TCHAR must be CHAR"); auto FileGathererImpl::FindFiles( - const std::pmr::string& root, - std::function<void(const std::pmr::string&, const FILINFO&)> cb) -> void { - std::pmr::deque<std::pmr::string> to_explore(&memory::kSpiRamResource); + const std::string& root, + std::function<void(const std::string&, const FILINFO&)> cb) -> void { + std::deque<std::string> to_explore; to_explore.push_back(root); while (!to_explore.empty()) { - std::pmr::string next_path_str = to_explore.front(); + std::string next_path_str = to_explore.front(); const TCHAR* next_path = static_cast<const TCHAR*>(next_path_str.c_str()); FF_DIR dir; @@ -54,7 +54,7 @@ auto FileGathererImpl::FindFiles( // System or hidden file. Ignore it and move on. continue; } else { - std::pmr::string full_path{&memory::kSpiRamResource}; + std::string full_path; full_path += next_path_str; full_path += "/"; full_path += info.fname; diff --git a/src/database/include/database.hpp b/src/database/include/database.hpp index 327db3cb..c75dbf96 100644 --- a/src/database/include/database.hpp +++ b/src/database/include/database.hpp @@ -35,61 +35,18 @@ namespace database { -struct Continuation { - std::pmr::string prefix; - std::pmr::string start_key; - bool forward; - bool was_prev_forward; - size_t page_size; -}; +struct SearchKey; +class Record; +class Iterator; /* - * Wrapper for a set of results from the database. Owns the list of results, as - * well as a continuation token that can be used to continue fetching more - * results if they were paginated. + * Handle to an open database. This can be used to store large amounts of + * persistent data on the SD card, in a manner that can be retrieved later very + * quickly. + * + * A database includes a number of 'indexes'. Each index is a sorted, + * hierarchical view of all the playable tracks on the device. */ -template <typename T> -class Result { - public: - auto values() const -> const std::vector<std::shared_ptr<T>>& { - return values_; - } - - auto next_page() -> std::optional<Continuation>& { return next_page_; } - auto prev_page() -> std::optional<Continuation>& { return prev_page_; } - - Result(const std::vector<std::shared_ptr<T>>&& values, - std::optional<Continuation> next, - std::optional<Continuation> prev) - : values_(values), next_page_(next), prev_page_(prev) {} - - Result(const Result&) = delete; - Result& operator=(const Result&) = delete; - - private: - std::vector<std::shared_ptr<T>> values_; - std::optional<Continuation> next_page_; - std::optional<Continuation> prev_page_; -}; - -class IndexRecord { - public: - explicit IndexRecord(const IndexKey&, - std::optional<std::pmr::string>, - std::optional<TrackId>); - - auto text() const -> std::optional<std::pmr::string>; - auto track() const -> std::optional<TrackId>; - - auto Expand(std::size_t) const -> std::optional<Continuation>; - auto ExpandHeader() const -> IndexKey::Header; - - private: - IndexKey key_; - std::optional<std::pmr::string> override_text_; - std::optional<TrackId> track_; -}; - class Database { public: enum DatabaseError { @@ -106,31 +63,19 @@ class Database { ~Database(); - auto Put(const std::string& key, const std::string& val) -> void; - auto Get(const std::string& key) -> std::optional<std::string>; - - auto Update() -> std::future<void>; + /* Adds an arbitrary record to the database. */ + auto put(const std::string& key, const std::string& val) -> void; - auto GetTrackPath(TrackId id) -> std::future<std::optional<std::pmr::string>>; + /* Retrives a value previously stored with `put`. */ + auto get(const std::string& key) -> std::optional<std::string>; - auto GetTrack(TrackId id) -> std::future<std::shared_ptr<Track>>; + auto getTrackPath(TrackId id) -> std::optional<std::string>; + auto getTrack(TrackId id) -> std::shared_ptr<Track>; - /* - * Fetches data for multiple tracks more efficiently than multiple calls to - * GetTrack. - */ - auto GetBulkTracks(std::vector<TrackId> id) - -> std::future<std::vector<std::shared_ptr<Track>>>; - - auto GetIndexes() -> std::vector<IndexInfo>; - auto GetTracksByIndex(IndexId index, std::size_t page_size) - -> std::future<Result<IndexRecord>*>; - auto GetTracks(std::size_t page_size) -> std::future<Result<Track>*>; - auto GetDump(std::size_t page_size) -> std::future<Result<std::pmr::string>*>; - - template <typename T> - auto GetPage(Continuation* c) -> std::future<Result<T>*>; + auto getIndexes() -> std::vector<IndexInfo>; + auto updateIndexes() -> void; + // Cannot be copied or moved. Database(const Database&) = delete; Database& operator=(const Database&) = delete; @@ -157,106 +102,134 @@ class Database { std::shared_ptr<tasks::Worker> worker); auto dbMintNewTrackId() -> TrackId; - auto dbEntomb(TrackId track, uint64_t hash) -> void; + auto dbEntomb(TrackId track, uint64_t hash) -> void; auto dbPutTrackData(const TrackData& s) -> void; auto dbGetTrackData(TrackId id) -> std::shared_ptr<TrackData>; auto dbPutHash(const uint64_t& hash, TrackId i) -> void; auto dbGetHash(const uint64_t& hash) -> std::optional<TrackId>; + auto dbCreateIndexesForTrack(const Track& track) -> void; auto dbRemoveIndexes(std::shared_ptr<TrackData>) -> void; + auto dbIngestTagHashes(const TrackTags&, std::pmr::unordered_map<Tag, uint64_t>&) -> void; auto dbRecoverTagsFromHashes(const std::pmr::unordered_map<Tag, uint64_t>&) -> std::shared_ptr<TrackTags>; - template <typename T> - auto dbGetPage(const Continuation& c) -> Result<T>*; + auto getRecord(const SearchKey& c) + -> std::optional<std::pair<std::pmr::string, Record>>; + auto countRecords(const SearchKey& c) -> size_t; +}; - auto dbCount(const Continuation& c) -> size_t; +/* + * Container for the data needed to iterate through database records. This is a + * lower-level type that the higher-level iterators are built from; most users + * outside this namespace shouldn't need to work with continuations. + */ +struct SearchKey { + std::pmr::string prefix; + /* If not given, then iteration starts from `prefix`. */ + std::optional<std::pmr::string> key; + int offset; - template <typename T> - auto ParseRecord(const leveldb::Slice& key, const leveldb::Slice& val) - -> std::shared_ptr<T>; + auto startKey() const -> std::string_view; }; -template <> -auto Database::ParseRecord<IndexRecord>(const leveldb::Slice& key, - const leveldb::Slice& val) - -> std::shared_ptr<IndexRecord>; -template <> -auto Database::ParseRecord<Track>(const leveldb::Slice& key, - const leveldb::Slice& val) - -> std::shared_ptr<Track>; -template <> -auto Database::ParseRecord<std::pmr::string>(const leveldb::Slice& key, - const leveldb::Slice& val) - -> std::shared_ptr<std::pmr::string>; - /* - * Utility for accessing a large set of database records, one record at a time. + * A record belonging to one of the database's indexes. This may either be a + * leaf record, containing a track id, or a branch record, containing a new + * Header to retrieve results at the next level of the index. */ -class Iterator { +class Record { public: - static auto Parse(std::weak_ptr<Database>, const cppbor::Array&) - -> std::optional<Iterator>; + Record(const IndexKey&, const leveldb::Slice&); - Iterator(std::weak_ptr<Database>, const IndexInfo&); - Iterator(std::weak_ptr<Database>, const Continuation&); - Iterator(const Iterator&); + Record(const Record&) = default; + Record& operator=(const Record& other) = default; - Iterator& operator=(const Iterator& other); + auto text() const -> std::string_view; + auto contents() const -> const std::variant<TrackId, IndexKey::Header>&; - auto database() const { return db_; } + private: + std::pmr::string text_; + std::variant<TrackId, IndexKey::Header> contents_; +}; - using Callback = std::function<void(std::optional<IndexRecord>)>; +/* + * Utility for accessing a large set of database records, one record at a time. + */ +class Iterator { + public: + Iterator(std::shared_ptr<Database>, IndexId); + Iterator(std::shared_ptr<Database>, const IndexKey::Header&); - auto Next(Callback) -> void; - auto NextSync() -> std::optional<IndexRecord>; + Iterator(const Iterator&) = default; + Iterator& operator=(const Iterator& other) = default; - auto Prev(Callback) -> void; + auto value() const -> const std::optional<Record>&; + std::optional<Record> operator*() const { return value(); } - auto PeekSync() -> std::optional<IndexRecord>; + auto next() -> void; + std::optional<Record> operator++() { + next(); + return value(); + } + std::optional<Record> operator++(int) { + auto val = value(); + next(); + return val; + } - auto Size() const -> size_t; + auto prev() -> void; + std::optional<Record> operator--() { + prev(); + return value(); + } + std::optional<Record> operator--(int) { + auto val = value(); + prev(); + return val; + } - auto cbor() const -> cppbor::Array&&; + auto count() const -> size_t; private: - Iterator(std::weak_ptr<Database>, - std::optional<Continuation>&&, - std::optional<Continuation>&&); + auto iterate(const SearchKey& key) -> void; friend class TrackIterator; - auto InvokeNull(Callback) -> void; - std::weak_ptr<Database> db_; - - std::mutex pos_mutex_; - std::optional<Continuation> current_pos_; - std::optional<Continuation> prev_pos_; + SearchKey key_; + std::optional<Record> current_; }; class TrackIterator { public: - static auto Parse(std::weak_ptr<Database>, const cppbor::Array&) - -> std::optional<TrackIterator>; - TrackIterator(const Iterator&); - TrackIterator(const TrackIterator&); - TrackIterator& operator=(TrackIterator&& other); + TrackIterator(const TrackIterator&) = default; + TrackIterator& operator=(TrackIterator&& other) = default; + + auto value() const -> std::optional<TrackId>; + std::optional<TrackId> operator*() const { return value(); } - auto Next() -> std::optional<TrackId>; - auto Size() const -> size_t; + auto next() -> void; + std::optional<TrackId> operator++() { + next(); + return value(); + } + std::optional<TrackId> operator++(int) { + auto val = value(); + next(); + return val; + } - auto cbor() const -> cppbor::Array&&; + auto count() const -> size_t; private: TrackIterator(std::weak_ptr<Database>); - - auto NextLeaf() -> void; + auto next(bool advance) -> void; std::weak_ptr<Database> db_; std::vector<Iterator> levels_; diff --git a/src/database/include/file_gatherer.hpp b/src/database/include/file_gatherer.hpp index 378727f7..66127bb7 100644 --- a/src/database/include/file_gatherer.hpp +++ b/src/database/include/file_gatherer.hpp @@ -20,16 +20,16 @@ class IFileGatherer { virtual ~IFileGatherer(){}; virtual auto FindFiles( - const std::pmr::string& root, - std::function<void(const std::pmr::string&, const FILINFO&)> cb) + const std::string& root, + std::function<void(const std::string&, const FILINFO&)> cb) -> void = 0; }; class FileGathererImpl : public IFileGatherer { public: virtual auto FindFiles( - const std::pmr::string& root, - std::function<void(const std::pmr::string&, const FILINFO&)> cb) + const std::string& root, + std::function<void(const std::string&, const FILINFO&)> cb) -> void override; }; diff --git a/src/database/include/tag_parser.hpp b/src/database/include/tag_parser.hpp index 04817c59..977c9afc 100644 --- a/src/database/include/tag_parser.hpp +++ b/src/database/include/tag_parser.hpp @@ -16,24 +16,24 @@ namespace database { class ITagParser { public: virtual ~ITagParser() {} - virtual auto ReadAndParseTags(const std::pmr::string& path) + virtual auto ReadAndParseTags(const std::string& path) -> std::shared_ptr<TrackTags> = 0; }; class GenericTagParser : public ITagParser { public: - auto ReadAndParseTags(const std::pmr::string& path) + auto ReadAndParseTags(const std::string& path) -> std::shared_ptr<TrackTags> override; }; class TagParserImpl : public ITagParser { public: TagParserImpl(); - auto ReadAndParseTags(const std::pmr::string& path) + auto ReadAndParseTags(const std::string& path) -> std::shared_ptr<TrackTags> override; private: - std::map<std::pmr::string, std::unique_ptr<ITagParser>> extension_to_parser_; + std::map<std::string, std::unique_ptr<ITagParser>> extension_to_parser_; GenericTagParser generic_parser_; /* @@ -43,14 +43,14 @@ class TagParserImpl : public ITagParser { std::mutex cache_mutex_; util::LruCache<16, std::pmr::string, std::shared_ptr<TrackTags>> cache_; - // We could also consider keeping caches of artist name -> std::pmr::string - // and similar. This hasn't been done yet, as this isn't a common workload in + // We could also consider keeping caches of artist name -> std::string and + // similar. This hasn't been done yet, as this isn't a common workload in // any of our UI. }; class OpusTagParser : public ITagParser { public: - auto ReadAndParseTags(const std::pmr::string& path) + auto ReadAndParseTags(const std::string& path) -> std::shared_ptr<TrackTags> override; }; diff --git a/src/database/include/track.hpp b/src/database/include/track.hpp index 8a24024f..0497c94d 100644 --- a/src/database/include/track.hpp +++ b/src/database/include/track.hpp @@ -123,7 +123,7 @@ struct TrackData { public: TrackData() : id(0), - filepath(&memory::kSpiRamResource), + filepath(), tags_hash(0), individual_tag_hashes(&memory::kSpiRamResource), is_tombstoned(false), diff --git a/src/database/tag_parser.cpp b/src/database/tag_parser.cpp index eb5f3a43..885c71dd 100644 --- a/src/database/tag_parser.cpp +++ b/src/database/tag_parser.cpp @@ -32,7 +32,7 @@ const static std::array<std::pair<const char*, Tag>, 5> kVorbisIdToTag = {{ static auto convert_track_number(int number) -> std::pmr::string { std::ostringstream oss; oss << std::setw(4) << std::setfill('0') << number; - return std::pmr::string(oss.str()); + return std::pmr::string(oss.str(), &memory::kSpiRamResource); } static auto convert_track_number(const std::pmr::string& raw) @@ -131,11 +131,12 @@ TagParserImpl::TagParserImpl() { extension_to_parser_["opus"] = std::make_unique<OpusTagParser>(); } -auto TagParserImpl::ReadAndParseTags(const std::pmr::string& path) +auto TagParserImpl::ReadAndParseTags(const std::string& path) -> std::shared_ptr<TrackTags> { { std::lock_guard<std::mutex> lock{cache_mutex_}; - std::optional<std::shared_ptr<TrackTags>> cached = cache_.Get(path); + std::optional<std::shared_ptr<TrackTags>> cached = + cache_.Get({path.data(), path.size()}); if (cached) { return *cached; } @@ -143,8 +144,8 @@ auto TagParserImpl::ReadAndParseTags(const std::pmr::string& path) ITagParser* parser = &generic_parser_; auto dot_pos = path.find_last_of("."); - if (dot_pos != std::pmr::string::npos && path.size() - dot_pos > 1) { - std::pmr::string extension = path.substr(dot_pos + 1); + if (dot_pos != std::string::npos && path.size() - dot_pos > 1) { + std::string extension = path.substr(dot_pos + 1); std::transform(extension.begin(), extension.end(), extension.begin(), [](unsigned char c) { return std::tolower(c); }); if (extension_to_parser_.contains(extension)) { @@ -162,8 +163,9 @@ auto TagParserImpl::ReadAndParseTags(const std::pmr::string& path) // start. if (!tags->at(Tag::kAlbumTrack)) { auto slash_pos = path.find_last_of("/"); - if (slash_pos != std::pmr::string::npos && path.size() - slash_pos > 1) { - tags->set(Tag::kAlbumTrack, path.substr(slash_pos + 1)); + if (slash_pos != std::string::npos && path.size() - slash_pos > 1) { + std::string trunc = path.substr(slash_pos + 1); + tags->set(Tag::kAlbumTrack, {trunc.data(), trunc.size()}); } } @@ -174,13 +176,13 @@ auto TagParserImpl::ReadAndParseTags(const std::pmr::string& path) { std::lock_guard<std::mutex> lock{cache_mutex_}; - cache_.Put(path, tags); + cache_.Put({path.data(), path.size(), &memory::kSpiRamResource}, tags); } return tags; } -auto GenericTagParser::ReadAndParseTags(const std::pmr::string& path) +auto GenericTagParser::ReadAndParseTags(const std::string& path) -> std::shared_ptr<TrackTags> { libtags::Aux aux; auto out = std::make_shared<TrackTags>(); @@ -254,10 +256,10 @@ auto GenericTagParser::ReadAndParseTags(const std::pmr::string& path) return out; } -auto OpusTagParser::ReadAndParseTags(const std::pmr::string& path) +auto OpusTagParser::ReadAndParseTags(const std::string& path) -> std::shared_ptr<TrackTags> { auto lock = drivers::acquire_spi(); - std::pmr::string vfs_path = "/sdcard" + path; + std::string vfs_path = "/sdcard" + path; int err; OggOpusFile* f = op_test_file(vfs_path.c_str(), &err); if (f == NULL) { |