From 28cf749951a8f811606bb233efecfd36738c3c89 Mon Sep 17 00:00:00 2001
From: jacqueline <me@jacqueline.id.au>
Date: Thu, 8 Aug 2024 16:08:46 +1000
Subject: Make FileGatherer shaped more like a normal iterator

---
 src/tangara/database/track_finder.cpp | 84 +++++++++++++++++++++++++++++++++++
 1 file changed, 84 insertions(+)
 create mode 100644 src/tangara/database/track_finder.cpp

(limited to 'src/tangara/database/track_finder.cpp')

diff --git a/src/tangara/database/track_finder.cpp b/src/tangara/database/track_finder.cpp
new file mode 100644
index 00000000..86948e70
--- /dev/null
+++ b/src/tangara/database/track_finder.cpp
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2023 jacqueline <me@jacqueline.id.au>
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+#include "database/track_finder.hpp"
+
+#include <deque>
+#include <functional>
+#include <memory>
+#include <mutex>
+#include <sstream>
+#include <string>
+#include <string_view>
+
+#include "database/track_finder.hpp"
+#include "ff.h"
+
+#include "drivers/spi.hpp"
+#include "memory_resource.hpp"
+
+namespace database {
+
+static_assert(sizeof(TCHAR) == sizeof(char), "TCHAR must be CHAR");
+
+TrackFinder::TrackFinder(std::string_view root)
+    : to_explore_(&memory::kSpiRamResource) {
+  to_explore_.push_back({root.data(), root.size()});
+}
+
+auto TrackFinder::next(FILINFO& out_info) -> std::optional<std::string> {
+  std::scoped_lock<std::mutex> lock{mut_};
+  while (!to_explore_.empty() || current_) {
+    if (!current_) {
+      current_.emplace();
+
+      // Get the next directory to iterate through.
+      current_->first = to_explore_.front();
+      to_explore_.pop_front();
+      const TCHAR* next_path =
+          static_cast<const TCHAR*>(current_->first.data());
+
+      // Open it for iterating.
+      FRESULT res = f_opendir(&current_->second, next_path);
+      if (res != FR_OK) {
+        current_.reset();
+        continue;
+      }
+    }
+
+    FILINFO info;
+    FRESULT res = f_readdir(&current_->second, &info);
+    if (res != FR_OK || info.fname[0] == 0) {
+      // No more files in the directory.
+      f_closedir(&current_->second);
+      current_.reset();
+      continue;
+    } else if (info.fattrib & (AM_HID | AM_SYS) || info.fname[0] == '.') {
+      // System or hidden file. Ignore it and move on.
+      continue;
+    } else {
+      // A valid file or folder.
+      std::pmr::string full_path{&memory::kSpiRamResource};
+      full_path += current_->first;
+      full_path += "/";
+      full_path += info.fname;
+
+      if (info.fattrib & AM_DIR) {
+        // This is a directory. Add it to the explore queue.
+        to_explore_.push_back(full_path);
+      } else {
+        // This is a file! We can return now.
+        out_info = info;
+        return {{full_path.data(), full_path.size()}};
+      }
+    }
+  }
+
+  // Out of things to explore.
+  return {};
+}
+
+}  // namespace database
-- 
cgit v1.2.3


From 2ad83cb2108dc55c9eb0573b0645513a1e8a61f5 Mon Sep 17 00:00:00 2001
From: jacqueline <me@jacqueline.id.au>
Date: Fri, 9 Aug 2024 11:43:48 +1000
Subject: Shard searching for new tracks across multiple tasks

This also has the effect of breaking up the enormous 'updateIndexes'
method into one call per file, which means database updates also no
longer monopolise a single background task for their entire duration.

avg. time per new file is now <140ms for a completely fresh database,
which is pretty good i think!
---
 src/tangara/database/track_finder.cpp | 44 +++++++++++++++++++++++++++++++----
 1 file changed, 39 insertions(+), 5 deletions(-)

(limited to 'src/tangara/database/track_finder.cpp')

diff --git a/src/tangara/database/track_finder.cpp b/src/tangara/database/track_finder.cpp
index 86948e70..21a44339 100644
--- a/src/tangara/database/track_finder.cpp
+++ b/src/tangara/database/track_finder.cpp
@@ -24,12 +24,12 @@ namespace database {
 
 static_assert(sizeof(TCHAR) == sizeof(char), "TCHAR must be CHAR");
 
-TrackFinder::TrackFinder(std::string_view root)
+CandidateIterator::CandidateIterator(std::string_view root)
     : to_explore_(&memory::kSpiRamResource) {
   to_explore_.push_back({root.data(), root.size()});
 }
 
-auto TrackFinder::next(FILINFO& out_info) -> std::optional<std::string> {
+auto CandidateIterator::next(FILINFO& info) -> std::optional<std::string> {
   std::scoped_lock<std::mutex> lock{mut_};
   while (!to_explore_.empty() || current_) {
     if (!current_) {
@@ -49,7 +49,6 @@ auto TrackFinder::next(FILINFO& out_info) -> std::optional<std::string> {
       }
     }
 
-    FILINFO info;
     FRESULT res = f_readdir(&current_->second, &info);
     if (res != FR_OK || info.fname[0] == 0) {
       // No more files in the directory.
@@ -71,14 +70,49 @@ auto TrackFinder::next(FILINFO& out_info) -> std::optional<std::string> {
         to_explore_.push_back(full_path);
       } else {
         // This is a file! We can return now.
-        out_info = info;
         return {{full_path.data(), full_path.size()}};
       }
     }
   }
 
-  // Out of things to explore.
+  // Out of paths to explore.
   return {};
 }
 
+TrackFinder::TrackFinder(
+    tasks::WorkerPool& pool,
+    size_t parallelism,
+    std::function<void(FILINFO&, std::string_view)> processor,
+    std::function<void()> complete_cb)
+    : pool_{pool},
+      parallelism_(parallelism),
+      processor_(processor),
+      complete_cb_(complete_cb) {}
+
+auto TrackFinder::launch(std::string_view root) -> void {
+  iterator_ = std::make_unique<CandidateIterator>(root);
+  num_workers_ = parallelism_;
+  for (size_t i = 0; i < parallelism_; i++) {
+    schedule();
+  }
+}
+
+auto TrackFinder::schedule() -> void {
+  pool_.Dispatch<void>([&]() {
+    FILINFO info;
+    auto next = iterator_->next(info);
+    if (next) {
+      std::invoke(processor_, info, *next);
+      schedule();
+    } else {
+      std::scoped_lock<std::mutex> lock{workers_mutex_};
+      num_workers_ -= 1;
+      if (num_workers_ == 0) {
+        iterator_.reset();
+        std::invoke(complete_cb_);
+      }
+    }
+  });
+}
+
 }  // namespace database
-- 
cgit v1.2.3