add leveldb

7 files changed, 385 insertions, 0 deletions

diff --git a/src/database/include/database.hpp b/src/database/include/database.hpp
new file mode 100644
index 00000000..cfef0a7d
--- /dev/null
+++ b/src/database/include/database.hpp
@@ -0,0 +1,27 @@
+#pragma once
+
+#include <memory>
+
+#include "leveldb/cache.h"
+#include "leveldb/db.h"
+#include "result.hpp"
+
+namespace database {
+
+class Database {
+ public:
+  enum DatabaseError {
+    FAILED_TO_OPEN,
+  };
+  static auto Open() -> cpp::result<Database*, DatabaseError>;
+
+  ~Database();
+
+ private:
+  std::unique_ptr<leveldb::DB> db_;
+  std::unique_ptr<leveldb::Cache> cache_;
+
+  Database(leveldb::DB* db, leveldb::Cache* cache);
+};
+
+}  // namespace database
diff --git a/src/database/include/env_esp.hpp b/src/database/include/env_esp.hpp
new file mode 100644
index 00000000..c9b3fbc8
--- /dev/null
+++ b/src/database/include/env_esp.hpp
@@ -0,0 +1,152 @@
+#pragma once
+
+#include <mutex>
+#include <set>
+
+#include "freertos/FreeRTOS.h"
+#include "freertos/queue.h"
+#include "leveldb/env.h"
+#include "leveldb/status.h"
+
+namespace leveldb {
+
+// Tracks the files locked by EspEnv::LockFile().
+//
+// We maintain a separate set instead of relying on fcntl(F_SETLK) because
+// fcntl(F_SETLK) does not provide any protection against multiple uses from the
+// same process.
+//
+// Instances are thread-safe because all member data is guarded by a mutex.
+class InMemoryLockTable {
+ public:
+  bool Insert(const std::string& fname) {
+    mu_.lock();
+    bool succeeded = locked_files_.insert(fname).second;
+    mu_.unlock();
+    return succeeded;
+  }
+  void Remove(const std::string& fname) {
+    mu_.lock();
+    locked_files_.erase(fname);
+    mu_.unlock();
+  }
+
+ private:
+  std::mutex mu_;
+  std::set<std::string> locked_files_;
+};
+
+class EspEnv : public leveldb::Env {
+ public:
+  EspEnv();
+  ~EspEnv() override;
+
+  Status NewSequentialFile(const std::string& filename,
+                           SequentialFile** result) override;
+
+  Status NewRandomAccessFile(const std::string& filename,
+                             RandomAccessFile** result) override;
+
+  Status NewWritableFile(const std::string& filename,
+                         WritableFile** result) override;
+
+  Status NewAppendableFile(const std::string& filename,
+                           WritableFile** result) override;
+
+  bool FileExists(const std::string& filename) override;
+
+  Status GetChildren(const std::string& directory_path,
+                     std::vector<std::string>* result) override;
+
+  Status RemoveFile(const std::string& filename) override;
+
+  Status CreateDir(const std::string& dirname) override;
+
+  Status RemoveDir(const std::string& dirname) override;
+
+  Status GetFileSize(const std::string& filename, uint64_t* size) override;
+
+  Status RenameFile(const std::string& from, const std::string& to) override;
+
+  Status LockFile(const std::string& filename, FileLock** lock) override;
+
+  Status UnlockFile(FileLock* lock) override;
+
+  void Schedule(void (*background_work_function)(void* background_work_arg),
+                void* background_work_arg) override;
+
+  void StartThread(void (*thread_main)(void* thread_main_arg),
+                   void* thread_main_arg) override;
+
+  Status GetTestDirectory(std::string* result) override;
+
+  Status NewLogger(const std::string& filename, Logger** result) override;
+
+  uint64_t NowMicros() override;
+
+  void SleepForMicroseconds(int micros) override;
+
+  void BackgroundThreadMain();
+
+ private:
+  // Stores the work item data in a Schedule() call.
+  //
+  // Instances are constructed on the thread calling Schedule() and used on the
+  // background thread.
+  //
+  // This structure is thread-safe beacuse it is immutable.
+  struct BackgroundWorkItem {
+    explicit BackgroundWorkItem(void (*f)(void*), void* a)
+        : function(f), arg(a) {}
+
+    void (*const function)(void*);
+    void* const arg;
+  };
+
+  std::mutex background_work_mutex_;
+  bool started_background_thread_;
+
+  QueueHandle_t background_work_queue_;
+
+  InMemoryLockTable locks_;  // Thread-safe.
+};
+
+} // namespace leveldb
+
+namespace database {
+
+// Wraps an Env instance whose destructor is never created.
+//
+// Intended usage:
+//   using PlatformSingletonEnv = SingletonEnv<PlatformEnv>;
+//   void ConfigurePosixEnv(int param) {
+//     PlatformSingletonEnv::AssertEnvNotInitialized();
+//     // set global configuration flags.
+//   }
+//   Env* Env::Default() {
+//     static PlatformSingletonEnv default_env;
+//     return default_env.env();
+//   }
+template <typename EnvType>
+class SingletonEnv {
+ public:
+  SingletonEnv() {
+    static_assert(sizeof(env_storage_) >= sizeof(EnvType),
+                  "env_storage_ will not fit the Env");
+    static_assert(alignof(decltype(env_storage_)) >= alignof(EnvType),
+                  "env_storage_ does not meet the Env's alignment needs");
+    new (&env_storage_) EnvType();
+  }
+  ~SingletonEnv() = default;
+
+  SingletonEnv(const SingletonEnv&) = delete;
+  SingletonEnv& operator=(const SingletonEnv&) = delete;
+
+  leveldb::Env* env() { return reinterpret_cast<leveldb::Env*>(&env_storage_); }
+
+ private:
+  typename std::aligned_storage<sizeof(EnvType), alignof(EnvType)>::type
+      env_storage_;
+};
+
+} // namespace database
diff --git a/src/database/include/file_gatherer.hpp b/src/database/include/file_gatherer.hpp
new file mode 100644
index 00000000..47d40f88
--- /dev/null
+++ b/src/database/include/file_gatherer.hpp
@@ -0,0 +1,58 @@
+#pragma once
+
+#include <deque>
+#include <functional>
+#include <sstream>
+#include <string>
+
+#include "ff.h"
+
+namespace database {
+
+static_assert(sizeof(TCHAR) == sizeof(char), "TCHAR must be CHAR");
+
+template <typename Callback>
+auto FindFiles(FATFS* fs, const std::string& root, Callback cb) -> void {
+  std::deque<std::string> to_explore;
+  to_explore.push_back(root);
+
+  while (!to_explore.empty()) {
+    std::string next_path_str = to_explore.front();
+    const TCHAR* next_path = static_cast<const TCHAR*>(next_path_str.c_str());
+
+    DIR dir;
+    FRESULT res = f_opendir(&dir, next_path);
+    if (res != FR_OK) {
+      // TODO: log.
+      continue;
+    }
+
+    for (;;) {
+      FILINFO info;
+      res = f_readdir(&dir, &info);
+      if (info.fname == NULL) {
+        // No more files in the directory.
+        break;
+      } else if (info.fattrib & (AM_HID | AM_SYS)) {
+        // System or hidden file. Ignore it and move on.
+        continue;
+      } else {
+        std::stringstream full_path;
+        full_path << next_path_str << "/" << info.fname;
+
+        if (info.fattrib & AM_DIR) {
+          // This is a directory. Add it to the explore queue.
+          to_explore.push_back(full_path.str());
+        } else {
+          // This is a file! Let the callback know about it.
+          std::invoke(cb, full_path.str(), info);
+        }
+      }
+    }
+
+    f_closedir(&dir);
+    to_explore.pop_front();
+  }
+}
+
+}  // namespace database
diff --git a/src/database/include/table.hpp b/src/database/include/table.hpp
new file mode 100644
index 00000000..438c23b6
--- /dev/null
+++ b/src/database/include/table.hpp
@@ -0,0 +1,87 @@
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <string>
+#include <utility>
+
+#include "esp32/himem.h"
+#include "ff.h"
+#include "span.hpp"
+#include "sys/_stdint.h"
+
+namespace database {
+
+// Types used for indexing into files on disk. These should, at minimum, match
+// the size of the types that the underlying filesystem uses to address within
+// files. FAT32 uses 32 bit address. If we drop this and just support exFAT, we
+// can change these to 64 bit types.
+typedef uint32_t Index_t;
+typedef Index_t IndexOffset_t;
+
+// The amount of memory that will be used to page database columns in from disk.
+// Currently we only use a single 'page' in PSRAM per column, but with some
+// refactoring we could easily page more.
+// Keep this value 32KiB-aligned for himem compatibility.
+extern const std::size_t kRamBlockSize;
+
+struct DatabaseHeader {
+  uint32_t magic_number;
+  uint16_t db_version;
+  Index_t num_indices;
+};
+
+struct DatabaseEntry {
+  uint8_t type;
+  std::string path;
+
+  std::string title;
+  std::string album;
+  std::string artist;
+  std::string album_artist;
+};
+
+struct IndexEntry {
+  uint8_t type;
+  IndexOffset_t path;
+
+  IndexOffset_t title;
+  IndexOffset_t album;
+  IndexOffset_t artist;
+  IndexOffset_t album_artist;
+};
+
+struct RowData {
+  std::unique_ptr<std::byte[]> arr;
+  std::size_t length;
+};
+
+// Representation of a single column of data. Each column is simply a tightly
+// packed list of [size, [bytes, ...]] pairs. Callers are responsible for
+// parsing and encoding the actual bytes themselves.
+class Column {
+ public:
+  static auto Open(std::string) -> std::optional<Column>;
+
+  Column(FIL file, std::size_t file_size);
+  ~Column();
+
+  auto ReadDataAtOffset(esp_himem_rangehandle_t, IndexOffset_t) -> RowData;
+  auto AppendRow(cpp::span<std::byte> row) -> bool;
+  auto FlushChanges() -> void;
+
+ private:
+  FIL file_;
+  IndexOffset_t length_;
+
+  esp_himem_handle_t block_;
+  std::pair<IndexOffset_t, IndexOffset_t> loaded_range_;
+
+  auto IsOffsetLoaded(IndexOffset_t offset) -> bool;
+  auto LoadOffsetFromDisk(cpp::span<std::byte> dest, IndexOffset_t offset)
+      -> bool;
+};
+
+}  // namespace database
diff --git a/src/database/include/table_reader.hpp b/src/database/include/table_reader.hpp
new file mode 100644
index 00000000..9f7cc4ee
--- /dev/null
+++ b/src/database/include/table_reader.hpp
@@ -0,0 +1,53 @@
+#pragma once
+
+#include <string>
+
+#include "result.hpp"
+#include "span.hpp"
+
+#include "table.hpp"
+
+namespace database {
+
+class TableReader {
+ public:
+  enum ReadError {
+    OUT_OF_RANGE,
+    IO_ERROR,
+    PARSE_ERROR,
+  };
+
+  auto ReadEntryAtIndex(Index_t index) -> cpp::result<DatabaseEntry, ReadError>;
+
+  template <typename T>
+  auto ReadColumnOffsetAtIndex(Column<T> col, Index_t index)
+      -> cpp::result<IndexOffset_t, ReadError>;
+
+  template <typename T>
+  auto ParseColumnAtIndex(Column<T> col, Index_t index)
+      -> cpp::result<T, ReadError> {
+    return ReadColumnOffsetAtIndex(col, index).map([&](IndexOffset_t offset) {
+      return ReadColumnAtOffset(col, offset);
+    });
+  }
+
+  template <typename T>
+  auto ParseColumnAtOffset(Column<T> col, IndexOffset_t offset)
+      -> cpp::result<T, ReadError> {
+    return ReadDataAtOffset(col.Filename(), offset)
+        .flat_map([&](cpp::span<std::byte> data) {
+          auto res = = col.ParseValue(data);
+          if (res) {
+            return *res;
+          } else {
+            return cpp::fail(PARSE_ERROR);
+          }
+        });
+  }
+
+ private:
+  auto ReadDataAtOffset(std::string filename, IndexOffset_t offset)
+      -> cpp::span<std::byte>;
+};
+
+}  // namespace database
diff --git a/src/database/include/table_writer.hpp b/src/database/include/table_writer.hpp
new file mode 100644
index 00000000..9e01dd9d
--- /dev/null
+++ b/src/database/include/table_writer.hpp
@@ -0,0 +1,5 @@
+#pragma once
+
+#include "table.hpp"
+
+namespace database {}  // namespace database
diff --git a/src/database/include/tag_processor.hpp b/src/database/include/tag_processor.hpp
new file mode 100644
index 00000000..0257fc92
--- /dev/null
+++ b/src/database/include/tag_processor.hpp
@@ -0,0 +1,3 @@
+#pragma once
+
+namespace database {}  // namespace database