Add directory verification

Cargo.toml

@@ -36,6 +36,7 @@ libc = "0.2.186"
 num-traits = "0.2.19"
 owo-colors = "4"
 rand = "0.8.5"
+redb = "4.1.0"
 regex = "1.10.6"
 reqwest = { version = "0.13.3", features = ["blocking", "http2"] }
 rust-embed = { version = "8.11.0", features = ["mime-guess"] }

src/database.rs

@@ -0,0 +1,85 @@
+use {
+  super::*,
+  crate::server::DatabaseMetadata,
+  redb::{Key, TypeName, Value},
+};
+
+impl Key for DatabaseMetadata {
+  fn compare(a: &[u8], b: &[u8]) -> Ordering {
+    u64::compare(a, b)
+  }
+}
+
+impl Value for DatabaseMetadata {
+  type AsBytes<'a>
+    = <u64 as Value>::AsBytes<'a>
+  where
+    Self: 'a;
+
+  type SelfType<'a>
+    = Self
+  where
+    Self: 'a;
+
+  fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a>
+  where
+    Self: 'b,
+  {
+    u64::as_bytes(&(*value as u64))
+  }
+
+  fn fixed_width() -> Option<usize> {
+    u64::fixed_width()
+  }
+
+  fn from_bytes<'a>(data: &'a [u8]) -> Self
+  where
+    Self: 'a,
+  {
+    Self::from_repr(u64::from_bytes(data)).unwrap()
+  }
+
+  fn type_name() -> TypeName {
+    TypeName::new("filepack-metadata-key")
+  }
+}
+
+impl Key for Hash {
+  fn compare(a: &[u8], b: &[u8]) -> Ordering {
+    a.cmp(b)
+  }
+}
+
+impl Value for Hash {
+  type AsBytes<'a>
+    = &'a [u8; Self::LEN]
+  where
+    Self: 'a;
+
+  type SelfType<'a>
+    = Hash
+  where
+    Self: 'a;
+
+  fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a>
+  where
+    Self: 'b,
+  {
+    value.as_bytes()
+  }
+
+  fn fixed_width() -> Option<usize> {
+    Some(Self::LEN)
+  }
+
+  fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a>
+  where
+    Self: 'a,
+  {
+    <[u8; Self::LEN]>::try_from(data).unwrap().into()
+  }
+
+  fn type_name() -> TypeName {
+    TypeName::new("filepack-hash")
+  }
+}

src/error.rs

@@ -11,11 +11,6 @@ pub enum Error {
     actual: Hash,
     expected: Hash,
   },
-  #[snafu(display("cannot use authentication with non-HTTPS server `{server}`"))]
-  AuthenticationOverHttp {
-    backtrace: Option<Backtrace>,
-    server: Url,
-  },
   #[snafu(display("failed to decode bech32 `{bech32}`"))]
   Bech32Decode {
     backtrace: Option<Backtrace>,
@@ -67,6 +62,24 @@ pub enum Error {
   },
   #[snafu(display("failed to get local data directory"))]
   DataLocalDir { backtrace: Option<Backtrace> },
+  #[snafu(transparent)]
+  Database { source: redb::DatabaseError },
+  #[snafu(transparent)]
+  DatabaseCommit { source: redb::CommitError },
+  #[snafu(display("database schema version `{actual}` does not match expected `{expected}`"))]
+  DatabaseSchemaVersionMismatch {
+    actual: u64,
+    backtrace: Option<Backtrace>,
+    expected: u64,
+  },
+  #[snafu(display("database schema version missing"))]
+  DatabaseSchemaVersionMissing { backtrace: Option<Backtrace> },
+  #[snafu(transparent)]
+  DatabaseStorage { source: redb::StorageError },
+  #[snafu(transparent)]
+  DatabaseTable { source: redb::TableError },
+  #[snafu(transparent)]
+  DatabaseTransaction { source: redb::TransactionError },
   #[snafu(display("failed to decode manifest at `{path}`"))]
   DecodeManifest {
     backtrace: Option<Backtrace>,
@@ -383,6 +396,8 @@ pub enum Error {
     backtrace: Option<Backtrace>,
     source: SystemTimeError,
   },
+  #[snafu(display("authentication tokens may only be used over HTTPS or loopback"))]
+  TokenOverHttp { backtrace: Option<Backtrace> },
   #[snafu(display("failed to unarchive manifest"))]
   UnarchiveManifest {
     backtrace: Option<Backtrace>,

src/lib.rs

@@ -206,6 +206,7 @@ mod component_error;
 mod context;
 mod count;
 mod dalek_signature_error;
+mod database;
 mod date_time;
 mod decode;
 mod decode_error;

src/server.rs

@@ -1,11 +1,39 @@
-use super::*;
+use {
+  super::*,
+  redb::{Database, ReadableDatabase, ReadableTable, TableDefinition},
+};
+
+const DIRECTORIES: TableDefinition<Hash, ()> = TableDefinition::new("directories");
+const METADATA: TableDefinition<DatabaseMetadata, u64> = TableDefinition::new("metadata");
+const SCHEMA_VERSION: u64 = 0;
+
+#[derive(Copy, Clone, Debug, FromRepr)]
+#[repr(u64)]
+pub(crate) enum DatabaseMetadata {
+  Schema = 0,
+}
 
 pub(crate) struct Server {
+  database: Database,
   files: Utf8PathBuf,
   incoming: Utf8PathBuf,
 }
 
 impl Server {
+  pub(crate) async fn directory(&self, hash: Hash) -> ServerResult<Directory> {
+    ensure!(
+      self
+        .database
+        .begin_read()?
+        .open_table(DIRECTORIES)?
+        .get(&hash)?
+        .is_some(),
+      server_error::DirectoryNotFound { hash },
+    );
+
+    self.read_directory(hash).await
+  }
+
   fn file_path(&self, hash: Hash) -> Utf8PathBuf {
     self.files.join(hash.to_string())
   }
@@ -35,16 +63,13 @@ impl Server {
   pub(crate) async fn open_file(&self, hash: Hash) -> ServerResult<(tokio::fs::File, u64)> {
     let path = self.file_path(hash);
 
-    let file = match tokio::fs::File::open(&path).await {
-      Err(err) => {
-        return if err.kind() == io::ErrorKind::NotFound {
-          Err(server_error::FileNotFound { hash }.into_error(err))
-        } else {
-          Err(server_error::FilesystemIo { path }.into_error(err))
-        };
+    let file = tokio::fs::File::open(&path).await.map_err(|err| {
+      if err.kind() == io::ErrorKind::NotFound {
+        server_error::FileNotFound { hash }.into_error(err)
+      } else {
+        server_error::FilesystemIo { path: &path }.into_error(err)
       }
-      Ok(file) => file,
-    };
+    })?;
 
     let len = file
       .metadata()
@@ -55,14 +80,106 @@ impl Server {
     Ok((file, len))
   }
 
+  async fn read_directory(&self, hash: Hash) -> ServerResult<Directory> {
+    let path = self.file_path(hash);
+
+    let cbor = tokio::fs::read(&path).await.map_err(|err| {
+      if err.kind() == io::ErrorKind::NotFound {
+        server_error::FileNotFound { hash }.into_error(err)
+      } else {
+        server_error::FilesystemIo { path }.into_error(err)
+      }
+    })?;
+
+    Directory::decode_from_slice(&cbor).context(server_error::DirectoryDecode { hash })
+  }
+
+  pub(crate) async fn verify_directory(&self, hash: Hash) -> ServerResult {
+    let directory = self.read_directory(hash).await?;
+
+    for entry in directory.entries.values() {
+      if entry.ty == EntryType::File {
+        let path = self.file_path(entry.hash);
+        ensure!(
+          tokio::fs::try_exists(&path)
+            .await
+            .context(server_error::FilesystemIo { path })?,
+          server_error::DirectoryFileMissing {
+            directory: hash,
+            file: entry.hash,
+          },
+        );
+      }
+    }
+
+    let tx = self.database.begin_write()?;
+
+    {
+      let mut directories = tx.open_table(DIRECTORIES)?;
+
+      for entry in directory.entries.values() {
+        if entry.ty == EntryType::Directory {
+          ensure!(
+            directories.get(&entry.hash)?.is_some(),
+            server_error::DirectoryUnverified {
+              directory: hash,
+              subdirectory: entry.hash,
+            },
+          );
+        }
+      }
+
+      directories.insert(&hash, &())?;
+    }
+
+    tx.commit()?;
+
+    Ok(())
+  }
+
   pub(crate) fn with_data_dir(data_dir: &Utf8Path) -> Result<Self> {
+    let database = Database::create(data_dir.join("database.redb"))?;
+
+    let tx = database.begin_write()?;
+
+    if tx.list_tables()?.count() == 0 && tx.list_multimap_tables()?.count() == 0 {
+      {
+        tx.open_table(METADATA)?
+          .insert(DatabaseMetadata::Schema, &SCHEMA_VERSION)?;
+
+        tx.open_table(DIRECTORIES)?;
+      }
+
+      tx.commit()?;
+    } else {
+      let actual = tx
+        .open_table(METADATA)?
+        .get(DatabaseMetadata::Schema)?
+        .context(error::DatabaseSchemaVersionMissing)?
+        .value();
+
+      ensure!(
+        actual == SCHEMA_VERSION,
+        error::DatabaseSchemaVersionMismatch {
+          actual,
+          expected: SCHEMA_VERSION,
+        },
+      );
+
+      drop(tx);
+    }
+
     let files = data_dir.join("files");
     filesystem::create_dir_all(&files)?;
 
     let incoming = data_dir.join("incoming");
     filesystem::create_dir_all(&incoming)?;
 
-    Ok(Self { files, incoming })
+    Ok(Self {
+      database,
+      files,
+      incoming,
+    })
   }
 
   pub(crate) async fn write_file(&self, hash: Hash, body: Body) -> ServerResult {
@@ -124,3 +241,49 @@ impl Server {
     Ok(())
   }
 }
+
+#[cfg(test)]
+mod tests {
+  use super::*;
+
+  #[test]
+  fn database_schema_version_mismatch() {
+    let (_tempdir, data_dir) = tempdir();
+
+    {
+      let database = Database::create(data_dir.join("database.redb")).unwrap();
+      let tx = database.begin_write().unwrap();
+      tx.open_table(METADATA)
+        .unwrap()
+        .insert(DatabaseMetadata::Schema, &SCHEMA_VERSION + 1)
+        .unwrap();
+      tx.commit().unwrap();
+    }
+
+    assert_matches!(
+      Server::with_data_dir(&data_dir).map(drop),
+      Err(Error::DatabaseSchemaVersionMismatch {
+        actual,
+        backtrace: _,
+        expected: SCHEMA_VERSION,
+      }) if actual == SCHEMA_VERSION + 1,
+    );
+  }
+
+  #[test]
+  fn database_schema_version_missing() {
+    let (_tempdir, data_dir) = tempdir();
+
+    {
+      let database = Database::create(data_dir.join("database.redb")).unwrap();
+      let tx = database.begin_write().unwrap();
+      tx.open_table(DIRECTORIES).unwrap();
+      tx.commit().unwrap();
+    }
+
+    assert_matches!(
+      Server::with_data_dir(&data_dir).map(drop),
+      Err(Error::DatabaseSchemaVersionMissing { backtrace: _ }),
+    );
+  }
+}