feat: Add a function to get the exact size of a PublicFile and `Pri…

…vateFile` (#405) Adds a `size` function to `PublicFile` and `PrivateFile`, and their wasm equivalents, that returns the exact size of the contents without downloading all the content blocks. Added doc tests for the core lib, and tests for the wasm lib. --- * feat: Add a function to get the exact size of a PublicFile and PrivateFile * refactor: Change `get_size` return type to `u64` * chore: Fix nightly lints * refactor: Rename `get_size` -> `size` in `wnfs` crate * chore: Fix nightly lints & wasm --------- Co-authored-by: Philipp Krüger <[email protected]>
wnfs-wg · Feb 28, 2024 · 17c980c · 17c980c
1 parent 8ac4021
commit 17c980c
Show file tree

Hide file tree

Showing 12 changed files with 215 additions and 69 deletions.
diff --git a/wnfs-common/src/metadata.rs b/wnfs-common/src/metadata.rs
@@ -7,7 +7,7 @@ use serde::{
     de::{DeserializeOwned, Error as DeError},
     Deserialize, Deserializer, Serialize, Serializer,
 };
-use std::{collections::BTreeMap, convert::TryInto, fmt::Display};
+use std::{collections::BTreeMap, fmt::Display};
 
 //--------------------------------------------------------------------------------------------------
 // Type Definitions

diff --git a/wnfs-common/src/storable.rs b/wnfs-common/src/storable.rs
@@ -35,8 +35,6 @@ macro_rules! impl_storable_from_serde {
     };
 }
 
-pub use impl_storable_from_serde;
-
 //--------------------------------------------------------------------------------------------------
 // Type Definitions
 //--------------------------------------------------------------------------------------------------

diff --git a/wnfs-hamt/src/diff.rs b/wnfs-hamt/src/diff.rs
@@ -283,7 +283,7 @@ where
             }
 
             for Pair { key, value } in &other_values {
-                if main_map.get(key).is_none() {
+                if !main_map.contains_key(key) {
                     changes.push(KeyValueChange {
                         r#type: ChangeType::Remove,
                         key: key.clone(),

diff --git a/wnfs-hamt/src/merge.rs b/wnfs-hamt/src/merge.rs
@@ -13,16 +13,15 @@ use wnfs_common::{
 //--------------------------------------------------------------------------------------------------
 
 /// Merges a node with another with the help of a resolver function.
-pub async fn merge<K: CondSync, V: CondSync, H, F, B: BlockStore>(
+pub async fn merge<K, V, H>(
     main_link: Link<Arc<Node<K, V, H>>>,
     other_link: Link<Arc<Node<K, V, H>>>,
-    f: F,
-    store: &B,
+    f: impl Fn(&V, &V) -> Result<V>,
+    store: &impl BlockStore,
 ) -> Result<Arc<Node<K, V, H>>>
 where
-    F: Fn(&V, &V) -> Result<V>,
-    K: Storable + Eq + Clone + Hash + AsRef<[u8]>,
-    V: Storable + Eq + Clone,
+    K: Storable + Eq + Clone + CondSync + Hash + AsRef<[u8]>,
+    V: Storable + Eq + Clone + CondSync,
     K::Serializable: Serialize + DeserializeOwned,
     V::Serializable: Serialize + DeserializeOwned,
     H: Hasher + CondSync,

diff --git a/wnfs-hamt/src/strategies/kv.rs b/wnfs-hamt/src/strategies/kv.rs
@@ -12,13 +12,14 @@ use wnfs_common::{
 // Functions
 //--------------------------------------------------------------------------------------------------
 
-pub fn generate_kvs<K: Debug + Clone, V: Debug + Clone>(
+pub fn generate_kvs<K, V>(
     key: impl Strategy<Value = K>,
     value: impl Strategy<Value = V>,
     size: impl Into<SizeRange>,
 ) -> impl Strategy<Value = Vec<(K, V)>>
 where
-    K: Eq + Hash,
+    K: Debug + Clone + Eq + Hash,
+    V: Debug + Clone,
 {
     vec((key, value), size).prop_map(|vec| {
         vec.into_iter()

diff --git a/wnfs-hamt/src/strategies/operations.rs b/wnfs-hamt/src/strategies/operations.rs
@@ -183,13 +183,13 @@ where
 ///     println!("{:?}", node);
 /// }
 /// ```
-pub async fn node_from_operations<K: CondSync, V: CondSync>(
+pub async fn node_from_operations<K, V>(
     operations: &Operations<K, V>,
     store: &impl BlockStore,
 ) -> Result<Arc<Node<K, V>>>
 where
-    K: Storable + Clone + Debug + AsRef<[u8]>,
-    V: Storable + Clone + Debug,
+    K: Storable + Clone + Debug + CondSync + AsRef<[u8]>,
+    V: Storable + Clone + Debug + CondSync,
     K::Serializable: Serialize + DeserializeOwned,
     V::Serializable: Serialize + DeserializeOwned,
 {

diff --git a/wnfs-wasm/src/fs/private/file.rs b/wnfs-wasm/src/fs/private/file.rs
@@ -89,6 +89,23 @@ impl PrivateFile {
         self.read_at(value!(0).into(), None, forest, store)
     }
 
+    /// Gets the exact content size without fetching all content blocks.
+    #[wasm_bindgen(js_name = "getSize")]
+    pub fn get_size(&self, forest: &PrivateForest, store: BlockStore) -> JsResult<Promise> {
+        let file = Rc::clone(&self.0);
+        let store = ForeignBlockStore(store);
+        let forest = Rc::clone(&forest.0);
+
+        Ok(future_to_promise(async move {
+            let size = file
+                .size(&forest, &store)
+                .await
+                .map_err(error("Cannot determine file size"))?;
+
+            Ok(value!(size as usize))
+        }))
+    }
+
     /// Gets the metadata of this file.
     pub fn metadata(&self) -> JsResult<JsValue> {
         JsMetadata(self.0.get_metadata()).try_into()

diff --git a/wnfs-wasm/src/fs/public/file.rs b/wnfs-wasm/src/fs/public/file.rs
@@ -95,6 +95,22 @@ impl PublicFile {
         self.read_at(value!(0).into(), None, store)
     }
 
+    /// Gets the exact content size without fetching all content blocks.
+    #[wasm_bindgen(js_name = "getSize")]
+    pub fn get_size(&self, store: BlockStore) -> JsResult<Promise> {
+        let file = Rc::clone(&self.0);
+        let store = ForeignBlockStore(store);
+
+        Ok(future_to_promise(async move {
+            let size = file
+                .size(&store)
+                .await
+                .map_err(error("Cannot determine file size"))?;
+
+            Ok(value!(size as usize))
+        }))
+    }
+
     /// Gets the metadata of this file.
     pub fn metadata(&self) -> JsResult<JsValue> {
         JsMetadata(self.0.get_metadata()).try_into()

diff --git a/wnfs-wasm/tests/private.spec.ts b/wnfs-wasm/tests/private.spec.ts
@@ -453,6 +453,31 @@ test.describe("PrivateFile", () => {
     expect(new Uint8Array(Object.values(content))).toEqual(new Uint8Array([2, 3, 4]));
   });
 
+  test("getSize returns the exact content size", async ({ page }) => {
+    const size = await page.evaluate(async () => {
+      const {
+        wnfs: { PrivateFile, PrivateForest },
+        mock: { MemoryBlockStore, Rng },
+      } = await window.setup();
+
+      const rng = new Rng();
+      const initialForest = new PrivateForest(rng);
+      const store = new MemoryBlockStore();
+      var [file, forest] = await PrivateFile.withContent(
+        initialForest.emptyName(),
+        new Date(),
+        new Uint8Array(2 * 1024 * 1024),
+        initialForest,
+        store,
+        rng,
+      );
+
+      return await file.getSize(forest, store);
+    });
+
+    expect(size).toEqual(2 * 1024 * 1024);
+  });
+
   test("A PrivateDirectory has the correct metadata", async ({ page }) => {
     const result = await page.evaluate(async () => {
       const {

diff --git a/wnfs-wasm/tests/public.spec.ts b/wnfs-wasm/tests/public.spec.ts
@@ -29,9 +29,7 @@ test.describe("PublicDirectory", () => {
     expect(result).toBeDefined();
   });
 
-  test("lookupNode cannot fetch file not added to directory", async ({
-    page,
-  }) => {
+  test("lookupNode cannot fetch file not added to directory", async ({ page }) => {
     const result = await page.evaluate(async () => {
       const {
         wnfs: { PublicDirectory },
@@ -65,18 +63,12 @@ test.describe("PublicDirectory", () => {
         ["pictures", "cats", "tabby.png"],
         sampleCID,
         time,
-        store
+        store,
       );
 
-      let result0 = await rootDir.getNode(
-        ["pictures", "cats", "tabby.png"],
-        store
-      );
+      let result0 = await rootDir.getNode(["pictures", "cats", "tabby.png"], store);
 
-      let result1 = await rootDir.getNode(
-        ["pictures", "dogs", "bingo.png"],
-        store
-      );
+      let result1 = await rootDir.getNode(["pictures", "dogs", "bingo.png"], store);
 
       return [result0, result1];
     });
@@ -102,7 +94,7 @@ test.describe("PublicDirectory", () => {
         ["pictures", "cats", "tabby.png"],
         sampleCID,
         time,
-        store
+        store,
       );
 
       await rootDir.getNode(["pictures", "cats", "tabby.png"], store);
@@ -130,7 +122,7 @@ test.describe("PublicDirectory", () => {
         ["pictures", "cats", "tabby.png"],
         sampleCID,
         time,
-        store
+        store,
       );
 
       const result = await rootDir.ls(["pictures"], store);
@@ -158,14 +150,14 @@ test.describe("PublicDirectory", () => {
         ["pictures", "dogs", "billie.jpeg"],
         sampleCID,
         time,
-        store
+        store,
       );
 
       var { rootDir } = await rootDir.write(
         ["pictures", "cats", "tabby.png"],
         sampleCID,
         time,
-        store
+        store,
       );
 
       var { rootDir } = await rootDir.rm(["pictures", "cats"], store);
@@ -190,18 +182,13 @@ test.describe("PublicDirectory", () => {
       const store = new MemoryBlockStore();
       const root = new PublicDirectory(time);
 
-      var { rootDir } = await root.write(
-        ["pictures", "cats", "luna.jpeg"],
-        sampleCID,
-        time,
-        store
-      );
+      var { rootDir } = await root.write(["pictures", "cats", "luna.jpeg"], sampleCID, time, store);
 
       var { rootDir } = await rootDir.write(
         ["pictures", "cats", "tabby.png"],
         sampleCID,
         time,
-        store
+        store,
       );
 
       var { rootDir } = await rootDir.mkdir(["images"], time, store);
@@ -210,7 +197,7 @@ test.describe("PublicDirectory", () => {
         ["pictures", "cats"],
         ["images", "cats"],
         time,
-        store
+        store,
       );
 
       const imagesContent = await rootDir.ls(["images"], store);
@@ -236,28 +223,18 @@ test.describe("PublicDirectory", () => {
       const store = new MemoryBlockStore();
       const root = new PublicDirectory(time);
 
-      var { rootDir } = await root.write(
-        ["pictures", "cats", "luna.jpeg"],
-        sampleCID,
-        time,
-        store
-      );
+      var { rootDir } = await root.write(["pictures", "cats", "luna.jpeg"], sampleCID, time, store);
 
       var { rootDir } = await rootDir.write(
         ["pictures", "cats", "tabby.png"],
         sampleCID,
         time,
-        store
+        store,
       );
 
       var { rootDir } = await rootDir.mkdir(["images"], time, store);
 
-      var { rootDir } = await rootDir.cp(
-        ["pictures", "cats"],
-        ["images", "cats"],
-        time,
-        store
-      );
+      var { rootDir } = await rootDir.cp(["pictures", "cats"], ["images", "cats"], time, store);
 
       const imagesContent = await rootDir.ls(["images"], store);
 
@@ -314,10 +291,7 @@ test.describe("PublicDirectory", () => {
       const readBack = await file2.getContent(store);
       const partialRead = await file2.readAt(7, 5, store);
 
-      return [
-        new TextDecoder().decode(readBack),
-        new TextDecoder().decode(partialRead)
-      ];
+      return [new TextDecoder().decode(readBack), new TextDecoder().decode(partialRead)];
     });
 
     expect(result[0]).toEqual("Hello, World!");
@@ -370,4 +344,25 @@ test.describe("PublicDirectory", () => {
     expect(result).not.toBeUndefined();
     expect(result).toEqual("bafkr4ihkr4ld3m4gqkjf4reryxsy2s5tkbxprqkow6fin2iiyvreuzzab4");
   });
+
+  test("A PublicFile has a content size", async ({ page }) => {
+    const result = await page.evaluate(async () => {
+      const {
+        wnfs: { PublicFile },
+        mock: { MemoryBlockStore },
+      } = await window.setup();
+
+      const store = new MemoryBlockStore();
+      const time = new Date();
+      const file = new PublicFile(time);
+
+      const longString = "x".repeat(5 * 1024 * 1024);
+      const content = new TextEncoder().encode(longString);
+      const file2 = await file.setContent(time, content, store);
+
+      return await file2.getSize(store);
+    });
+
+    expect(result).toEqual(5 * 1024 * 1024);
+  });
 });