feat(api): self lookup & store at construction (#138)

* Add privateref serialization example

* Empty path segment means self lookup

* Add constructor with store

* Update instructions

* Minor fix

* Fix typo

* Add more re-exports and tidy debug output

* Fix compilation errors

* Expose search_latest and change constructor names

* Minor rename

wnfs/Cargo.toml

@@ -21,7 +21,6 @@ aes-gcm = "0.10"
 anyhow = "1.0"
 async-once-cell = "0.4"
 async-recursion = "1.0"
-async-std = { version = "1.11", features = ["attributes"] }
 async-stream = "0.3"
 async-trait = "0.1"
 bitvec = { version = "1.0", features = ["serde"] }
@@ -44,11 +43,13 @@ thiserror = "1.0"
 xxhash-rust = { version = "0.8", features = ["xxh3"] }
 
 [dev-dependencies]
+async-std = { version = "1.11", features = ["attributes"] }
 env_logger = "0.10"
 proptest = "1.0"
 rand = "0.8"
 test-log = "0.2"
 test-strategy = "0.2"
+tokio = { version = "1.0", features = ["full"] }
 
 [lib]
 name = "wnfs"

wnfs/examples/private.rs

@@ -6,7 +6,6 @@ use libipld::Cid;
 use rand::{thread_rng, RngCore};
 use std::rc::Rc;
 use wnfs::{
-    dagcbor,
     private::{PrivateForest, PrivateRef},
     BlockStore, MemoryBlockStore, Namefilter, PrivateDirectory, PrivateOpResult,
 };
@@ -23,14 +22,11 @@ async fn main() {
     let (forest_cid, private_ref) = get_forest_cid_and_private_ref(store, rng).await;
 
     // Fetch CBOR bytes of private forest from the blockstore.
-    let cbor_bytes = store
-        .get_deserializable::<Vec<u8>>(&forest_cid)
+    let forest = store
+        .get_deserializable::<PrivateForest>(&forest_cid)
         .await
         .unwrap();
 
-    // Decode private forest CBOR bytes.
-    let forest = dagcbor::decode::<PrivateForest>(cbor_bytes.as_ref()).unwrap();
-
     // Fetch and decrypt a directory from the private forest using provided private ref.
     let dir = forest
         .get(&private_ref, PrivateForest::resolve_lowest, store)
@@ -70,11 +66,8 @@ async fn get_forest_cid_and_private_ref(
         .await
         .unwrap();
 
-    // Serialize the private forest to DAG CBOR.
-    let cbor_bytes = dagcbor::async_encode(&forest, store).await.unwrap();
-
     // Persist encoded private forest to the block store.
-    let forest_cid = store.put_serializable(&cbor_bytes).await.unwrap();
+    let forest_cid = store.put_async_serializable(&forest).await.unwrap();
 
     // Private ref contains data and keys for fetching and decrypting the directory node in the private forest.
     let private_ref = root_dir.header.get_private_ref();

wnfs/examples/privateref.rs

@@ -0,0 +1,119 @@
+use chrono::Utc;
+use rand::thread_rng;
+use sha3::Sha3_256;
+use std::{io::Cursor, rc::Rc};
+use wnfs::{
+    ipld::{DagCborCodec, Decode, Encode, Ipld, Serializer},
+    private::{Key, PrivateForest, PrivateRef, RevisionKey},
+    ratchet::Ratchet,
+    rng::RngCore,
+    utils, Hasher, MemoryBlockStore, Namefilter, PrivateDirectory, PrivateOpResult,
+};
+
+#[tokio::main(flavor = "multi_thread")]
+async fn main() -> anyhow::Result<()> {
+    // ----------- Prerequisites -----------
+
+    let store = &mut MemoryBlockStore::default();
+    let rng = &mut thread_rng();
+    let forest = Rc::new(PrivateForest::new());
+
+    // ----------- Create a private directory -----------
+
+    // Some existing user key.
+    let some_key = Key::new(utils::get_random_bytes::<32>(rng));
+
+    // Creating ratchet_seed from our user key. And intializing the inumber and namefilter.
+    let ratchet_seed = Sha3_256::hash(&some_key.as_bytes());
+    let inumber = utils::get_random_bytes::<32>(rng); // Needs to be random
+
+    // Create a root directory from the ratchet_seed, inumber and namefilter. Directory gets saved in forest.
+    let PrivateOpResult {
+        forest, root_dir, ..
+    } = PrivateDirectory::new_with_seed_and_store(
+        Namefilter::default(),
+        Utc::now(),
+        ratchet_seed,
+        inumber,
+        forest,
+        store,
+        rng,
+    )
+    .await
+    .unwrap();
+
+    // ----------- Create a subdirectory -----------
+
+    // Add a /movies/anime to the directory.
+    let PrivateOpResult {
+        forest, root_dir, ..
+    } = root_dir
+        .mkdir(
+            &["movies".into(), "anime".into()],
+            true,
+            Utc::now(),
+            forest,
+            store,
+            rng,
+        )
+        .await?;
+
+    // --------- Generate a private ref (Method 1) -----------
+
+    // We can create a revision_key from our ratchet_seed.
+    let ratchet = Ratchet::zero(ratchet_seed);
+    let revision_key = RevisionKey::from(Key::new(ratchet.derive_key()));
+
+    // Now let's serialize the root_dir's private_ref.
+    let cbor = encode(&root_dir.header.get_private_ref(), &revision_key, rng)?;
+
+    // We can deserialize the private_ref using the revision_key at hand.
+    let private_ref = decode(cbor, &revision_key)?;
+
+    // Now we can fetch the directory from the forest using the private_ref.
+    let fetched_node = forest
+        .get(&private_ref, PrivateForest::resolve_lowest, store)
+        .await?;
+
+    println!("{:#?}", fetched_node);
+
+    // --------- Generate a private ref (Method 2) -----------
+
+    // We can also create a private_ref from scratch if we remember the parameters.
+    let private_ref = PrivateRef::with_seed(Namefilter::default(), ratchet_seed, inumber);
+
+    // And we can fetch the directory again using the generated private_ref.
+    let fetched_node = forest
+        .get(&private_ref, PrivateForest::resolve_lowest, store)
+        .await?;
+
+    println!("{:#?}", fetched_node);
+
+    // The private_ref might point to some old revision of the root_dir.
+    // We can do the following to get the latest revision.
+    let fetched_dir = fetched_node
+        .unwrap()
+        .search_latest(&forest, store)
+        .await?
+        .as_dir()?;
+
+    println!("{:#?}", fetched_dir);
+
+    Ok(())
+}
+
+fn encode(
+    private_ref: &PrivateRef,
+    revision_key: &RevisionKey,
+    rng: &mut impl RngCore,
+) -> anyhow::Result<Vec<u8>> {
+    let mut bytes = Vec::new();
+    let ipld = private_ref.serialize(Serializer, revision_key, rng)?;
+    ipld.encode(DagCborCodec, &mut bytes)?;
+    Ok(bytes)
+}
+
+fn decode(bytes: Vec<u8>, revision_key: &RevisionKey) -> anyhow::Result<PrivateRef> {
+    let ipld = Ipld::decode(DagCborCodec, &mut Cursor::new(bytes))?;
+    PrivateRef::deserialize(ipld, revision_key).map_err(Into::into)
+}

wnfs/src/lib.rs

@@ -30,10 +30,15 @@ pub mod ipld {
         cbor::DagCborCodec,
         cid::Version,
         codec::{Codec, Decode, Encode},
-        Cid, IpldCodec,
+        serde::Serializer,
+        Cid, Ipld, IpldCodec,
     };
 }
 
 pub mod rng {
     pub use rand_core::RngCore;
 }
+
+pub mod ratchet {
+    pub use skip_ratchet::{Ratchet, RatchetSeeker};
+}

wnfs/src/private/directory.rs

@@ -15,6 +15,7 @@ use semver::Version;
 use serde::{de::Error as DeError, ser::Error as SerError, Deserialize, Deserializer, Serialize};
 use std::{
     collections::{BTreeMap, BTreeSet},
+    fmt::Debug,
     rc::Rc,
 };
 
@@ -143,6 +144,74 @@ impl PrivateDirectory {
         }
     }
 
+    pub async fn new_and_store<B: BlockStore, R: RngCore>(
+        parent_bare_name: Namefilter,
+        time: DateTime<Utc>,
+        forest: Rc<PrivateForest>,
+        store: &mut B,
+        rng: &mut R,
+    ) -> Result<PrivateOpResult<()>> {
+        let dir = Rc::new(Self {
+            persisted_as: OnceCell::new(),
+            version: Version::new(0, 2, 0),
+            header: PrivateNodeHeader::new(parent_bare_name, rng),
+            metadata: Metadata::new(time),
+            previous: None,
+            entries: BTreeMap::new(),
+        });
+
+        let forest = forest
+            .put(
+                dir.header.get_saturated_name(),
+                &dir.header.get_private_ref(),
+                &PrivateNode::Dir(Rc::clone(&dir)),
+                store,
+                rng,
+            )
+            .await?;
+
+        Ok(PrivateOpResult {
+            root_dir: dir,
+            forest,
+            result: (),
+        })
+    }
+
+    pub async fn new_with_seed_and_store<B: BlockStore, R: RngCore>(
+        parent_bare_name: Namefilter,
+        time: DateTime<Utc>,
+        ratchet_seed: HashOutput,
+        inumber: HashOutput,
+        forest: Rc<PrivateForest>,
+        store: &mut B,
+        rng: &mut R,
+    ) -> Result<PrivateOpResult<()>> {
+        let dir = Rc::new(Self {
+            persisted_as: OnceCell::new(),
+            version: Version::new(0, 2, 0),
+            header: PrivateNodeHeader::with_seed(parent_bare_name, ratchet_seed, inumber),
+            previous: None,
+            metadata: Metadata::new(time),
+            entries: BTreeMap::new(),
+        });
+
+        let forest = forest
+            .put(
+                dir.header.get_saturated_name(),
+                &dir.header.get_private_ref(),
+                &PrivateNode::Dir(Rc::clone(&dir)),
+                store,
+                rng,
+            )
+            .await?;
+
+        Ok(PrivateOpResult {
+            root_dir: dir,
+            forest,
+            result: (),
+        })
+    }
+
     /// Gets the metadata of the directory
     ///
     /// # Examples
@@ -455,11 +524,14 @@ impl PrivateDirectory {
                     NotADirectory(_, _) => bail!(FsError::NotFound),
                 }
             }
-            None => PrivateOpResult {
-                root_dir,
-                forest,
-                result: Some(PrivateNode::Dir(self)),
-            },
+            None => {
+                let result = self.lookup_node("", search_latest, &forest, store).await?;
+                PrivateOpResult {
+                    root_dir,
+                    forest,
+                    result,
+                }
+            }
         })
     }
 
@@ -742,6 +814,60 @@ impl PrivateDirectory {
         })
     }
 
+    /// Gets the latest version of the directory using exponential search.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use std::rc::Rc;
+    /// use chrono::Utc;
+    /// use rand::thread_rng;
+    /// use wnfs::{
+    ///     private::{PrivateForest, PrivateRef, PrivateNode},
+    ///     BlockStore, MemoryBlockStore, Namefilter, PrivateDirectory, PrivateOpResult,
+    /// };
+    ///
+    /// #[async_std::main]
+    /// async fn main() {
+    ///     let store = &mut MemoryBlockStore::default();
+    ///     let rng = &mut thread_rng();
+    ///     let forest = Rc::new(PrivateForest::new());
+    ///
+    ///     let PrivateOpResult { forest, root_dir: init_dir, .. } = PrivateDirectory::new_and_store(
+    ///         Default::default(),
+    ///         Utc::now(),
+    ///         forest,
+    ///         store,
+    ///         rng
+    ///     ).await.unwrap();
+    ///
+    ///     let PrivateOpResult { forest, root_dir, .. } = Rc::clone(&init_dir)
+    ///         .mkdir(&["pictures".into(), "cats".into()], true, Utc::now(), forest, store, rng)
+    ///         .await
+    ///         .unwrap();
+    ///
+    ///     let latest_dir = init_dir.search_latest(&forest, store).await.unwrap();
+    ///
+    ///     let found_node = latest_dir
+    ///         .lookup_node("pictures", true, &forest, store)
+    ///         .await
+    ///         .unwrap();
+    ///
+    ///     assert!(found_node.is_some());
+    /// }
+    /// ```
+    #[inline]
+    pub async fn search_latest(
+        self: Rc<Self>,
+        forest: &PrivateForest,
+        store: &impl BlockStore,
+    ) -> Result<Rc<PrivateDirectory>> {
+        PrivateNode::Dir(self)
+            .search_latest(forest, store)
+            .await?
+            .as_dir()
+    }
+
     /// Creates a new directory at the specified path.
     ///
     /// # Examples

wnfs/src/private/namefilter/bloomfilter.rs

@@ -263,8 +263,13 @@ impl<'de, const N: usize, const K: usize> Deserialize<'de> for BloomFilter<N, K>
 impl<const N: usize, const K: usize> Debug for BloomFilter<N, K> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "0x")?;
-        for byte in self.as_bytes().iter() {
-            write!(f, "{byte:02X}")?;
+        for (i, byte) in self.as_bytes().iter().enumerate() {
+            if i > 32 {
+                write!(f, "..")?;
+                break;
+            } else {
+                write!(f, "{byte:02X}")?;
+            }
         }
 
         Ok(())