Validate vlen codec length32 encoding against zarr-python v2

LDeakin · Jul 18, 2024 · ac7ebc7 · ac7ebc7
1 parent db563a2
commit ac7ebc7
Show file tree

Hide file tree

Showing 100 changed files with 88 additions and 4 deletions.
diff --git a/tests/cities.rs b/tests/cities.rs
@@ -14,7 +14,10 @@ use zarrs::{
     },
     array_subset::ArraySubset,
     metadata::v3::codec::vlen::Encoding,
-    storage::{store::MemoryStore, ListableStorageTraits},
+    storage::{
+        store::{FilesystemStore, MemoryStore},
+        ReadableWritableListableStorage,
+    },
 };
 
 fn read_cities() -> std::io::Result<Vec<String>> {
@@ -33,7 +36,12 @@ fn cities_impl(
     chunk_size: u64,
     shard_size: Option<u64>,
 ) -> Result<u64, Box<dyn Error>> {
-    let store = std::sync::Arc::new(MemoryStore::default());
+    let store: ReadableWritableListableStorage =
+        if encoding == Encoding::Length32 && compression_level.is_none() {
+            std::sync::Arc::new(FilesystemStore::new("tests/data/v3/cities.zarr")?)
+        } else {
+            std::sync::Arc::new(MemoryStore::default())
+        };
 
     let mut builder = ArrayBuilder::new(
         vec![cities.len() as u64], // array shape
@@ -60,7 +68,8 @@ fn cities_impl(
         ]);
     }
 
-    let array = builder.build(store.clone(), "/array")?;
+    let array = builder.build(store.clone(), "/")?;
+    array.store_metadata()?;
 
     let subset_all = ArraySubset::new_with_shape(array.shape().to_vec());
     array.store_array_subset_elements(&subset_all, &cities)?;
@@ -71,7 +80,7 @@ fn cities_impl(
     let (_bytes, offsets) = last_block.into_variable()?;
     assert_eq!(offsets.len() as u64, chunk_size + 1);
 
-    Ok(store.size()?)
+    Ok(store.size_prefix(&"c/".try_into().unwrap())?) // only chunks
 }
 
 #[rustfmt::skip]

diff --git a/tests/data/v2/cities.zarr/.zarray b/tests/data/v2/cities.zarr/.zarray
@@ -0,0 +1,18 @@
+{
+    "chunks": [
+        1000
+    ],
+    "compressor": null,
+    "dtype": "|O",
+    "fill_value": "",
+    "filters": [
+        {
+            "id": "vlen-utf8"
+        }
+    ],
+    "order": "C",
+    "shape": [
+        47868
+    ],
+    "zarr_format": 2
+}
diff --git a/tests/data/v2/cities.zarr/0 b/tests/data/v2/cities.zarr/0
diff --git a/tests/data/v2/cities.zarr/1 b/tests/data/v2/cities.zarr/1
diff --git a/tests/data/v2/cities.zarr/10 b/tests/data/v2/cities.zarr/10
diff --git a/tests/data/v2/cities.zarr/11 b/tests/data/v2/cities.zarr/11
diff --git a/tests/data/v2/cities.zarr/12 b/tests/data/v2/cities.zarr/12
diff --git a/tests/data/v2/cities.zarr/13 b/tests/data/v2/cities.zarr/13
diff --git a/tests/data/v2/cities.zarr/14 b/tests/data/v2/cities.zarr/14
diff --git a/tests/data/v2/cities.zarr/15 b/tests/data/v2/cities.zarr/15
diff --git a/tests/data/v2/cities.zarr/16 b/tests/data/v2/cities.zarr/16
diff --git a/tests/data/v2/cities.zarr/17 b/tests/data/v2/cities.zarr/17
diff --git a/tests/data/v2/cities.zarr/18 b/tests/data/v2/cities.zarr/18
diff --git a/tests/data/v2/cities.zarr/19 b/tests/data/v2/cities.zarr/19
diff --git a/tests/data/v2/cities.zarr/2 b/tests/data/v2/cities.zarr/2
diff --git a/tests/data/v2/cities.zarr/20 b/tests/data/v2/cities.zarr/20
diff --git a/tests/data/v2/cities.zarr/21 b/tests/data/v2/cities.zarr/21
diff --git a/tests/data/v2/cities.zarr/22 b/tests/data/v2/cities.zarr/22
diff --git a/tests/data/v2/cities.zarr/23 b/tests/data/v2/cities.zarr/23
diff --git a/tests/data/v2/cities.zarr/24 b/tests/data/v2/cities.zarr/24
diff --git a/tests/data/v2/cities.zarr/25 b/tests/data/v2/cities.zarr/25
diff --git a/tests/data/v2/cities.zarr/26 b/tests/data/v2/cities.zarr/26
diff --git a/tests/data/v2/cities.zarr/27 b/tests/data/v2/cities.zarr/27
diff --git a/tests/data/v2/cities.zarr/28 b/tests/data/v2/cities.zarr/28
diff --git a/tests/data/v2/cities.zarr/29 b/tests/data/v2/cities.zarr/29
diff --git a/tests/data/v2/cities.zarr/3 b/tests/data/v2/cities.zarr/3
diff --git a/tests/data/v2/cities.zarr/30 b/tests/data/v2/cities.zarr/30
diff --git a/tests/data/v2/cities.zarr/31 b/tests/data/v2/cities.zarr/31
diff --git a/tests/data/v2/cities.zarr/32 b/tests/data/v2/cities.zarr/32
diff --git a/tests/data/v2/cities.zarr/33 b/tests/data/v2/cities.zarr/33
diff --git a/tests/data/v2/cities.zarr/34 b/tests/data/v2/cities.zarr/34
diff --git a/tests/data/v2/cities.zarr/35 b/tests/data/v2/cities.zarr/35
diff --git a/tests/data/v2/cities.zarr/36 b/tests/data/v2/cities.zarr/36
diff --git a/tests/data/v2/cities.zarr/37 b/tests/data/v2/cities.zarr/37
diff --git a/tests/data/v2/cities.zarr/38 b/tests/data/v2/cities.zarr/38
diff --git a/tests/data/v2/cities.zarr/39 b/tests/data/v2/cities.zarr/39
diff --git a/tests/data/v2/cities.zarr/4 b/tests/data/v2/cities.zarr/4
diff --git a/tests/data/v2/cities.zarr/40 b/tests/data/v2/cities.zarr/40
diff --git a/tests/data/v2/cities.zarr/41 b/tests/data/v2/cities.zarr/41
diff --git a/tests/data/v2/cities.zarr/42 b/tests/data/v2/cities.zarr/42
diff --git a/tests/data/v2/cities.zarr/43 b/tests/data/v2/cities.zarr/43
diff --git a/tests/data/v2/cities.zarr/44 b/tests/data/v2/cities.zarr/44
diff --git a/tests/data/v2/cities.zarr/45 b/tests/data/v2/cities.zarr/45
diff --git a/tests/data/v2/cities.zarr/46 b/tests/data/v2/cities.zarr/46
diff --git a/tests/data/v2/cities.zarr/47 b/tests/data/v2/cities.zarr/47
diff --git a/tests/data/v2/cities.zarr/5 b/tests/data/v2/cities.zarr/5
diff --git a/tests/data/v2/cities.zarr/6 b/tests/data/v2/cities.zarr/6
diff --git a/tests/data/v2/cities.zarr/7 b/tests/data/v2/cities.zarr/7
diff --git a/tests/data/v2/cities.zarr/8 b/tests/data/v2/cities.zarr/8
diff --git a/tests/data/v2/cities.zarr/9 b/tests/data/v2/cities.zarr/9
diff --git a/tests/data/v2_cities.py b/tests/data/v2_cities.py
@@ -0,0 +1,19 @@
+import zarr # v2
+import pandas as pd
+
+print(zarr.__version__)
+
+df = pd.read_csv("tests/data/cities.csv", header=None)
+cities = df[0]
+
+path_out = 'tests/data/v2/cities.zarr'
+array = zarr.open(path_out, mode='w', dtype=str, shape=(len(cities),), chunks=(1000,), compressor = None, fill_value='')
+array[:] = cities.values
+print(array.info)
+
+for i in range(48):
+    v2 = open(f'tests/data/v2/cities.zarr/{i}', 'rb').read()
+    v3 = open(f'tests/data/v3/cities.zarr/c/{i}', 'rb').read()
+    assert v2 == v3
+
+print("V2 and V3 chunks are identical!")
diff --git a/tests/data/v3/cities.zarr/c/0 b/tests/data/v3/cities.zarr/c/0
diff --git a/tests/data/v3/cities.zarr/c/1 b/tests/data/v3/cities.zarr/c/1
diff --git a/tests/data/v3/cities.zarr/c/10 b/tests/data/v3/cities.zarr/c/10
diff --git a/tests/data/v3/cities.zarr/c/11 b/tests/data/v3/cities.zarr/c/11
diff --git a/tests/data/v3/cities.zarr/c/12 b/tests/data/v3/cities.zarr/c/12
diff --git a/tests/data/v3/cities.zarr/c/13 b/tests/data/v3/cities.zarr/c/13
diff --git a/tests/data/v3/cities.zarr/c/14 b/tests/data/v3/cities.zarr/c/14
diff --git a/tests/data/v3/cities.zarr/c/15 b/tests/data/v3/cities.zarr/c/15
diff --git a/tests/data/v3/cities.zarr/c/16 b/tests/data/v3/cities.zarr/c/16
diff --git a/tests/data/v3/cities.zarr/c/17 b/tests/data/v3/cities.zarr/c/17
diff --git a/tests/data/v3/cities.zarr/c/18 b/tests/data/v3/cities.zarr/c/18
diff --git a/tests/data/v3/cities.zarr/c/19 b/tests/data/v3/cities.zarr/c/19
diff --git a/tests/data/v3/cities.zarr/c/2 b/tests/data/v3/cities.zarr/c/2
diff --git a/tests/data/v3/cities.zarr/c/20 b/tests/data/v3/cities.zarr/c/20
diff --git a/tests/data/v3/cities.zarr/c/21 b/tests/data/v3/cities.zarr/c/21
diff --git a/tests/data/v3/cities.zarr/c/22 b/tests/data/v3/cities.zarr/c/22
diff --git a/tests/data/v3/cities.zarr/c/23 b/tests/data/v3/cities.zarr/c/23
diff --git a/tests/data/v3/cities.zarr/c/24 b/tests/data/v3/cities.zarr/c/24
diff --git a/tests/data/v3/cities.zarr/c/25 b/tests/data/v3/cities.zarr/c/25
diff --git a/tests/data/v3/cities.zarr/c/26 b/tests/data/v3/cities.zarr/c/26
diff --git a/tests/data/v3/cities.zarr/c/27 b/tests/data/v3/cities.zarr/c/27
diff --git a/tests/data/v3/cities.zarr/c/28 b/tests/data/v3/cities.zarr/c/28
diff --git a/tests/data/v3/cities.zarr/c/29 b/tests/data/v3/cities.zarr/c/29
diff --git a/tests/data/v3/cities.zarr/c/3 b/tests/data/v3/cities.zarr/c/3
diff --git a/tests/data/v3/cities.zarr/c/30 b/tests/data/v3/cities.zarr/c/30
diff --git a/tests/data/v3/cities.zarr/c/31 b/tests/data/v3/cities.zarr/c/31
diff --git a/tests/data/v3/cities.zarr/c/32 b/tests/data/v3/cities.zarr/c/32
diff --git a/tests/data/v3/cities.zarr/c/33 b/tests/data/v3/cities.zarr/c/33
diff --git a/tests/data/v3/cities.zarr/c/34 b/tests/data/v3/cities.zarr/c/34
diff --git a/tests/data/v3/cities.zarr/c/35 b/tests/data/v3/cities.zarr/c/35
diff --git a/tests/data/v3/cities.zarr/c/36 b/tests/data/v3/cities.zarr/c/36
diff --git a/tests/data/v3/cities.zarr/c/37 b/tests/data/v3/cities.zarr/c/37
diff --git a/tests/data/v3/cities.zarr/c/38 b/tests/data/v3/cities.zarr/c/38
diff --git a/tests/data/v3/cities.zarr/c/39 b/tests/data/v3/cities.zarr/c/39
diff --git a/tests/data/v3/cities.zarr/c/4 b/tests/data/v3/cities.zarr/c/4
diff --git a/tests/data/v3/cities.zarr/c/40 b/tests/data/v3/cities.zarr/c/40
diff --git a/tests/data/v3/cities.zarr/c/41 b/tests/data/v3/cities.zarr/c/41
diff --git a/tests/data/v3/cities.zarr/c/42 b/tests/data/v3/cities.zarr/c/42
diff --git a/tests/data/v3/cities.zarr/c/43 b/tests/data/v3/cities.zarr/c/43
diff --git a/tests/data/v3/cities.zarr/c/44 b/tests/data/v3/cities.zarr/c/44
diff --git a/tests/data/v3/cities.zarr/c/45 b/tests/data/v3/cities.zarr/c/45
diff --git a/tests/data/v3/cities.zarr/c/46 b/tests/data/v3/cities.zarr/c/46
diff --git a/tests/data/v3/cities.zarr/c/47 b/tests/data/v3/cities.zarr/c/47
diff --git a/tests/data/v3/cities.zarr/c/5 b/tests/data/v3/cities.zarr/c/5
diff --git a/tests/data/v3/cities.zarr/c/6 b/tests/data/v3/cities.zarr/c/6
diff --git a/tests/data/v3/cities.zarr/c/7 b/tests/data/v3/cities.zarr/c/7
diff --git a/tests/data/v3/cities.zarr/c/8 b/tests/data/v3/cities.zarr/c/8
diff --git a/tests/data/v3/cities.zarr/c/9 b/tests/data/v3/cities.zarr/c/9
diff --git a/tests/data/v3/cities.zarr/zarr.json b/tests/data/v3/cities.zarr/zarr.json
@@ -0,0 +1,38 @@
+{
+  "zarr_format": 3,
+  "node_type": "array",
+  "shape": [
+    47868
+  ],
+  "data_type": "string",
+  "chunk_grid": {
+    "name": "regular",
+    "configuration": {
+      "chunk_shape": [
+        1000
+      ]
+    }
+  },
+  "chunk_key_encoding": {
+    "name": "default",
+    "configuration": {
+      "separator": "/"
+    }
+  },
+  "fill_value": null,
+  "codecs": [
+    {
+      "name": "vlen",
+      "configuration": {
+        "encoding": "length32"
+      }
+    }
+  ],
+  "attributes": {
+    "_zarrs": {
+      "description": "This array was created with zarrs",
+      "repository": "https://github.com/LDeakin/zarrs",
+      "version": "0.16.0"
+    }
+  }
+}