Start sharding

grimbough · Aug 14, 2023 · 5ef1b5d · 5ef1b5d
1 parent a2a9fdf
commit 5ef1b5d
Show file tree

Hide file tree

Showing 3 changed files with 28 additions and 0 deletions.
diff --git a/R/read_data_v3.R b/R/read_data_v3.R
@@ -151,6 +151,8 @@ read_chunk_v3 <- function(zarr_array_path, chunk_id, metadata, s3_client = NULL,
       chunk <- blosc_codec(chunk, cname = config$cname, 
                            clevel = config$clevel, shuffle = config$shuffle,
                            encoding = FALSE)
+    } else if (codec_name == "sharding_indexed") {
+
     } else {
       stop("Unsupported codec: ", codec_name)
     }

diff --git a/R/sharding.R b/R/sharding.R
diff --git a/inst/scripts/create_test_data_v3.py b/inst/scripts/create_test_data_v3.py
@@ -159,3 +159,29 @@
         ],
     )
 a[:] = data
+
+###################
+
+shape = (10,)
+dtype = 'int8'
+
+data = np.full(shape, 0, dtype=dtype)
+data[:] = np.arange(start=1, stop=11)
+path = "sharded/1d-sub-chunks.zarr"
+shutil.rmtree(os.path.join(data_store, path), ignore_errors=True)
+a = zarrita.Array.create(
+        store / path,
+        shape=shape,
+        dtype=dtype,
+        chunk_shape=(10,),
+        codecs=[
+            zarrita.codecs.sharding_codec(
+                chunk_shape=(5,),
+                codecs=[
+                    zarrita.codecs.endian_codec(),
+                    zarrita.codecs.gzip_codec()
+                ],
+            )
+        ],
+    )
+a[:] = data