Read directly from remote paths, upload Dataset with reserve_manual_upload

docs/src/webknossos-py/examples/publish_to_webknossos.md

@@ -0,0 +1,9 @@
+# Publish dataset to WEBKNOSSOS
+
+This example demonstrates the process of uploading a Dataset. 
+
+```python
+--8<--
+webknossos/examples/publish_to_webknossos.py
+--8<--
+```

webknossos/examples/apply_merger_mode.py

@@ -1,11 +1,7 @@
-from pathlib import Path
-
 import fastremap
 
 import webknossos as wk
 
-TESTDATA_DIR = Path(__file__).parent.parent / "testdata"
-
 
 def main() -> None:
     ####################################
@@ -17,15 +13,14 @@ def main() -> None:
     ).skeleton
 
     ###############################################
-    # Download and open the corresponding dataset #
+    # Open the corresponding dataset #
     ###############################################
 
-    dataset = wk.Dataset.download(
+    input_dataset = wk.Dataset.open_remote(
         "l4_sample",
-        bbox=wk.BoundingBox((3457, 3323, 1204), (40, 10, 10)),
-        path="testoutput/l4_sample",
+        organization_id="scalable_minds",
     )
-    in_layer = dataset.get_segmentation_layer("segmentation")
+    in_layer = input_dataset.get_segmentation_layer("segmentation")
     in_mag1 = in_layer.get_mag("1")
 
     ##############################
@@ -51,8 +46,10 @@ def main() -> None:
     ############################
     # Creating an output layer #
     ############################
-
-    out_layer = dataset.add_layer(
+    output_dataset = wk.Dataset(
+        "testoutput/l4_sample", voxel_size=input_dataset.voxel_size
+    )
+    out_layer = output_dataset.add_layer(
         "segmentation_remapped",
         wk.SEGMENTATION_CATEGORY,
         dtype_per_channel=in_layer.dtype_per_channel,
@@ -83,16 +80,9 @@ def apply_mapping_for_chunk(args: tuple[wk.View, wk.View, int]) -> None:
     ########################################
 
     out_layer.downsample()
-    dataset.delete_layer("segmentation")
-    dataset.upload(
+    output_dataset.upload(
         "l4_sample_remapped",
-        layers_to_link=[
-            wk.LayerToLink(
-                dataset_name="l4_sample",
-                layer_name="color",
-                organization_id="scalable_minds",
-            )
-        ],
+        layers_to_link=[input_dataset.get_layer("color")],
     )
 
 

webknossos/examples/create_dataset_from_images.py

@@ -1,7 +1,6 @@
 from pathlib import Path
 
-from webknossos import Dataset
-from webknossos.dataset import COLOR_CATEGORY
+from webknossos import COLOR_CATEGORY, Dataset
 
 INPUT_DIR = Path(__file__).parent.parent / "testdata" / "tiff"
 OUTPUT_DIR = Path("testoutput/my_tiff_dataset")

webknossos/examples/image_stack_to_dataset.py

@@ -1,7 +1,6 @@
 from pathlib import Path
 
-from webknossos import Dataset
-from webknossos.dataset import COLOR_CATEGORY
+from webknossos import COLOR_CATEGORY, Dataset
 
 INPUT_FILES = (
     Path(__file__).parent.parent / "testdata" / "tiff" / "test.0000.tiff",

webknossos/examples/publish_to_webknossos.py

@@ -0,0 +1,32 @@
+import webknossos as wk
+from webknossos import LayerToLink
+from webknossos.dataset.remote_folder import RemoteFolder
+
+
+def main() -> None:
+    # Get the folder id of the upload destination:
+    folder_id = RemoteFolder.get_by_path("Datasets/").id
+
+    # Open your unpublished dataset
+    dataset = wk.Dataset.open("my_dataset.wkw")
+
+    existing_dataset = wk.Dataset.open_remote(
+        "https://webknossos.org/datasets/liconn_expid82-678f98e3010000dd017f4449/#3825,4866,795,0,1.3"
+    )
+    # Publish the dataset to webknossos
+    # The data will be uploaded to the upload destination folder
+    dataset.upload(
+        new_dataset_name="my_new_dataset_name",
+        folder_id=folder_id,
+        layers_to_link=[
+            LayerToLink(
+                existing_dataset.dataset_id,
+                layer_name="image_230130b",
+                new_layer_name="image",
+            )
+        ],
+    )
+
+
+if __name__ == "__main__":
+    main()

webknossos/examples/upload_image_data.py

@@ -2,8 +2,8 @@
 from skimage import data
 
 import webknossos as wk
-from webknossos.dataset import COLOR_CATEGORY
-from webknossos.dataset.properties import (
+from webknossos import COLOR_CATEGORY
+from webknossos.dataset_properties import (
     DatasetViewConfiguration,
     LayerViewConfiguration,
 )

webknossos/local_wk_setup.sh

@@ -49,7 +49,7 @@ function ensure_local_test_wk {
         echo "Using the already running local webknossos setup at localhost:9000. Make sure l4_sample exists and is set to public first!"
     fi
 
-    if ! curl -s -H "X-Auth-Token: $WK_TOKEN" localhost:9000/api/user | grep user_A@scalableminds.com > /dev/null; then
+    if ! curl -s -H "X-Auth-Token: $WK_TOKEN" localhost:9000/api/user | grep user_a@scalableminds.com > /dev/null; then
         echo "The login user user_A@scalableminds.com could not be found or changed."
         echo "Please ensure that the test-db is prepared by running this in the webknossos repo"
         echo "(⚠️ this overwrites your local webknossos database):"

webknossos/pyproject.toml

@@ -49,6 +49,7 @@ dependencies = [
     "python-dateutil ~=2.8.0",
     "python-dotenv ~=1.0.1",
     "rich ~=13.8.0",
+    "s3fs>=2024.6.1",
     "scipy >=1.13",
     "tensorstore~=0.1.74",
     "typer ~=0.12.5",