Merge pull request #37 from tbirdso/http-read

ENH: Read from `http` sources

src/itkOMEZarrNGFFImageIO.cxx

@@ -78,6 +78,29 @@ ReadFromStore(const tensorstore::TensorStore<> & store, const ImageIORegion & io
   }
 }
 
+// Update an existing "read" specification for an "http" driver to retrieve remote files.
+// Note that an "http" driver specification may operate on an HTTP or HTTPS connection.
+void
+MakeKVStoreHTTPDriverSpec(nlohmann::json & spec, const std::string & fullPath)
+{
+  // Decompose path into a base URL and reference subpath according to TensorStore HTTP KVStore driver spec
+  // https://google.github.io/tensorstore/kvstore/http/index.html
+  spec["kvstore"] = { { "driver", "http" } };
+
+  // Naively decompose the URL into "base" and "resource" components.
+  // Generally assumes that the spec will only be used once to access a specific resource.
+  // For example, the URL "http://localhost/path/to/resource.json" will be split
+  // into components "http://localhost/path/to" and "resource.json".
+  //
+  // Could be revisited for a better root "base_url" at the top level allowing acces
+  // to multiple subpaths. For instance, decomposing the example above into
+  // "http://localhost/" and "path/to/resource.json" would allow for a given HTTP spec
+  // to be more easily reused with different subpaths.
+  //
+  spec["kvstore"]["base_url"] = fullPath.substr(0, fullPath.find_last_of("/"));
+  spec["kvstore"]["path"] = fullPath.substr(fullPath.find_last_of("/") + 1);
+}
+
 } // namespace
 
 thread_local tensorstore::Context tsContext = tensorstore::Context::Default();
@@ -229,6 +252,10 @@ getKVstoreDriver(std::string path)
   {
     return "file";
   }
+  if (path.substr(0, 4) == "http")
+  { // http or https
+    return "http";
+  }
   if (path.substr(path.size() - 4) == ".zip" || path.substr(path.size() - 7) == ".memory")
   {
     return "zip";
@@ -262,10 +289,13 @@ bool
 jsonRead(const std::string path, nlohmann::json & result, std::string driver)
 {
   // Reading JSON via TensorStore allows it to be in the cloud
-  auto attrs_store = tensorstore::Open<nlohmann::json, 0>(
-                       { { "driver", "json" }, { "kvstore", { { "driver", driver }, { "path", path } } } }, tsContext)
-                       .result()
-                       .value();
+  nlohmann::json readSpec = { { "driver", "json" }, { "kvstore", { { "driver", driver }, { "path", path } } } };
+  if (driver == "http")
+  {
+    MakeKVStoreHTTPDriverSpec(readSpec, path);
+  }
+
+  auto attrs_store = tensorstore::Open<nlohmann::json, 0>(readSpec, tsContext).result().value();
 
   auto attrs_array_result = tensorstore::Read(attrs_store).result();
 
@@ -336,12 +366,17 @@ thread_local tensorstore::TensorStore<> store; // initialized by ReadImageInform
 void
 OMEZarrNGFFImageIO::ReadArrayMetadata(std::string path, std::string driver)
 {
-  auto openFuture =
-    tensorstore::Open({ { "driver", "zarr" }, { "kvstore", { { "driver", driver }, { "path", path } } } },
-                      tsContext,
-                      tensorstore::OpenMode::open,
-                      tensorstore::RecheckCached{ false },
-                      tensorstore::ReadWriteMode::read);
+  nlohmann::json readSpec = { { "driver", "zarr" }, { "kvstore", { { "driver", driver }, { "path", path } } } };
+  if (driver == "http")
+  {
+    MakeKVStoreHTTPDriverSpec(readSpec, path);
+  }
+
+  auto openFuture = tensorstore::Open(readSpec,
+                                      tsContext,
+                                      tensorstore::OpenMode::open,
+                                      tensorstore::RecheckCached{ false },
+                                      tensorstore::ReadWriteMode::read);
   TS_EVAL_CHECK(openFuture);
   store = openFuture.value();
   auto shape_span = store.domain().shape();

test/CMakeLists.txt

@@ -1,8 +1,10 @@
 itk_module_test()
 
 set(IOOMEZarrNGFFTests
+  itkOMEZarrNGFFHTTPTest.cxx
   itkOMEZarrNGFFImageIOTest.cxx
   itkOMEZarrNGFFInMemoryTest.cxx
+  itkOMEZarrNGFFReadTest.cxx
   itkOMEZarrNGFFReadSubregionTest.cxx
   )
 
@@ -102,3 +104,17 @@ COMMAND IOOMEZarrNGFFTestDriver
     ${ITK_TEST_OUTPUT_DIR}/cthead1.zarr
     ${ITK_TEST_OUTPUT_DIR}/cthead1Subregion.mha
 )
+
+# HTTP test with encoded test cases
+itk_add_test(
+  NAME IOOMEZarrNGFFHTTP_2D
+  COMMAND IOOMEZarrNGFFTestDriver
+    itkOMEZarrNGFFHTTPTest
+    0
+)
+itk_add_test(
+  NAME IOOMEZarrNGFFHTTP_3D
+  COMMAND IOOMEZarrNGFFTestDriver
+    itkOMEZarrNGFFHTTPTest
+    1
+)

test/itkOMEZarrNGFFHTTPTest.cxx

@@ -0,0 +1,185 @@
+/*=========================================================================
+ *
+ *  Copyright NumFOCUS
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *         https://www.apache.org/licenses/LICENSE-2.0.txt
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *=========================================================================*/
+
+// Read an OME-Zarr image from a remote store.
+// Example data is available at https://github.com/ome/ome-ngff-prototypes
+
+#include <fstream>
+#include "itkImageFileReader.h"
+#include "itkImageFileWriter.h"
+#include "itkOMEZarrNGFFImageIO.h"
+#include "itkOMEZarrNGFFImageIOFactory.h"
+#include "itkTestingMacros.h"
+#include "itkImageIOBase.h"
+
+namespace
+{
+bool
+test2DImage()
+{
+  using ImageType = itk::Image<unsigned char, 2>;
+  const std::string resourceURL = "https://s3.embl.de/i2k-2020/ngff-example-data/v0.4/yx.ome.zarr";
+
+  itk::OMEZarrNGFFImageIOFactory::RegisterOneFactory();
+
+  // Baseline image metadata
+  auto baselineImage0 = ImageType::New();
+  baselineImage0->SetRegions(itk::MakeSize(1024, 930));
+  // unit origin, spacing, direction
+
+  auto baselineImage1 = ImageType::New();
+  baselineImage1->SetRegions(itk::MakeSize(512, 465));
+  const float spacing1[2] = { 2.0, 2.0 };
+  baselineImage1->SetSpacing(spacing1);
+  // unit origin, direction
+
+  auto baselineImage2 = ImageType::New();
+  baselineImage2->SetRegions(itk::MakeSize(256, 232));
+  const float spacing2[2] = { 4.0, 4.0 };
+  baselineImage2->SetSpacing(spacing2);
+  // unit origin, direction
+
+  // Resolution 0
+  auto image = itk::ReadImage<ImageType>(resourceURL);
+  image->Print(std::cout);
+
+  ITK_TEST_EXPECT_EQUAL(image->GetLargestPossibleRegion(), baselineImage0->GetLargestPossibleRegion());
+  ITK_TEST_EXPECT_EQUAL(image->GetBufferedRegion(), baselineImage0->GetLargestPossibleRegion());
+  ITK_TEST_EXPECT_EQUAL(image->GetSpacing(), baselineImage0->GetSpacing());
+  ITK_TEST_EXPECT_EQUAL(image->GetOrigin(), baselineImage0->GetOrigin());
+
+  // Resolution 1
+  auto imageIO = itk::OMEZarrNGFFImageIO::New();
+  imageIO->SetDatasetIndex(1);
+  auto reader1 = itk::ImageFileReader<ImageType>::New();
+  reader1->SetFileName(resourceURL);
+  reader1->SetImageIO(imageIO);
+  reader1->Update();
+  image = reader1->GetOutput();
+  image->Print(std::cout);
+  ITK_TEST_EXPECT_EQUAL(image->GetLargestPossibleRegion(), baselineImage1->GetLargestPossibleRegion());
+  ITK_TEST_EXPECT_EQUAL(image->GetBufferedRegion(), baselineImage1->GetLargestPossibleRegion());
+  ITK_TEST_EXPECT_EQUAL(image->GetSpacing(), baselineImage1->GetSpacing());
+  ITK_TEST_EXPECT_EQUAL(image->GetOrigin(), baselineImage1->GetOrigin());
+
+  // Resolution 2
+  imageIO->SetDatasetIndex(2);
+  auto reader2 = itk::ImageFileReader<ImageType>::New();
+  reader2->SetFileName(resourceURL);
+  reader2->SetImageIO(imageIO);
+  reader2->Update();
+  image = reader2->GetOutput();
+  image->Print(std::cout);
+  ITK_TEST_EXPECT_EQUAL(image->GetLargestPossibleRegion(), baselineImage2->GetLargestPossibleRegion());
+  ITK_TEST_EXPECT_EQUAL(image->GetBufferedRegion(), baselineImage2->GetLargestPossibleRegion());
+  ITK_TEST_EXPECT_EQUAL(image->GetSpacing(), baselineImage2->GetSpacing());
+  ITK_TEST_EXPECT_EQUAL(image->GetOrigin(), baselineImage2->GetOrigin());
+
+  return EXIT_SUCCESS;
+}
+
+bool
+test3DImage()
+{
+  using ImageType = itk::Image<unsigned char, 3>;
+  const std::string resourceURL = "https://s3.embl.de/i2k-2020/ngff-example-data/v0.4/zyx.ome.zarr";
+
+  // Baseline image metadata
+  auto baselineImage0 = ImageType::New();
+  baselineImage0->SetRegions(itk::MakeSize(483, 393, 603));
+  const float spacing0[3] = { 64, 64, 64 };
+  baselineImage0->SetSpacing(spacing0);
+
+  auto baselineImage1 = ImageType::New();
+  baselineImage1->SetRegions(itk::MakeSize(242, 196, 302));
+  const float spacing1[3] = { 128, 128, 128 };
+  baselineImage1->SetSpacing(spacing1);
+
+  auto baselineImage2 = ImageType::New();
+  baselineImage2->SetRegions(itk::MakeSize(121, 98, 151));
+  const float spacing2[3] = { 256, 256, 256 };
+  baselineImage2->SetSpacing(spacing2);
+
+  itk::OMEZarrNGFFImageIOFactory::RegisterOneFactory();
+
+  // Resolution 0
+  auto image = itk::ReadImage<ImageType>(resourceURL);
+  image->Print(std::cout);
+
+  ITK_TEST_EXPECT_EQUAL(image->GetLargestPossibleRegion(), baselineImage0->GetLargestPossibleRegion());
+  ITK_TEST_EXPECT_EQUAL(image->GetBufferedRegion(), baselineImage0->GetLargestPossibleRegion());
+  ITK_TEST_EXPECT_EQUAL(image->GetSpacing(), baselineImage0->GetSpacing());
+  ITK_TEST_EXPECT_EQUAL(image->GetOrigin(), baselineImage0->GetOrigin());
+
+  // Resolution 1
+  auto imageIO = itk::OMEZarrNGFFImageIO::New();
+  imageIO->SetDatasetIndex(1);
+  auto reader1 = itk::ImageFileReader<ImageType>::New();
+  reader1->SetFileName(resourceURL);
+  reader1->SetImageIO(imageIO);
+  reader1->Update();
+  image = reader1->GetOutput();
+  image->Print(std::cout);
+  ITK_TEST_EXPECT_EQUAL(image->GetLargestPossibleRegion(), baselineImage1->GetLargestPossibleRegion());
+  ITK_TEST_EXPECT_EQUAL(image->GetBufferedRegion(), baselineImage1->GetLargestPossibleRegion());
+  ITK_TEST_EXPECT_EQUAL(image->GetSpacing(), baselineImage1->GetSpacing());
+  ITK_TEST_EXPECT_EQUAL(image->GetOrigin(), baselineImage1->GetOrigin());
+
+  // Resolution 2
+  imageIO->SetDatasetIndex(2);
+  auto reader2 = itk::ImageFileReader<ImageType>::New();
+  reader2->SetFileName(resourceURL);
+  reader2->SetImageIO(imageIO);
+  reader2->Update();
+  image = reader2->GetOutput();
+  image->Print(std::cout);
+  ITK_TEST_EXPECT_EQUAL(image->GetLargestPossibleRegion(), baselineImage2->GetLargestPossibleRegion());
+  ITK_TEST_EXPECT_EQUAL(image->GetBufferedRegion(), baselineImage2->GetLargestPossibleRegion());
+  ITK_TEST_EXPECT_EQUAL(image->GetSpacing(), baselineImage2->GetSpacing());
+  ITK_TEST_EXPECT_EQUAL(image->GetOrigin(), baselineImage2->GetOrigin());
+
+  return EXIT_SUCCESS;
+}
+} // namespace
+
+int
+itkOMEZarrNGFFHTTPTest(int argc, char * argv[])
+{
+  if (argc < 2)
+  {
+    std::cerr << "Missing parameters." << std::endl;
+    std::cerr << "Usage: " << std::endl;
+    std::cerr << itkNameOfTestExecutableMacro(argv) << " <test-case-id>" << std::endl;
+    return EXIT_FAILURE;
+  }
+  size_t testCase = std::atoi(argv[1]);
+
+  switch (testCase)
+  {
+    case 0:
+      return test2DImage();
+      break;
+    case 1:
+      return test3DImage();
+      break;
+    default:
+      throw std::invalid_argument("Invalid test case ID: " + std::to_string(testCase));
+  }
+
+  return EXIT_FAILURE;
+}

test/itkOMEZarrNGFFReadTest.cxx

@@ -0,0 +1,101 @@
+/*=========================================================================
+ *
+ *  Copyright NumFOCUS
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *         https://www.apache.org/licenses/LICENSE-2.0.txt
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *=========================================================================*/
+
+/// This test utility may be used to validate that an OME-Zarr image can be
+/// read from disk or from an HTTP source.
+///
+/// No attempt is made to validate input data. A summary of the retrieved image
+/// is printed to `std::cout`.
+///
+/// Does not currently support multichannel sources.
+/// https://github.com/InsightSoftwareConsortium/ITKIOOMEZarrNGFF/issues/32
+
+#include <fstream>
+#include <string>
+
+#include "itkImageRegionConstIteratorWithIndex.h"
+#include "itkImageFileReader.h"
+#include "itkImageFileWriter.h"
+#include "itkExtractImageFilter.h"
+#include "itkOMEZarrNGFFImageIO.h"
+#include "itkOMEZarrNGFFImageIOFactory.h"
+#include "itkTestingMacros.h"
+#include "itkImageIOBase.h"
+
+namespace
+{
+template <typename PixelType = unsigned char, size_t ImageDimension = 3>
+void
+doRead(const std::string & path, const int datasetIndex)
+{
+  using ImageType = itk::Image<PixelType, ImageDimension>;
+  auto imageReader = itk::ImageFileReader<ImageType>::New();
+  imageReader->SetFileName(path);
+
+  auto imageIO = itk::OMEZarrNGFFImageIO::New();
+  imageIO->SetDatasetIndex(datasetIndex);
+  imageReader->SetImageIO(imageIO);
+
+  imageReader->UpdateOutputInformation();
+  auto output = imageReader->GetOutput();
+  output->Print(std::cout);
+
+  imageReader->Update();
+  output->Print(std::cout);
+}
+} // namespace
+
+int
+itkOMEZarrNGFFReadTest(int argc, char * argv[])
+{
+  if (argc < 2)
+  {
+    std::cerr << "Missing parameters." << std::endl;
+    std::cerr << "Usage: " << std::endl;
+    std::cerr << itkNameOfTestExecutableMacro(argv) << " Input <ImageDimension> <DatasetIndex> [NumChannels]"
+              << std::endl;
+    return EXIT_FAILURE;
+  }
+  const char * inputFileName = argv[1];
+  const size_t imageDimension = (argc > 2 ? std::atoi(argv[2]) : 3);
+  const size_t datasetIndex = (argc > 3 ? std::atoi(argv[3]) : 0);
+  const size_t numChannels = (argc > 4 ? std::atoi(argv[4]) : 1);
+
+  if (numChannels != 1)
+  {
+    throw std::runtime_error("Multichannel image reading is not currently supported");
+  }
+  else if (imageDimension == 2)
+  {
+    doRead<unsigned char, 2>(inputFileName, datasetIndex);
+  }
+  else if (imageDimension == 3)
+  {
+    doRead<unsigned char, 3>(inputFileName, datasetIndex);
+  }
+  else if (imageDimension == 4)
+  {
+    doRead<unsigned char, 4>(inputFileName, datasetIndex);
+  }
+  else
+  {
+    throw std::invalid_argument("Received an invalid test case");
+  }
+
+  return EXIT_SUCCESS;
+}