Dynamic loading of Ogawa sparse fields working.

Author: magnusw

export/OgawaFwd.h

@@ -45,6 +45,8 @@
 #ifndef _INCLUDED_Field3D_OgawaFwd_H_
 #define _INCLUDED_Field3D_OgawaFwd_H_
 
+#include <boost/shared_ptr.hpp>
+
 #include "ns.h"
 
 //----------------------------------------------------------------------------//
@@ -66,6 +68,8 @@ class OgIDataset;
 template <typename T>
 class OgODataset;
 
+typedef boost::shared_ptr<OgIGroup> OgIGroupPtr;
+
 FIELD3D_NAMESPACE_HEADER_CLOSE
 
 //----------------------------------------------------------------------------//
@@ -80,6 +84,8 @@ namespace Alembic {
   }
 }
 
+typedef boost::shared_ptr<Alembic::Ogawa::IArchive> IArchivePtr;
+
 //----------------------------------------------------------------------------//
 
 #endif // include guard

export/SparseField.h

@@ -568,7 +568,7 @@ class SparseField
   //! Internal function to create a Reference for the current field,
   //! for use in dynamic reading.
   void addReference(const std::string &filename, const std::string &layerPath,
-                    int valuesPerBlock, int occupiedBlocks);
+                    int valuesPerBlock, int numVoxels, int occupiedBlocks);
   //! Internal function to setup the Reference's block pointers, for
   //! use with dynamic reading.
   void setupReferenceBlocks();
@@ -1336,6 +1336,7 @@ SparseField<Data_T>::copySparseField(const SparseField<Data_T> &o)
       m_fileManager->reference<Data_T>(o.m_fileId);
     addReference(oldReference->filename, oldReference->layerPath,
                  oldReference->valuesPerBlock,
+                 oldReference->numVoxels, 
                  oldReference->occupiedBlocks);
     copyBlockStates(o);
     setupReferenceBlocks();
@@ -1365,6 +1366,7 @@ template <class Data_T>
 void SparseField<Data_T>::addReference(const std::string &filename,
                                        const std::string &layerPath,
                                        int valuesPerBlock,
+                                       int numVoxels, 
                                        int occupiedBlocks)
 {
   m_fileManager = &SparseFileManager::singleton();
@@ -1373,6 +1375,7 @@ void SparseField<Data_T>::addReference(const std::string &filename,
   SparseFile::Reference<Data_T> *reference =
     m_fileManager->reference<Data_T>(m_fileId);
   reference->valuesPerBlock = valuesPerBlock;
+  reference->numVoxels = numVoxels;
   reference->occupiedBlocks = occupiedBlocks;
   reference->setNumBlocks(m_numBlocks);
 }

export/SparseFile.h

@@ -54,6 +54,7 @@
 
 #include "Exception.h"
 #include "Hdf5Util.h"
+#include "OgawaFwd.h"
 #include "SparseDataReader.h"
 #include "Traits.h"
 
@@ -77,6 +78,9 @@ namespace Sparse {
 template <typename Data_T>
 class SparseField;
 
+template <typename Data_T>
+class OgSparseDataReader;
+
 //----------------------------------------------------------------------------//
 
 namespace SparseFile {
@@ -113,6 +117,7 @@ class Reference
   std::string filename;
   std::string layerPath;
   int valuesPerBlock;
+  int numVoxels;
   int occupiedBlocks;
 
   //! Index in file for each block
@@ -216,8 +221,19 @@ class Reference
   //! openFile().
   SparseDataReader<Data_T> *m_reader;
 
+  //! Shared pointer to the ogawa reader. 
+  boost::shared_ptr<OgSparseDataReader<Data_T> > m_ogReaderPtr;
+  //! Pointer to the ogawa reader. NULL at construction time. Created in 
+  //! openFile().
+  OgSparseDataReader<Data_T> *m_ogReader;
+  //! Ogawa archive
+  IArchivePtr m_ogArchive;
+  //! Ogawa archive root
+  OgIGroupPtr m_ogRoot;
+  //! Ogawa layer group
+  OgIGroupPtr m_ogLayerGroup;
+
   //! Mutex to prevent two threads from modifying conflicting data
-  // boost::mutex m_mutex;
   Mutex m_mutex;
 
   //! Number of currently active blocks
@@ -520,8 +536,8 @@ template <class Data_T>
 Reference<Data_T>::Reference(const std::string a_filename, 
                              const std::string a_layerPath)
   : filename(a_filename), layerPath(a_layerPath),
-    valuesPerBlock(-1), occupiedBlocks(-1),
-    blockMutex(NULL), m_fileHandle(-1), m_reader(NULL),
+    valuesPerBlock(-1), numVoxels(-1), occupiedBlocks(-1),
+    blockMutex(NULL), m_fileHandle(-1), m_reader(NULL), m_ogReader(NULL), 
     m_numActiveBlocks(0)
 { 
   /* Empty */ 
@@ -534,8 +550,9 @@ Reference<Data_T>::~Reference()
 {
   closeFile();
 
-  if (m_reader)
+  if (m_reader) {
     delete m_reader;
+  }
 
   if (blockMutex)
     delete [] blockMutex;
@@ -546,6 +563,8 @@ Reference<Data_T>::~Reference()
 template <class Data_T>
 Reference<Data_T>::Reference(const Reference<Data_T> &o)
 {
+  m_ogReaderPtr.reset();
+  m_ogReader = NULL;
   m_reader = NULL;
   blockMutex = NULL;
   *this = o;
@@ -565,6 +584,7 @@ Reference<Data_T>::operator=(const Reference<Data_T> &o)
   filename = o.filename;
   layerPath = o.layerPath;
   valuesPerBlock = o.valuesPerBlock;
+  numVoxels = o.numVoxels;
   occupiedBlocks = o.occupiedBlocks;
   fileBlockIndices = o.fileBlockIndices;
   blockLoaded = o.blockLoaded;
@@ -594,6 +614,9 @@ Reference<Data_T>::operator=(const Reference<Data_T> &o)
     delete m_reader;
   m_reader = NULL;
 
+  m_ogReaderPtr.reset();
+  m_ogReader = NULL;
+
   return *this;
 }
 
@@ -635,48 +658,6 @@ void Reference<Data_T>::setNumBlocks(int numBlocks)
 
 //----------------------------------------------------------------------------//
 
-template <class Data_T>
-void Reference<Data_T>::openFile()
-{
-  using namespace Exc;
-  using namespace Hdf5Util;
-
-  boost::mutex::scoped_lock lock_A(m_mutex);
-
-  // check that the file wasn't already opened before obtaining the lock
-  if (fileIsOpen()) {
-    return;
-  }
-
-  {
-    // Hold the global lock
-    GlobalLock lock(g_hdf5Mutex);
-    // Open the file
-    m_fileHandle = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
-    if (m_fileHandle < 0) {
-      printf("openFile(): No file handle. Dying.\n");
-      throw NoSuchFileException(filename);
-    }
-    // Open the layer group
-    m_layerGroup.open(m_fileHandle, layerPath.c_str());
-    if (m_layerGroup.id() < 0) {
-      Msg::print(Msg::SevWarning, "In SparseFile::Reference::openFile: "
-                 "Couldn't find layer group " + layerPath + 
-                 " in .f3d file ");
-      throw FileIntegrityException(filename);
-    }
-  }
-
-  // Re-allocate reader
-  if (m_reader) {
-    delete m_reader;
-  }
-  m_reader = new SparseDataReader<Data_T>(m_layerGroup.id(), valuesPerBlock, 
-                                          occupiedBlocks);
-}
-
-//----------------------------------------------------------------------------//
-
 template <class Data_T>
 void Reference<Data_T>::closeFile()
 {
@@ -689,25 +670,6 @@ void Reference<Data_T>::closeFile()
 
 //----------------------------------------------------------------------------//
 
-template <class Data_T>
-void Reference<Data_T>::loadBlock(int blockIdx)
-{
-  boost::mutex::scoped_lock lock(m_mutex);
-
-  // Allocate the block  
-  blocks[blockIdx]->resize(valuesPerBlock);
-  assert(blocks[blockIdx]->data != NULL);
-  // Read the data
-  assert(m_reader);
-  m_reader->readBlock(fileBlockIndices[blockIdx], *blocks[blockIdx]->data);
-  // Mark block as loaded
-  blockLoaded[blockIdx] = 1;
-  // Track count
-  m_numActiveBlocks++;
-}
-
-//----------------------------------------------------------------------------//
-
 template <class Data_T>
 void Reference<Data_T>::unloadBlock(int blockIdx)
 {
@@ -748,7 +710,7 @@ void Reference<Data_T>::decBlockRef(int blockIdx)
 template <class Data_T>
 int Reference<Data_T>::blockSize(int /* blockIdx */) const
 {
-  return valuesPerBlock * sizeof(Data_T);
+  return numVoxels * sizeof(Data_T);
 }
 
 //----------------------------------------------------------------------------//

export/Types.h

@@ -85,38 +85,6 @@ struct Interval
 
 typedef std::vector<Interval> IntervalVec;
 
-//----------------------------------------------------------------------------//
-// Ogawa Types
-//----------------------------------------------------------------------------//
-
-#if !defined(_MSC_VER)
-using ::uint8_t;
-using ::int8_t;
-using ::uint16_t;
-using ::int16_t;
-using ::uint32_t;
-using ::int32_t;
-using ::uint64_t;
-using ::int64_t;
-#else
-typedef unsigned char           uint8_t;
-typedef signed char             int8_t;
-typedef unsigned short          uint16_t;
-typedef signed short            int16_t;
-typedef unsigned int            uint32_t;
-typedef int                     int32_t;
-typedef unsigned long long      uint64_t;
-typedef long long               int64_t;
-#endif
-
-typedef half                    float16_t;
-typedef float                   float32_t;
-typedef double                  float64_t;
-
-typedef Field3D::V3h            vec16_t;
-typedef Field3D::V3f            vec32_t;
-typedef Field3D::V3d            vec64_t;
-
 //----------------------------------------------------------------------------//
 
 #endif // Include guard

include/MIPFieldIO.h

@@ -278,7 +278,7 @@ class GenericLazyLoadActionHDF5
     // Instantiate I/O
     FieldIO::Ptr io = 
       ClassFactory::singleton().createFieldIO(Field_T::staticClassName());
-    FieldBase::Ptr field = io->read(levelGroup, m_filename, m_path, m_typeEnum);
+    FieldBase::Ptr field = io->read(*levelGroup, m_filename, m_path, m_typeEnum);
     if (!field) {
       throw Exc::MIPFieldIOException("Failed to read MIP level from disk.");
     }

src/Field3DFile.cpp

@@ -712,8 +712,6 @@ Field3DInputFile::~Field3DInputFile()
 
 bool Field3DInputFile::open(const string &filename)
 {
-  GlobalLock lock(g_hdf5Mutex);
-
   clear();
 
   bool success = true;
@@ -1249,8 +1247,6 @@ Field3DOutputFile::writeGroupMembership()
   using namespace std;
   using namespace Hdf5Util;
 
-  GlobalLock lock(g_hdf5Mutex);
-
   if (!m_groupMembership.size())
     return true;
 
@@ -1316,14 +1312,10 @@ void Field3DFileBase::printHierarchy() const
       cout << "  Mapping: " << (**i).mapping->className() << endl;
     else 
       cout << "  Mapping: NULL" << endl;
-    cout << "  Scalar layers: " << endl;
-    vector<string> sNames;
-    (**i).getScalarLayerNames(sNames);
-    for_each(sNames.begin(), sNames.end(), print<string>(4));
-    cout << "  Vector layers: " << endl;
-    vector<string> vNames;
-    (**i).getVectorLayerNames(vNames);
-    for_each(vNames.begin(), vNames.end(), print<string>(4));
+    cout << "  Layers: " << endl;
+    vector<string> names;
+    (**i).getLayerNames(names);
+    for_each(names.begin(), names.end(), print<string>(4));    
   }
 }
 

src/Field3DFileHDF5.cpp

@@ -1586,19 +1586,6 @@ void Field3DFileHDF5Base::printHierarchy() const
 // Function Implementations
 //----------------------------------------------------------------------------//
 
-bool fileExists(const std::string &filename)
-{
-#ifdef WIN32
-  struct __stat64 statbuf;
-  return (_stat64(filename.c_str(), &statbuf) != -1);
-#else
-  struct stat statbuf;
-  return (stat(filename.c_str(), &statbuf) != -1);
-#endif
-}
-
-//----------------------------------------------------------------------------//
-
 bool writeField(hid_t layerGroup, FieldBase::Ptr field)
 {
   ClassFactory &factory = ClassFactory::singleton();

src/SparseFieldIO.cpp

@@ -761,7 +761,8 @@ SparseFieldIO::readData(const OgIGroup &location, const Box3i &extents,
   if (dynamicLoading) {
     // Set up the field reference
     //! \todo The valuesPerBlock is wrong. Fix
-    result->addReference(filename, layerPath, valuesPerBlock, occupiedBlocks);
+    result->addReference(filename, layerPath, valuesPerBlock, numVoxels,
+                         occupiedBlocks);
   }
 
   // Read the block info data sets ---
@@ -1159,7 +1160,7 @@ bool SparseFieldIO::readData(hid_t location,
   if (dynamicLoading) {
     // Set up the field reference
     result->addReference(filename, layerPath,
-                         valuesPerBlock,
+                         valuesPerBlock, numVoxels, 
                          occupiedBlocks);
   }