Increase checksum noting and matching for imports. (rebase)

components/blitz/src/ome/formats/OMEROMetadataStoreClient.java

@@ -1588,7 +1588,7 @@ public String getDefaultGroupName() throws ServerError
 	}
 
 	/**
-	 * Retrieve teh default groups permission 'level'.
+	 * Retrieve the default group's permission 'level'.
 	 *
 	 * @return ImportEvent's group level
 	 * @throws ServerError

components/blitz/src/ome/formats/importer/ImportLibrary.java

@@ -80,18 +80,19 @@ public class ImportLibrary implements IObservable
     /** The class used to identify the screen target.*/
     private static final String SCREEN_CLASS = "omero.model.Screen";
 
+    /* adjusts file paths to the format required by managed repository */
     private static final ClientFilePathTransformer sanitizer = 
             new ClientFilePathTransformer(new MakePathComponentSafe());
-
+
+    /* checksum provider factory for verifying file integrity in upload */
+    private static final ChecksumProviderFactory checksumProviderFactory = new ChecksumProviderFactoryImpl();
+
     private final ArrayList<IObserver> observers = new ArrayList<IObserver>();
 
     private final OMEROMetadataStoreClient store;
 
     private final ManagedRepositoryPrx repo;
 
-    private final ChecksumProviderFactory checksumProviderFactory =
-            new ChecksumProviderFactoryImpl();
-
     /**
      * The library will not close the client instance. The reader will be closed
      * between calls to import.
@@ -237,9 +238,9 @@ public ImportProcessPrx createImport(final ImportContainer container)
     public List<String> uploadFilesToRepository(
             final String[] srcFiles, final ImportProcessPrx proc)
     {
-        final List<String> checksums = new ArrayList<String>();
         final byte[] buf = new byte[store.getDefaultBlockSize()];
         final int fileTotal = srcFiles.length;
+        final List<String> checksums = new ArrayList<String>(fileTotal);
 
         log.debug("Used files created:");
         for (int i = 0; i < fileTotal; i++) {

components/blitz/src/ome/services/blitz/repo/ManagedImportProcessI.java

@@ -24,18 +24,15 @@
 import org.apache.commons.logging.LogFactory;
 
 import Ice.Current;
-import Ice.ObjectFactory;
 
 import ome.services.blitz.impl.AbstractAmdServant;
 import ome.services.blitz.impl.ServiceFactoryI;
 import ome.services.blitz.repo.PublicRepositoryI.AMD_submit;
 import ome.services.blitz.repo.path.FsFile;
 import ome.services.blitz.util.ServiceFactoryAware;
 
-import omero.InternalException;
 import omero.ServerError;
 import omero.api.RawFileStorePrx;
-import omero.cmd.AMD_Session_submit;
 import omero.cmd.HandlePrx;
 import omero.grid.ImportLocation;
 import omero.grid.ImportProcessPrx;
@@ -46,7 +43,6 @@
 import omero.grid._ImportProcessTie;
 import omero.model.Fileset;
 import omero.model.FilesetJobLink;
-import omero.util.IceMapper;
 
 /**
  * Represents a single import within a defined-session
@@ -254,13 +250,12 @@ public HandlePrx verifyUpload(List<String> hashes, Current ignore)
         for (int i = 0; i < size; i++) {
             String usedFile = location.sharedPath + FsFile.separatorChar + location.usedFiles.get(i);
             CheckedPath cp = repo.checkPath(usedFile, this.current);
-            String client = hashes.get(i);
-            String server = cp.sha1();
-            if (!server.equals(client)) {
+            final String clientHash = hashes.get(i);
+            final String serverHash = cp.sha1();
+            if (!clientHash.equals(serverHash)) {
                 throw new omero.ValidationException(null, null,
-                        String.format("Hash mis-match for %s: " +
-					"server:%s<>client:%s",
-                                usedFile, server, client));
+                        "file checksum mismatch on upload: " + usedFile +
+                        " (client has " + clientHash + ", server has " + serverHash + ")");
             }
         }
 

components/insight/SRC/org/openmicroscopy/shoola/env/data/OMEROGateway.java

@@ -77,6 +77,10 @@
 import ome.formats.importer.ImportLibrary;
 import ome.formats.importer.OMEROWrapper;
 import ome.system.UpgradeCheck;
+import ome.util.checksum.ChecksumProvider;
+import ome.util.checksum.ChecksumProviderFactory;
+import ome.util.checksum.ChecksumProviderFactoryImpl;
+import ome.util.checksum.ChecksumType;
 import omero.ApiUsageException;
 import omero.AuthenticationException;
 import omero.ConcurrencyException;
@@ -316,6 +320,9 @@ class OMEROGateway
 	/** The collection of scripts that have a UI available. */
 	private static final List<String>		SCRIPTS_NOT_AVAILABLE_TO_USER;
 
+	/* checksum provider factory for verifying file integrity in upload */
+	private static final ChecksumProviderFactory checksumProviderFactory = new ChecksumProviderFactoryImpl();
+
 	static {
 		SUPPORTED_SPECIAL_CHAR = new ArrayList<Character>();
 		SUPPORTED_SPECIAL_CHAR.add(Character.valueOf('-'));
@@ -4295,6 +4302,7 @@ private synchronized OriginalFile upload(SecurityContext ctx, File file,
 		}
 		byte[] buf = new byte[INC]; 
 		FileInputStream stream = null;
+		final ChecksumProvider hasher = checksumProviderFactory.getProvider(ChecksumType.SHA1);
 		try {
 			stream = new FileInputStream(file);
 			long pos = 0;
@@ -4305,11 +4313,18 @@ private synchronized OriginalFile upload(SecurityContext ctx, File file,
 				pos += rlen;
 				bbuf = ByteBuffer.wrap(buf);
 				bbuf.limit(rlen);
+				hasher.putBytes(bbuf);
 			}
 			stream.close();
 			OriginalFile f = store.save();
 			closeService(ctx, store);
 			if (f != null) save = f;
+			final String clientHash = hasher.checksumAsString();
+			final String serverHash = save.getSha1().getValue();
+			if (!clientHash.equals(serverHash)) {
+			    throw new ImportException("file checksum mismatch on upload: " + file +
+			            " (client has " + clientHash + ", server has " + serverHash + ")");
+			}
 		} catch (Exception e) {
 			try {
 				if (fileCreated) deleteObject(ctx, save);

components/tools/OmeroPy/src/omero/util/populate_metadata.py

@@ -457,6 +457,7 @@ def write_to_omero(self):
         log.info('Table initialized with %d columns.' % (len(self.columns)))
         table.addData(self.columns)
         log.info('Added data column data.')
+        table.close()
         file_annotation = FileAnnotationI()
         file_annotation.ns = \
                 rstring('openmicroscopy.org/omero/bulk_annotations')

components/tools/OmeroPy/src/omero/util/populate_roi.py

@@ -632,6 +632,7 @@ def update_table(self, columns):
             column_report[column.name] = len(column.values)
         log.debug("Column report: %r" % column_report)
         self.table.addData(columns)
+        self.table.close()
         log.info("Table update took %sms" % (int(time.time() * 1000) - t0))
 
     def create_file_annotation(self, set_of_columns):