Merge pull request #7 from takipi/feature/search-file-access

Feature/search file access

pom.xml

@@ -5,7 +5,7 @@
 
 	<groupId>com.takipi</groupId>
 	<artifactId>takipi-storage</artifactId>
-	<version>1.5.0</version>
+	<version>1.6.0</version>
 
 	<properties>
 		<!-- use UTF-8 for everything -->

src/main/java/com/takipi/oss/storage/TakipiStorageMain.java

@@ -19,6 +19,7 @@
 import com.takipi.oss.storage.resources.fs.JsonMultiDeleteStorageResource;
 import com.takipi.oss.storage.resources.fs.JsonMultiFetchStorageResource;
 import com.takipi.oss.storage.resources.fs.JsonSimpleFetchStorageResource;
+import com.takipi.oss.storage.resources.fs.JsonSimpleSearchStorageResource;
 
 public class TakipiStorageMain extends Application<TakipiStorageConfiguration> {
     public static void main(String[] args) throws Exception {
@@ -46,6 +47,7 @@ public void run(TakipiStorageConfiguration configuration, Environment environmen
         environment.jersey().register(new JsonMultiDeleteStorageResource(configuration));
 
         environment.jersey().register(new JsonSimpleFetchStorageResource(configuration));
+        environment.jersey().register(new JsonSimpleSearchStorageResource(configuration));
 
         environment.jersey().register(new PingStorageResource());
         environment.jersey().register(new TreeStorageResource(configuration));

src/main/java/com/takipi/oss/storage/data/simple/SimpleSearchRequest.java

@@ -0,0 +1,10 @@
+package com.takipi.oss.storage.data.simple;
+
+import com.takipi.oss.storage.data.EncodingType;
+
+public class SimpleSearchRequest {
+    public EncodingType encodingType;
+    public String name;
+    public String baseSearchPath;
+    public boolean preventDuplicates;
+}

src/main/java/com/takipi/oss/storage/data/simple/SimpleSearchResponse.java

@@ -0,0 +1,19 @@
+package com.takipi.oss.storage.data.simple;
+
+public class SimpleSearchResponse {
+    String data;
+    String path;
+
+    public SimpleSearchResponse(String data, String path) {
+        this.data = data;
+        this.path = path;
+    }
+
+    public String getData() {
+        return data;
+    }
+
+    public String getPath() {
+        return path;
+    }
+}

src/main/java/com/takipi/oss/storage/fs/folder/FolderFilesystem.java

@@ -16,7 +16,11 @@ public abstract class FolderFilesystem<T> extends FolderFilesystemHealth impleme
     public FolderFilesystem(String rootFolder, double maxUsedStoragePercentage) {
         super(rootFolder, maxUsedStoragePercentage);
     }
-
+
+    public File getRoot() {
+        return root;
+    }
+
     @Override
     public InputStream get(T record) throws IOException {
         File file = new File(buildPath(record));

src/main/java/com/takipi/oss/storage/fs/folder/simple/SimpleFilesystem.java

@@ -1,10 +1,10 @@
 package com.takipi.oss.storage.fs.folder.simple;
 
-import java.io.File;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 
 import com.takipi.oss.storage.fs.folder.FolderFilesystem;
+import com.takipi.oss.storage.helper.FilesystemUtil;
 
 public class SimpleFilesystem extends FolderFilesystem<String> {
     public SimpleFilesystem(String rootFolder, double maxUsedStoragePercentage) {
@@ -19,6 +19,6 @@ protected String buildPath(String record) {
     }
 
     protected String escape(String value) {
-        return value.replace("/", File.separator).replace("\\", File.separator);
+        return FilesystemUtil.fixPath(value);
     }
 }

src/main/java/com/takipi/oss/storage/helper/FilesystemUtil.java

@@ -1,35 +1,46 @@
 package com.takipi.oss.storage.helper;
 
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Deque;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.Set;
 
 import org.apache.commons.io.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
+import com.google.common.base.Predicate;
 import com.takipi.oss.storage.data.EncodingType;
 import com.takipi.oss.storage.fs.api.Filesystem;
 
 public class FilesystemUtil {
-    public static<T> String read(Filesystem<T> fs, T record, EncodingType encodingType) {
+    private static final Logger logger = LoggerFactory.getLogger(FilesystemUtil.class);
+
+    public static String fixPath(String path) {
+        return path.replace("/", File.separator).replace("\\", File.separator);
+    }
+
+    public static <T> String read(Filesystem<T> fs, T record, EncodingType encodingType) {
         InputStream is = null;
-        String result = null;
-
+
         try {
             is = fs.get(record);
-
-            result = encode(is, encodingType);
+            return encode(is, encodingType);
         } catch (IOException e) {
-
+            return null;
         } finally {
             if (is != null) {
                 try {
                     is.close();
-                } catch (IOException e) {}
+                } catch (IOException e) {
+                }
             }
         }
-
-        return result;
     }
-    
+
     private static String encode(InputStream is, EncodingType type) throws IOException {
         switch (type) {
             case PLAIN:
@@ -45,4 +56,57 @@ private static String encode(InputStream is, EncodingType type) throws IOExcepti
 
         throw new IllegalArgumentException("problem encoding - " + type);
     }
+
+    public static File listFilesRecursively(File baseFolder, Predicate<File> callback) {
+        try {
+            Set<File> seenFolders = new HashSet<>();
+            Deque<File> pendingFolders = new LinkedList<>();
+
+            pendingFolders.add(baseFolder);
+
+            while (!pendingFolders.isEmpty()) {
+                File folder = pendingFolders.pop();
+
+                if (seenFolders.contains(folder)) {
+                    continue;
+                }
+
+                seenFolders.add(folder);
+
+                try {
+                    if ((!folder.exists()) || (!folder.canRead())) {
+                        continue;
+                    }
+
+                    if (!folder.isDirectory()) {
+                        continue;
+                    }
+
+                    File[] files = folder.listFiles();
+
+                    if (files == null) {
+                        continue;
+                    }
+
+                    for (File file : files) {
+                        if (file.isDirectory()) {
+                            pendingFolders.add(file);
+                            continue;
+                        }
+
+                        if (callback.apply(file)) {
+                            return file;
+                        }
+                    }
+                } catch (Exception e) {
+                    logger.error("Error accessing folder {}.", folder);
+                }
+            }
+
+            return null;
+        } catch (Exception e) {
+            logger.error("Error searcing in {}.", baseFolder);
+            return null;
+        }
+    }
 }

src/main/java/com/takipi/oss/storage/resources/fs/JsonMultiDeleteStorageResource.java

@@ -38,7 +38,7 @@ public Response post(MultiDeleteRequest request) {
 
             return Response.ok(response).build();
         } catch (Exception e) {
-            return Response.serverError().entity("Problem getting keys").build();
+            return Response.serverError().entity("Problem deleting keys").build();
         }
     }
 

src/main/java/com/takipi/oss/storage/resources/fs/JsonSimpleFetchStorageResource.java

@@ -33,7 +33,7 @@ public Response post(SimpleFetchRequest request) {
         try {
             return handleResponse(request);
         } catch (Exception e) {
-            return Response.serverError().entity("Problem getting keys").build();
+            return Response.serverError().entity("Problem simple fetching").build();
         }
     }
 

src/main/java/com/takipi/oss/storage/resources/fs/JsonSimpleSearchStorageResource.java

@@ -0,0 +1,115 @@
+package com.takipi.oss.storage.resources.fs;
+
+import java.io.File;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.codahale.metrics.annotation.Timed;
+import com.google.common.base.Predicate;
+import com.takipi.oss.storage.TakipiStorageConfiguration;
+import com.takipi.oss.storage.data.simple.SimpleSearchRequest;
+import com.takipi.oss.storage.data.simple.SimpleSearchResponse;
+import com.takipi.oss.storage.helper.FilesystemUtil;
+import com.takipi.oss.storage.resources.fs.base.SimpleFileSystemStorageResource;
+
+@Path("/storage/v1/json/simplesearch")
+@Consumes(MediaType.APPLICATION_JSON)
+@Produces(MediaType.APPLICATION_JSON)
+public class JsonSimpleSearchStorageResource extends SimpleFileSystemStorageResource {
+    private static final Logger logger = LoggerFactory.getLogger(JsonSimpleSearchStorageResource.class);
+
+    public JsonSimpleSearchStorageResource(TakipiStorageConfiguration configuration) {
+        super(configuration);
+    }
+
+    @POST
+    @Timed
+    public Response post(SimpleSearchRequest request) {
+        try {
+            return handleResponse(request);
+        } catch (Exception e) {
+            return Response.serverError().entity("Problem simple searching").build();
+        }
+    }
+
+    private Response handleResponse(SimpleSearchRequest request) {
+        try {
+            File searchRoot = new File(fs.getRoot(), FilesystemUtil.fixPath(request.baseSearchPath));
+
+            ResourceFileCallback fileCallback = new ResourceFileCallback(request.name, request.preventDuplicates);
+            FilesystemUtil.listFilesRecursively(searchRoot, fileCallback);
+            File result = fileCallback.getFoundFile();
+
+            if (result == null) {
+                return searchFailed(request.name);
+            }
+
+            String relFSPath = result.getAbsolutePath().replace(fs.getRoot().getAbsolutePath(), "");
+            String data = FilesystemUtil.read(fs, relFSPath, request.encodingType);
+
+            if (data == null) {
+                return searchFailed(request.name);
+            }
+
+            return Response.ok(new SimpleSearchResponse(data, relFSPath.replace(request.name, ""))).build();
+
+        } catch (Exception e) {
+            logger.error("Problem getting: " + request.name, e);
+            return Response.serverError().entity("Problem getting " + request.name).build();
+        }
+    }
+
+    private Response searchFailed(String name) {
+        logger.warn("File not found: {}", name);
+        return Response.status(404).entity("File not found" + name).build();
+    }
+
+    private static class ResourceFileCallback implements Predicate<File>
+    {
+        private final String resourceName;
+        private final boolean preventDuplicates;
+
+        private File foundFile;
+
+        protected ResourceFileCallback(String resourceName, boolean preventDuplicates)
+        {
+            this.resourceName = resourceName;
+            this.preventDuplicates = preventDuplicates;
+
+            this.foundFile = null;
+        }
+
+        @Override
+        public boolean apply(File file)
+        {
+            if (!resourceName.equals(file.getName()))
+            {
+                return false;
+            }
+
+            if ((preventDuplicates) &&
+                (foundFile != null))
+            {
+                foundFile = null; // never find more than one result if preventing duplicates
+                return true;
+            }
+
+            foundFile = file;
+
+            return !preventDuplicates; // if we don't prevent duplicates, we stop right now
+        }
+
+        public File getFoundFile()
+        {
+            return foundFile;
+        }
+    }
+}

src/main/java/com/takipi/oss/storage/resources/fs/base/FolderFileSystemStorageResource.java

@@ -0,0 +1,14 @@
+package com.takipi.oss.storage.resources.fs.base;
+
+import com.takipi.oss.storage.TakipiStorageConfiguration;
+import com.takipi.oss.storage.fs.folder.FolderFilesystem;
+
+public abstract class FolderFileSystemStorageResource<T> {
+    protected final FolderFilesystem<T> fs;
+
+    public FolderFileSystemStorageResource(TakipiStorageConfiguration configuration) {
+        this.fs = getNewFileSystem(configuration);
+    }
+
+    protected abstract FolderFilesystem<T> getNewFileSystem(TakipiStorageConfiguration configuration);
+}

src/main/java/com/takipi/oss/storage/resources/fs/base/HashFileSystemStorageResource.java

@@ -2,16 +2,16 @@
 
 import com.takipi.oss.storage.TakipiStorageConfiguration;
 import com.takipi.oss.storage.fs.Record;
-import com.takipi.oss.storage.fs.api.Filesystem;
+import com.takipi.oss.storage.fs.folder.FolderFilesystem;
 import com.takipi.oss.storage.fs.folder.record.HashSubfolderFilesystem;
 
-public class HashFileSystemStorageResource extends FileSystemStorageResource<Record> {
+public class HashFileSystemStorageResource extends FolderFileSystemStorageResource<Record> {
     public HashFileSystemStorageResource(TakipiStorageConfiguration configuration) {
         super(configuration);
     }
 
     @Override
-    protected Filesystem<Record> getNewFileSystem(TakipiStorageConfiguration configuration) {
+    protected FolderFilesystem<Record> getNewFileSystem(TakipiStorageConfiguration configuration) {
         return new HashSubfolderFilesystem(configuration.getFolderPath(), configuration.getMaxUsedStoragePercentage());
     }
 }