Added a Service interface to allow the Memory service to provide caching infrastructure for the store without the store binding to the memory service

ieb · ieb · commit 080433ec68e2 · 2011-01-27T18:37:09.000Z
diff --git a/src/main/java/org/sakaiproject/nakamura/api/lite/CacheHolder.java b/src/main/java/org/sakaiproject/nakamura/api/lite/CacheHolder.java
@@ -15,7 +15,7 @@
  * KIND, either express or implied. See the License for the
  * specific language governing permissions and limitations under the License.
  */
-package org.sakaiproject.nakamura.lite.accesscontrol;
+package org.sakaiproject.nakamura.api.lite;
 
 import java.util.Map;
 
diff --git a/src/main/java/org/sakaiproject/nakamura/api/lite/StorageCacheManager.java b/src/main/java/org/sakaiproject/nakamura/api/lite/StorageCacheManager.java
@@ -0,0 +1,17 @@
+package org.sakaiproject.nakamura.api.lite;
+
+import java.util.Map;
+
+/**
+ * Provides Cache implementations for all the three areas represented as Maps.
+ * If an implementation of this interface is present it will be used.
+ */
+public interface StorageCacheManager {
+
+    Map<String, CacheHolder> getAccessControlCache();
+
+    Map<String, CacheHolder> getAuthorizableCache();
+
+    Map<String, CacheHolder> getContentCache();
+
+}
diff --git a/src/main/java/org/sakaiproject/nakamura/api/lite/StorageClientUtils.java b/src/main/java/org/sakaiproject/nakamura/api/lite/StorageClientUtils.java
@@ -35,7 +35,6 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.UnsupportedEncodingException;
-import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
diff --git a/src/main/java/org/sakaiproject/nakamura/lite/CachingManager.java b/src/main/java/org/sakaiproject/nakamura/lite/CachingManager.java
@@ -17,8 +17,8 @@
  */
 package org.sakaiproject.nakamura.lite;
 
+import org.sakaiproject.nakamura.api.lite.CacheHolder;
 import org.sakaiproject.nakamura.api.lite.StorageClientException;
-import org.sakaiproject.nakamura.lite.accesscontrol.CacheHolder;
 import org.sakaiproject.nakamura.lite.storage.StorageClient;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/src/main/java/org/sakaiproject/nakamura/lite/RepositoryImpl.java b/src/main/java/org/sakaiproject/nakamura/lite/RepositoryImpl.java
@@ -31,7 +31,6 @@
 import org.sakaiproject.nakamura.api.lite.accesscontrol.AccessDeniedException;
 import org.sakaiproject.nakamura.api.lite.authorizable.User;
 import org.sakaiproject.nakamura.lite.accesscontrol.AuthenticatorImpl;
-import org.sakaiproject.nakamura.lite.accesscontrol.CacheHolder;
 import org.sakaiproject.nakamura.lite.authorizable.AuthorizableActivator;
 import org.sakaiproject.nakamura.lite.storage.StorageClient;
 import org.sakaiproject.nakamura.lite.storage.StorageClientPool;
@@ -51,7 +50,6 @@ public class RepositoryImpl implements Repository {
     @Reference 
     protected StoreListener storeListener;
 
-    private Map<String, CacheHolder> sharedCache;
 
     public RepositoryImpl() {
     }
@@ -69,7 +67,6 @@ public void activate(Map<String, Object> properties) throws ClientPoolException,
             client.close();
             clientPool.getClient();
         }
-        sharedCache = clientPool.getSharedCache();
     }
 
     @Deactivate
@@ -106,7 +103,7 @@ private Session openSession(String username, String password) throws StorageClie
             if (currentUser == null) {
                 throw new StorageClientException("User " + username + " cant login with password");
             }
-            return new SessionImpl(this, currentUser, client, configuration, sharedCache, storeListener);
+            return new SessionImpl(this, currentUser, client, configuration, clientPool.getStorageCacheManager(), storeListener);
         } catch (ClientPoolException e) {
             clientPool.getClient();
             throw e;
@@ -133,7 +130,7 @@ private Session openSession(String username) throws StorageClientException,
                 throw new StorageClientException("User " + username
                         + " does not exist, cant login administratively as this user");
             }
-            return new SessionImpl(this, currentUser, client, configuration, sharedCache, storeListener);
+            return new SessionImpl(this, currentUser, client, configuration,  clientPool.getStorageCacheManager(), storeListener);
         } catch (ClientPoolException e) {
             clientPool.getClient();
             throw e;
diff --git a/src/main/java/org/sakaiproject/nakamura/lite/SessionImpl.java b/src/main/java/org/sakaiproject/nakamura/lite/SessionImpl.java
@@ -21,20 +21,18 @@
 import org.sakaiproject.nakamura.api.lite.Configuration;
 import org.sakaiproject.nakamura.api.lite.Repository;
 import org.sakaiproject.nakamura.api.lite.Session;
+import org.sakaiproject.nakamura.api.lite.StorageCacheManager;
 import org.sakaiproject.nakamura.api.lite.StorageClientException;
 import org.sakaiproject.nakamura.api.lite.StoreListener;
 import org.sakaiproject.nakamura.api.lite.accesscontrol.AccessDeniedException;
 import org.sakaiproject.nakamura.api.lite.accesscontrol.Authenticator;
 import org.sakaiproject.nakamura.api.lite.authorizable.User;
 import org.sakaiproject.nakamura.lite.accesscontrol.AccessControlManagerImpl;
 import org.sakaiproject.nakamura.lite.accesscontrol.AuthenticatorImpl;
-import org.sakaiproject.nakamura.lite.accesscontrol.CacheHolder;
 import org.sakaiproject.nakamura.lite.authorizable.AuthorizableManagerImpl;
 import org.sakaiproject.nakamura.lite.content.ContentManagerImpl;
 import org.sakaiproject.nakamura.lite.storage.StorageClient;
 
-import java.util.Map;
-
 public class SessionImpl implements Session {
 
     private AccessControlManagerImpl accessControlManager;
@@ -48,17 +46,17 @@ public class SessionImpl implements Session {
     private StoreListener storeListener;
 
     public SessionImpl(Repository repository, User currentUser, StorageClient client,
-            Configuration configuration, Map<String, CacheHolder> sharedCache, StoreListener storeListener)
+            Configuration configuration, StorageCacheManager storageCacheManager, StoreListener storeListener)
             throws ClientPoolException, StorageClientException, AccessDeniedException {
         this.currentUser = currentUser;
         this.repository = repository;
         this.client = client;
         accessControlManager = new AccessControlManagerImpl(client, currentUser, configuration,
-                sharedCache, storeListener);
+                storageCacheManager.getAccessControlCache(), storeListener);
         authorizableManager = new AuthorizableManagerImpl(currentUser, client, configuration,
-                accessControlManager, sharedCache, storeListener);
+                accessControlManager, storageCacheManager.getAuthorizableCache(), storeListener);
 
-        contentManager = new ContentManagerImpl(client, accessControlManager, configuration, sharedCache, storeListener);
+        contentManager = new ContentManagerImpl(client, accessControlManager, configuration, storageCacheManager.getContentCache(), storeListener);
 
         authenticator = new AuthenticatorImpl(client, configuration);
         this.storeListener = storeListener;
diff --git a/src/main/java/org/sakaiproject/nakamura/lite/accesscontrol/AccessControlManagerImpl.java b/src/main/java/org/sakaiproject/nakamura/lite/accesscontrol/AccessControlManagerImpl.java
@@ -20,6 +20,7 @@
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 
+import org.sakaiproject.nakamura.api.lite.CacheHolder;
 import org.sakaiproject.nakamura.api.lite.Configuration;
 import org.sakaiproject.nakamura.api.lite.StorageClientException;
 import org.sakaiproject.nakamura.api.lite.StorageClientUtils;
diff --git a/src/main/java/org/sakaiproject/nakamura/lite/authorizable/AuthorizableManagerImpl.java b/src/main/java/org/sakaiproject/nakamura/lite/authorizable/AuthorizableManagerImpl.java
@@ -22,6 +22,7 @@
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Maps;
 
+import org.sakaiproject.nakamura.api.lite.CacheHolder;
 import org.sakaiproject.nakamura.api.lite.Configuration;
 import org.sakaiproject.nakamura.api.lite.StorageClientException;
 import org.sakaiproject.nakamura.api.lite.StorageClientUtils;
@@ -38,7 +39,6 @@
 import org.sakaiproject.nakamura.api.lite.util.PreemptiveIterator;
 import org.sakaiproject.nakamura.lite.CachingManager;
 import org.sakaiproject.nakamura.lite.accesscontrol.AuthenticatorImpl;
-import org.sakaiproject.nakamura.lite.accesscontrol.CacheHolder;
 import org.sakaiproject.nakamura.lite.storage.StorageClient;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/src/main/java/org/sakaiproject/nakamura/lite/content/ContentManagerImpl.java b/src/main/java/org/sakaiproject/nakamura/lite/content/ContentManagerImpl.java
@@ -49,6 +49,7 @@
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 
+import org.sakaiproject.nakamura.api.lite.CacheHolder;
 import org.sakaiproject.nakamura.api.lite.Configuration;
 import org.sakaiproject.nakamura.api.lite.StorageClientException;
 import org.sakaiproject.nakamura.api.lite.StorageClientUtils;
@@ -60,7 +61,6 @@
 import org.sakaiproject.nakamura.api.lite.content.Content;
 import org.sakaiproject.nakamura.api.lite.content.ContentManager;
 import org.sakaiproject.nakamura.lite.CachingManager;
-import org.sakaiproject.nakamura.lite.accesscontrol.CacheHolder;
 import org.sakaiproject.nakamura.lite.storage.RemoveProperty;
 import org.sakaiproject.nakamura.lite.storage.StorageClient;
 import org.slf4j.Logger;
diff --git a/src/main/java/org/sakaiproject/nakamura/lite/storage/ConcurrentLRUMap.java b/src/main/java/org/sakaiproject/nakamura/lite/storage/ConcurrentLRUMap.java
@@ -106,6 +106,7 @@ public boolean containsValue(Object value) {
 
     public V put(K key, V value) {
         if (delegate.size() > maxSize) {
+            // this is horribly slow for large sets
             List<Holder<V>> l = new ArrayList<Holder<V>>(delegate.values());
             Collections.sort(l, new Comparator<Holder<V>>() {
 
@@ -114,7 +115,7 @@ public int compare(Holder<V> o1, Holder<V> o2) {
                 }
             });
             int i = 0;
-            while (delegate.size() > maxSize && i < l.size()) {
+            while (delegate.size() > ((75*maxSize)/100) && i < l.size()) {
                 delegate.remove(l.get(i++).key);
             }
         }
diff --git a/src/main/java/org/sakaiproject/nakamura/lite/storage/StorageClientPool.java b/src/main/java/org/sakaiproject/nakamura/lite/storage/StorageClientPool.java
@@ -18,9 +18,7 @@
 package org.sakaiproject.nakamura.lite.storage;
 
 import org.sakaiproject.nakamura.api.lite.ClientPoolException;
-import org.sakaiproject.nakamura.lite.accesscontrol.CacheHolder;
-
-import java.util.Map;
+import org.sakaiproject.nakamura.api.lite.StorageCacheManager;
 
 public interface StorageClientPool {
 
@@ -30,6 +28,6 @@ public interface StorageClientPool {
      */
     StorageClient getClient() throws ClientPoolException;
 
-    Map<String, CacheHolder> getSharedCache();
+    StorageCacheManager getStorageCacheManager();
 
 }
diff --git a/src/main/java/org/sakaiproject/nakamura/lite/storage/cassandra/CassandraClientPool.java b/src/main/java/org/sakaiproject/nakamura/lite/storage/cassandra/CassandraClientPool.java
@@ -24,14 +24,18 @@
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Deactivate;
 import org.apache.felix.scr.annotations.Property;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.apache.felix.scr.annotations.ReferencePolicy;
 import org.apache.felix.scr.annotations.Service;
 import org.apache.thrift.TException;
 import org.apache.thrift.protocol.TBinaryProtocol;
 import org.apache.thrift.protocol.TProtocol;
 import org.apache.thrift.transport.TSocket;
+import org.sakaiproject.nakamura.api.lite.CacheHolder;
+import org.sakaiproject.nakamura.api.lite.StorageCacheManager;
 import org.sakaiproject.nakamura.api.lite.StorageClientException;
 import org.sakaiproject.nakamura.api.lite.StorageClientUtils;
-import org.sakaiproject.nakamura.lite.accesscontrol.CacheHolder;
 import org.sakaiproject.nakamura.lite.storage.AbstractClientConnectionPool;
 import org.sakaiproject.nakamura.lite.storage.ConcurrentLRUMap;
 import org.sakaiproject.nakamura.lite.storage.StorageClientPool;
@@ -47,6 +51,10 @@ public class CassandraClientPool extends AbstractClientConnectionPool {
     private static final Logger LOGGER = LoggerFactory.getLogger(CassandraClientPool.class);
     @Property(value = { "localhost:9610" })
     private static final String CONNECTION_POOL = "conection-pool";
+    
+    @Reference(cardinality=ReferenceCardinality.OPTIONAL_UNARY, policy=ReferencePolicy.DYNAMIC)
+    private StorageCacheManager storageManagerCache;
+
 
     public static class ClientConnectionPoolFactory extends BasePoolableObjectFactory {
 
@@ -156,6 +164,7 @@ public boolean validateObject(Object obj) {
     private String[] connections;
     private Map<String, Object> properties;
     private Map<String, CacheHolder> sharedCache;
+    private StorageCacheManager defaultStorageManagerCache;
 
     public CassandraClientPool() {
     }
@@ -168,6 +177,20 @@ public void activate(Map<String, Object> properties) throws ClassNotFoundExcepti
         super.activate(properties);
         // this should come from the memory service ultimately.
         sharedCache = new ConcurrentLRUMap<String, CacheHolder>(10000);
+        defaultStorageManagerCache = new StorageCacheManager() {
+            
+            public Map<String, CacheHolder> getContentCache() {
+                return sharedCache;
+            }
+            
+            public Map<String, CacheHolder> getAuthorizableCache() {
+                return sharedCache;
+            }
+            
+            public Map<String, CacheHolder> getAccessControlCache() {
+                return sharedCache;
+            }
+        };
 
     }
 
@@ -181,7 +204,13 @@ protected PoolableObjectFactory getConnectionPoolFactory() {
         return new ClientConnectionPoolFactory(this, connections, properties);
     }
 
-    public Map<String, CacheHolder> getSharedCache() {
-        return sharedCache;
+    public StorageCacheManager getStorageCacheManager() {
+        if ( storageManagerCache != null ) {
+            if ( sharedCache.size() > 0 ) {
+                sharedCache.clear(); // dump any memory consumed by the default cache.
+            }
+            return storageManagerCache ;
+        }
+        return defaultStorageManagerCache;
     }
 }
diff --git a/src/main/java/org/sakaiproject/nakamura/lite/storage/jdbc/JDBCStorageClientPool.java b/src/main/java/org/sakaiproject/nakamura/lite/storage/jdbc/JDBCStorageClientPool.java
@@ -28,11 +28,15 @@
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Deactivate;
 import org.apache.felix.scr.annotations.Property;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.apache.felix.scr.annotations.ReferencePolicy;
 import org.apache.felix.scr.annotations.Service;
+import org.sakaiproject.nakamura.api.lite.CacheHolder;
 import org.sakaiproject.nakamura.api.lite.ClientPoolException;
+import org.sakaiproject.nakamura.api.lite.StorageCacheManager;
 import org.sakaiproject.nakamura.api.lite.StorageClientException;
 import org.sakaiproject.nakamura.api.lite.StorageClientUtils;
-import org.sakaiproject.nakamura.lite.accesscontrol.CacheHolder;
 import org.sakaiproject.nakamura.lite.storage.AbstractClientConnectionPool;
 import org.sakaiproject.nakamura.lite.storage.ConcurrentLRUMap;
 import org.sakaiproject.nakamura.lite.storage.StorageClientPool;
@@ -65,6 +69,10 @@ public class JDBCStorageClientPool extends AbstractClientConnectionPool {
     private static final String USERNAME = "username";
     @Property(value = { "" })
     private static final String PASSWORD = "password";
+ 
+    @Reference(cardinality=ReferenceCardinality.OPTIONAL_UNARY, policy=ReferencePolicy.DYNAMIC)
+    private StorageCacheManager storageManagerCache;
+
 
     private static final String BASESQLPATH = "org/sakaiproject/nakamura/lite/storage/jdbc/config/client";
 
@@ -110,8 +118,6 @@ public boolean validateObject(Object obj) {
     private Map<String, Object> sqlConfig;
     private Object sqlConfigLock = new Object();
 
-    private Map<String, CacheHolder> sharedCache;
-
     private Properties connectionProperties;
 
     private String username;
@@ -124,6 +130,11 @@ public boolean validateObject(Object obj) {
 
     private Timer timer;
 
+    private StorageCacheManager defaultStorageManagerCache;
+
+    private Map<String, CacheHolder> sharedCache;
+
+
     @Activate
     @SuppressWarnings(value={"NP_CLOSING_NULL"},justification="Invalid report, if this was the case then nothing would work")
     public void activate(Map<String, Object> properties) throws ClassNotFoundException {
@@ -135,6 +146,21 @@ public void activate(Map<String, Object> properties) throws ClassNotFoundExcepti
         timer.schedule(connectionManager, 30000L, 30000L);
 
         sharedCache = new ConcurrentLRUMap<String, CacheHolder>(10000);
+        // this is a default cache used where none has been provided.
+        defaultStorageManagerCache = new StorageCacheManager() {
+            
+            public Map<String, CacheHolder> getContentCache() {
+                return sharedCache;
+            }
+            
+            public Map<String, CacheHolder> getAuthorizableCache() {
+                return sharedCache;
+            }
+            
+            public Map<String, CacheHolder> getAccessControlCache() {
+                return sharedCache;
+            }
+        };
 
         String jdbcDriver = (String) properties.get(JDBC_DRIVER);
         Class<?> clazz = Class.forName(jdbcDriver);
@@ -278,8 +304,14 @@ protected PoolableObjectFactory getConnectionPoolFactory() {
         return new JCBCStorageClientConnection();
     }
 
-    public Map<String, CacheHolder> getSharedCache() {
-        return sharedCache;
+    public StorageCacheManager getStorageCacheManager() {
+        if ( storageManagerCache != null ) {
+            if ( sharedCache.size() > 0 ) {
+                sharedCache.clear(); // dump any memory consumed by the default cache.
+            }
+            return storageManagerCache ;
+        }
+        return defaultStorageManagerCache;
     }
 
     public Connection getConnection() throws SQLException {
diff --git a/src/main/java/org/sakaiproject/nakamura/lite/storage/mem/MemoryStorageClientPool.java b/src/main/java/org/sakaiproject/nakamura/lite/storage/mem/MemoryStorageClientPool.java
diff --git a/src/test/java/org/sakaiproject/nakamura/lite/authorizable/AbstractAuthorizableManagerImplTest.java b/src/test/java/org/sakaiproject/nakamura/lite/authorizable/AbstractAuthorizableManagerImplTest.java
diff --git a/src/test/java/org/sakaiproject/nakamura/lite/content/AbstractContentManagerTest.java b/src/test/java/org/sakaiproject/nakamura/lite/content/AbstractContentManagerTest.java
diff --git a/src/test/java/org/sakaiproject/nakamura/lite/soak/authorizable/CreateUsersAndGroupsClient.java b/src/test/java/org/sakaiproject/nakamura/lite/soak/authorizable/CreateUsersAndGroupsClient.java
diff --git a/src/test/java/org/sakaiproject/nakamura/lite/soak/authorizable/CreateUsersAndGroupsWithMembersClient.java b/src/test/java/org/sakaiproject/nakamura/lite/soak/authorizable/CreateUsersAndGroupsWithMembersClient.java

Original file line number	Diff line number	Diff line change
`@@ -106,6 +106,7 @@ public boolean containsValue(Object value) {`
`106`	`106`
`107`	`107`	`public V put(K key, V value) {`
`108`	`108`	`if (delegate.size() > maxSize) {`
	`109`	`+ // this is horribly slow for large sets`
`109`	`110`	`List<Holder<V>> l = new ArrayList<Holder<V>>(delegate.values());`
`110`	`111`	`Collections.sort(l, new Comparator<Holder<V>>() {`
`111`	`112`
`@@ -114,7 +115,7 @@ public int compare(Holder<V> o1, Holder<V> o2) {`
`114`	`115`	`}`
`115`	`116`	`});`
`116`	`117`	`int i = 0;`
`117`		`- while (delegate.size() > maxSize && i < l.size()) {`
	`118`	`+ while (delegate.size() > ((75*maxSize)/100) && i < l.size()) {`
`118`	`119`	`delegate.remove(l.get(i++).key);`
`119`	`120`	`}`
`120`	`121`	`}`