NIFI-5673 Set up property/assembly for new auto-load directory

- Set up NarAutoLoader to watch directory for new files
- Move NarAutoLoader to JettyServer since it will need access to ExtensionManager
- Created NarLoader to shared between NarAutoLoader and the framework
- Created nifi-framework-nar-loading-utils so we can use nifi-documentation to call DocGenerator
- Add additional bundles to overall map in NarClassLoaders as they are loaded
- Added handling of skipped NARs to include them in next iteration
- Added check of last modified timestamp on NARs
- Refactored JettyServer so we can load additional web contexts while the application is running
- Setting up unit tests
- Remove static use of ExtensionManager
- Adding unit tests for NarLoader
- Extracting interface for ExtensionManager and splitting discovery into it's own interface

This closes #3119.

Signed-off-by: Mark Payne <markap14@hotmail.com>

nifi-assembly/src/main/assembly/common.xml

@@ -91,6 +91,15 @@
             </unpackOptions>
         </dependencySet>
     </dependencySets>
+    <fileSets>
+        <fileSet>
+            <directory>.</directory>
+            <outputDirectory>extensions</outputDirectory>
+            <excludes>
+                <exclude>*/**</exclude>
+            </excludes>
+        </fileSet>
+    </fileSets>
     <files>
         <file>
             <source>./README.md</source>

nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java

@@ -62,6 +62,7 @@ public abstract class NiFiProperties {
     public static final String FLOW_CONTROLLER_GRACEFUL_SHUTDOWN_PERIOD = "nifi.flowcontroller.graceful.shutdown.period";
     public static final String NAR_LIBRARY_DIRECTORY = "nifi.nar.library.directory";
     public static final String NAR_LIBRARY_DIRECTORY_PREFIX = "nifi.nar.library.directory.";
+    public static final String NAR_LIBRARY_AUTOLOAD_DIRECTORY = "nifi.nar.library.autoload.directory";
     public static final String NAR_WORKING_DIRECTORY = "nifi.nar.working.directory";
     public static final String COMPONENT_DOCS_DIRECTORY = "nifi.documentation.working.directory";
     public static final String SENSITIVE_PROPS_KEY = "nifi.sensitive.props.key";
@@ -247,6 +248,7 @@ public abstract class NiFiProperties {
     public static final String DEFAULT_NAR_WORKING_DIR = "./work/nar";
     public static final String DEFAULT_COMPONENT_DOCS_DIRECTORY = "./work/docs/components";
     public static final String DEFAULT_NAR_LIBRARY_DIR = "./lib";
+    public static final String DEFAULT_NAR_LIBRARY_AUTOLOAD_DIR = "./extensions";
     public static final String DEFAULT_FLOWFILE_REPO_PARTITIONS = "256";
     public static final String DEFAULT_FLOWFILE_CHECKPOINT_INTERVAL = "2 min";
     public static final int DEFAULT_MAX_FLOWFILES_PER_CLAIM = 100;
@@ -650,7 +652,8 @@ public List<Path> getNarLibraryDirectories() {
         for (String propertyName : getPropertyKeys()) {
             // determine if the property is a nar library path
             if (StringUtils.startsWith(propertyName, NAR_LIBRARY_DIRECTORY_PREFIX)
-                    || NAR_LIBRARY_DIRECTORY.equals(propertyName)) {
+                    || NAR_LIBRARY_DIRECTORY.equals(propertyName)
+                    || NAR_LIBRARY_AUTOLOAD_DIRECTORY.equals(propertyName)) {
                 // attempt to resolve the path specified
                 String narLib = getProperty(propertyName);
                 if (!StringUtils.isBlank(narLib)) {
@@ -666,6 +669,10 @@ public List<Path> getNarLibraryDirectories() {
         return narLibraryPaths;
     }
 
+    public File getNarAutoLoadDirectory() {
+        return new File(getProperty(NAR_LIBRARY_AUTOLOAD_DIRECTORY, DEFAULT_NAR_LIBRARY_AUTOLOAD_DIR));
+    }
+
     // getters for ui properties //
 
     /**

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizerFactoryBean.java

@@ -89,6 +89,7 @@ private static JAXBContext initializeJaxbContext() {
 
     private Authorizer authorizer;
     private NiFiProperties properties;
+    private ExtensionManager extensionManager;
     private final Map<String, UserGroupProvider> userGroupProviders = new HashMap<>();
     private final Map<String, AccessPolicyProvider> accessPolicyProviders = new HashMap<>();
     private final Map<String, Authorizer> authorizers = new HashMap<>();
@@ -208,7 +209,7 @@ private Authorizers loadAuthorizersConfiguration() throws Exception {
 
     private UserGroupProvider createUserGroupProvider(final String identifier, final String userGroupProviderClassName) throws Exception {
         // get the classloader for the specified user group provider
-        final List<Bundle> userGroupProviderBundles = ExtensionManager.getBundles(userGroupProviderClassName);
+        final List<Bundle> userGroupProviderBundles = extensionManager.getBundles(userGroupProviderClassName);
 
         if (userGroupProviderBundles.size() == 0) {
             throw new Exception(String.format("The specified user group provider class '%s' is not known to this nifi.", userGroupProviderClassName));
@@ -256,7 +257,7 @@ private UserGroupProvider createUserGroupProvider(final String identifier, final
 
     private AccessPolicyProvider createAccessPolicyProvider(final String identifier, final String accessPolicyProviderClassName) throws Exception {
         // get the classloader for the specified access policy provider
-        final List<Bundle> accessPolicyProviderBundles = ExtensionManager.getBundles(accessPolicyProviderClassName);
+        final List<Bundle> accessPolicyProviderBundles = extensionManager.getBundles(accessPolicyProviderClassName);
 
         if (accessPolicyProviderBundles.size() == 0) {
             throw new Exception(String.format("The specified access policy provider class '%s' is not known to this nifi.", accessPolicyProviderClassName));
@@ -304,7 +305,7 @@ private AccessPolicyProvider createAccessPolicyProvider(final String identifier,
 
     private Authorizer createAuthorizer(final String identifier, final String authorizerClassName, final String classpathResources) throws Exception {
         // get the classloader for the specified authorizer
-        final List<Bundle> authorizerBundles = ExtensionManager.getBundles(authorizerClassName);
+        final List<Bundle> authorizerBundles = extensionManager.getBundles(authorizerClassName);
 
         if (authorizerBundles.size() == 0) {
             throw new Exception(String.format("The specified authorizer class '%s' is not known to this nifi.", authorizerClassName));
@@ -537,4 +538,8 @@ public void setProperties(NiFiProperties properties) {
         this.properties = properties;
     }
 
+    public void setExtensionManager(ExtensionManager extensionManager) {
+        this.extensionManager = extensionManager;
+    }
+
 }

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/resources/nifi-authorizer-context.xml

@@ -21,6 +21,7 @@
     <!-- user/entity authorizer -->
     <bean id="authorizer" class="org.apache.nifi.authorization.AuthorizerFactoryBean">
         <property name="properties" ref="nifiProperties"/>
+        <property name="extensionManager" ref="extensionManager" />
     </bean>
 
 </beans>

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/main/java/org/apache/nifi/documentation/DocGenerator.java

@@ -56,14 +56,14 @@ public class DocGenerator {
      * @param properties to lookup nifi properties
      * @param extensionMapping extension mapping
      */
-    public static void generate(final NiFiProperties properties, final ExtensionMapping extensionMapping) {
+    public static void generate(final NiFiProperties properties, final ExtensionManager extensionManager, final ExtensionMapping extensionMapping) {
         final File explodedNiFiDocsDir = properties.getComponentDocumentationWorkingDirectory();
 
         logger.debug("Generating documentation for: " + extensionMapping.size() + " components in: " + explodedNiFiDocsDir);
 
-        documentConfigurableComponent(ExtensionManager.getExtensions(Processor.class), explodedNiFiDocsDir);
-        documentConfigurableComponent(ExtensionManager.getExtensions(ControllerService.class), explodedNiFiDocsDir);
-        documentConfigurableComponent(ExtensionManager.getExtensions(ReportingTask.class), explodedNiFiDocsDir);
+        documentConfigurableComponent(extensionManager.getExtensions(Processor.class), explodedNiFiDocsDir, extensionManager);
+        documentConfigurableComponent(extensionManager.getExtensions(ControllerService.class), explodedNiFiDocsDir, extensionManager);
+        documentConfigurableComponent(extensionManager.getExtensions(ReportingTask.class), explodedNiFiDocsDir, extensionManager);
     }
 
     /**
@@ -72,12 +72,12 @@ public static void generate(final NiFiProperties properties, final ExtensionMapp
      * @param extensionClasses types of a configurable component
      * @param explodedNiFiDocsDir base directory of component documentation
      */
-    private static void documentConfigurableComponent(final Set<Class> extensionClasses, final File explodedNiFiDocsDir) {
+    public static void documentConfigurableComponent(final Set<Class> extensionClasses, final File explodedNiFiDocsDir, final ExtensionManager extensionManager) {
         for (final Class<?> extensionClass : extensionClasses) {
             if (ConfigurableComponent.class.isAssignableFrom(extensionClass)) {
                 final String extensionClassName = extensionClass.getCanonicalName();
 
-                final Bundle bundle = ExtensionManager.getBundle(extensionClass.getClassLoader());
+                final Bundle bundle = extensionManager.getBundle(extensionClass.getClassLoader());
                 if (bundle == null) {
                     logger.warn("No coordinate found for {}, skipping...", new Object[] {extensionClassName});
                     continue;
@@ -91,7 +91,7 @@ private static void documentConfigurableComponent(final Set<Class> extensionClas
                 final Class<? extends ConfigurableComponent> componentClass = extensionClass.asSubclass(ConfigurableComponent.class);
                 try {
                     logger.debug("Documenting: " + componentClass);
-                    document(componentDirectory, componentClass, coordinate);
+                    document(extensionManager, componentDirectory, componentClass, coordinate);
                 } catch (Exception e) {
                     logger.warn("Unable to document: " + componentClass, e);
                 }
@@ -111,14 +111,17 @@ private static void documentConfigurableComponent(final Set<Class> extensionClas
      * @throws IOException ioe
      * @throws InitializationException ie
      */
-    private static void document(final File componentDocsDir, final Class<? extends ConfigurableComponent> componentClass, final BundleCoordinate bundleCoordinate)
+    private static void document(final ExtensionManager extensionManager,
+                                 final File componentDocsDir,
+                                 final Class<? extends ConfigurableComponent> componentClass,
+                                 final BundleCoordinate bundleCoordinate)
             throws InstantiationException, IllegalAccessException, IOException, InitializationException {
 
         // use temp components from ExtensionManager which should always be populated before doc generation
         final String classType = componentClass.getCanonicalName();
-        final ConfigurableComponent component = ExtensionManager.getTempComponent(classType, bundleCoordinate);
+        final ConfigurableComponent component = extensionManager.getTempComponent(classType, bundleCoordinate);
 
-        final DocumentationWriter writer = getDocumentWriter(componentClass);
+        final DocumentationWriter writer = getDocumentWriter(extensionManager, componentClass);
 
         final File baseDocumentationFile = new File(componentDocsDir, "index.html");
         if (baseDocumentationFile.exists()) {
@@ -138,13 +141,14 @@ private static void document(final File componentDocsDir, final Class<? extends
      * @return a DocumentationWriter capable of generating documentation for
      * that specific type of class
      */
-    private static DocumentationWriter getDocumentWriter(final Class<? extends ConfigurableComponent> componentClass) {
+    private static DocumentationWriter getDocumentWriter(final ExtensionManager extensionManager,
+                                                         final Class<? extends ConfigurableComponent> componentClass) {
         if (Processor.class.isAssignableFrom(componentClass)) {
-            return new HtmlProcessorDocumentationWriter();
+            return new HtmlProcessorDocumentationWriter(extensionManager);
         } else if (ControllerService.class.isAssignableFrom(componentClass)) {
-            return new HtmlDocumentationWriter();
+            return new HtmlDocumentationWriter(extensionManager);
         } else if (ReportingTask.class.isAssignableFrom(componentClass)) {
-            return new HtmlDocumentationWriter();
+            return new HtmlDocumentationWriter(extensionManager);
         }
 
         return null;

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/main/java/org/apache/nifi/documentation/html/HtmlDocumentationWriter.java

@@ -67,6 +67,12 @@ public class HtmlDocumentationWriter implements DocumentationWriter {
      */
     public static final String ADDITIONAL_DETAILS_HTML = "additionalDetails.html";
 
+    private final ExtensionManager extensionManager;
+
+    public HtmlDocumentationWriter(final ExtensionManager extensionManager) {
+        this.extensionManager = extensionManager;
+    }
+
     @Override
     public void write(final ConfigurableComponent configurableComponent, final OutputStream streamToWriteTo,
             final boolean includesAdditionalDocumentation) throws IOException {
@@ -848,7 +854,7 @@ private List<Class<? extends ControllerService>> lookupControllerServiceImpls(
         final List<Class<? extends ControllerService>> implementations = new ArrayList<>();
 
         // first get all ControllerService implementations
-        final Set<Class> controllerServices = ExtensionManager.getExtensions(ControllerService.class);
+        final Set<Class> controllerServices = extensionManager.getExtensions(ControllerService.class);
 
         // then iterate over all controller services looking for any that is a child of the parent
         // ControllerService API that was passed in as a parameter
@@ -891,7 +897,7 @@ protected void iterateAndLinkComponents(final XMLStreamWriter xmlStreamWriter, f
         int index = 0;
         for (final Class<? extends ConfigurableComponent> linkedComponent : linkedComponents ) {
             final String linkedComponentName = linkedComponent.getName();
-            final List<Bundle> linkedComponentBundles = ExtensionManager.getBundles(linkedComponentName);
+            final List<Bundle> linkedComponentBundles = extensionManager.getBundles(linkedComponentName);
             if (linkedComponentBundles != null && linkedComponentBundles.size() > 0) {
                 final Bundle firstLinkedComponentBundle = linkedComponentBundles.get(0);
                 final BundleCoordinate coordinate = firstLinkedComponentBundle.getBundleDetails().getCoordinate();
@@ -927,7 +933,7 @@ protected void iterateAndLinkComponents(final XMLStreamWriter xmlStreamWriter, f
                     }
                 }
 
-                final List<Bundle> linkedComponentBundles = ExtensionManager.getBundles(className);
+                final List<Bundle> linkedComponentBundles = extensionManager.getBundles(className);
 
                 if (linkedComponentBundles != null && linkedComponentBundles.size() > 0) {
                     final Bundle firstBundle = linkedComponentBundles.get(0);

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/main/java/org/apache/nifi/documentation/html/HtmlProcessorDocumentationWriter.java

@@ -29,6 +29,7 @@
 import org.apache.nifi.annotation.behavior.WritesAttribute;
 import org.apache.nifi.annotation.behavior.WritesAttributes;
 import org.apache.nifi.components.ConfigurableComponent;
+import org.apache.nifi.nar.ExtensionManager;
 import org.apache.nifi.processor.Processor;
 import org.apache.nifi.processor.Relationship;
 
@@ -40,6 +41,10 @@
  */
 public class HtmlProcessorDocumentationWriter extends HtmlDocumentationWriter {
 
+    public HtmlProcessorDocumentationWriter(ExtensionManager extensionManager) {
+        super(extensionManager);
+    }
+
     @Override
     protected void writeAdditionalBodyInfo(final ConfigurableComponent configurableComponent,
             final XMLStreamWriter xmlStreamWriter) throws XMLStreamException {

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/test/java/org/apache/nifi/documentation/DocGeneratorTest.java

@@ -16,27 +16,29 @@
  */
 package org.apache.nifi.documentation;
 
-import java.io.BufferedInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.Charset;
-import java.util.Properties;
-import java.util.Set;
 import org.apache.commons.io.FileUtils;
 import org.apache.nifi.bundle.Bundle;
 import org.apache.nifi.bundle.BundleCoordinate;
-import org.apache.nifi.nar.ExtensionManager;
+import org.apache.nifi.nar.ExtensionDiscoveringManager;
 import org.apache.nifi.nar.ExtensionMapping;
-import org.apache.nifi.nar.NarClassLoaders;
+import org.apache.nifi.nar.NarClassLoadersHolder;
 import org.apache.nifi.nar.NarUnpacker;
+import org.apache.nifi.nar.StandardExtensionDiscoveringManager;
 import org.apache.nifi.nar.SystemBundle;
 import org.apache.nifi.util.NiFiProperties;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.Charset;
+import java.util.Properties;
+import java.util.Set;
+
 public class DocGeneratorTest {
 
     @Test
@@ -51,11 +53,12 @@ public void testProcessorLoadsNarResources() throws IOException, ClassNotFoundEx
         final Bundle systemBundle = SystemBundle.create(properties);
         final ExtensionMapping mapping = NarUnpacker.unpackNars(properties, systemBundle);
 
-        NarClassLoaders.getInstance().init(properties.getFrameworkWorkingDirectory(), properties.getExtensionsWorkingDirectory());
+        NarClassLoadersHolder.getInstance().init(properties.getFrameworkWorkingDirectory(), properties.getExtensionsWorkingDirectory());
 
-        ExtensionManager.discoverExtensions(systemBundle, NarClassLoaders.getInstance().getBundles());
+        final ExtensionDiscoveringManager extensionManager = new StandardExtensionDiscoveringManager();
+        extensionManager.discoverExtensions(systemBundle, NarClassLoadersHolder.getInstance().getBundles());
 
-        DocGenerator.generate(properties, mapping);
+        DocGenerator.generate(properties, extensionManager, mapping);
 
         final String extensionClassName = "org.apache.nifi.processors.WriteResourceToStream";
         final BundleCoordinate coordinate = mapping.getProcessorNames().get(extensionClassName).stream().findFirst().get();