SDC-9394: Transition Amazon Stage library off commonlib

This patch removes the dependency on commonlib from AWS stage library,
effectively removing unnecessary dependencies and cleaning up the
classpath.

I had to move few files around  to enable that, but luckily the
dependencies weren't that crazy.

Change-Id: I978d9177f3e45a3d9bd5eb8097c5d09ca2eecd1c
Reviewed-on: https://review.streamsets.net/15288
Tested-by: StreamSets CI <streamsets-ci-spam@streamsets.com>
Reviewed-by: Ji Sun Kim <jisun@streamsets.com>

aws-lib/pom.xml

@@ -87,13 +87,11 @@
     <!-- Compile Dependencies -->
     <dependency>
       <groupId>com.streamsets</groupId>
-      <artifactId>streamsets-datacollector-common</artifactId>
-      <scope>compile</scope>
+      <artifactId>streamsets-datacollector-stagesupport</artifactId>
     </dependency>
     <dependency>
       <groupId>com.streamsets</groupId>
-      <artifactId>streamsets-datacollector-commonlib</artifactId>
-      <scope>compile</scope>
+      <artifactId>streamsets-datacollector-guavasupport</artifactId>
     </dependency>
     <dependency>
       <groupId>com.amazonaws</groupId>

aws-lib/src/main/java/com/streamsets/pipeline/stage/destination/s3/S3TargetConfigBean.java

@@ -20,7 +20,6 @@
 import com.streamsets.pipeline.api.Stage;
 import com.streamsets.pipeline.api.ValueChooserModel;
 import com.streamsets.pipeline.api.service.dataformats.DataFormatGeneratorService;
-import com.streamsets.pipeline.config.DataFormat;
 import com.streamsets.pipeline.config.TimeZoneChooserValues;
 import com.streamsets.pipeline.lib.el.RecordEL;
 import com.streamsets.pipeline.lib.el.TimeEL;

aws-lib/src/main/java/com/streamsets/pipeline/stage/origin/s3/AmazonS3Source.java

@@ -34,7 +34,6 @@
 import com.streamsets.pipeline.stage.common.DefaultErrorRecordHandler;
 import com.streamsets.pipeline.stage.common.ErrorRecordHandler;
 import com.streamsets.pipeline.stage.common.HeaderAttributeConstants;
-import org.apache.commons.io.IOUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -70,7 +69,14 @@ protected void initChild(List<ConfigIssue> issues) {
 
   @Override
   public void destroy() {
-    IOUtils.closeQuietly(parser);
+    try {
+      if(parser != null) {
+        parser.close();
+      }
+    } catch (IOException e) {
+      // Quiet close
+    }
+
     super.destroy();
   }
 

aws-lib/src/main/java/com/streamsets/pipeline/stage/origin/s3/AmazonS3SourceUpgrader.java

@@ -20,7 +20,6 @@
 import com.streamsets.pipeline.api.StageUpgrader;
 import com.streamsets.pipeline.api.impl.Utils;
 import com.streamsets.pipeline.api.service.dataformats.DataFormatParserService;
-import com.streamsets.pipeline.config.Compression;
 import com.streamsets.pipeline.config.upgrade.DataFormatUpgradeHelper;
 import com.streamsets.pipeline.stage.lib.aws.AWSUtil;
 
@@ -71,7 +70,7 @@ private static void upgradeV1ToV2(List<Config> configs) {
     configs.add(new Config(S3ConfigBean.S3_CONFIG_BEAN_PREFIX + "advancedConfig.proxyPort", 0));
     configs.add(new Config(S3ConfigBean.S3_CONFIG_BEAN_PREFIX + "advancedConfig.proxyUser", ""));
     configs.add(new Config(S3ConfigBean.S3_CONFIG_BEAN_PREFIX + "advancedConfig.proxyPassword", ""));
-    configs.add(new Config(S3ConfigBean.S3_DATA_FORMAT_CONFIG_PREFIX + "compression", Compression.NONE));
+    configs.add(new Config(S3ConfigBean.S3_DATA_FORMAT_CONFIG_PREFIX + "compression", "NONE"));
     configs.add(new Config(S3ConfigBean.S3_DATA_FORMAT_CONFIG_PREFIX + "filePatternInArchive", "*"));
     configs.add(new Config(S3ConfigBean.S3_DATA_FORMAT_CONFIG_PREFIX + "csvSkipStartLines", 0));
   }

aws-lib/src/main/java/com/streamsets/pipeline/stage/origin/s3/S3ConfigBean.java

@@ -19,24 +19,12 @@
 import com.streamsets.pipeline.api.ConfigDef;
 import com.streamsets.pipeline.api.ConfigDefBean;
 import com.streamsets.pipeline.api.Stage;
-import com.streamsets.pipeline.api.ValueChooserModel;
 import com.streamsets.pipeline.common.InterfaceAudience;
 import com.streamsets.pipeline.common.InterfaceStability;
-import com.streamsets.pipeline.config.DataFormat;
 import com.streamsets.pipeline.config.PostProcessingOptions;
 import com.streamsets.pipeline.stage.lib.aws.AWSUtil;
 import com.streamsets.pipeline.stage.lib.aws.ProxyConfig;
 import com.streamsets.pipeline.stage.origin.lib.BasicConfig;
-import com.streamsets.pipeline.stage.origin.lib.DataParserFormatConfig;
-import com.streamsets.pipeline.stage.origin.s3.DataFormatChooserValues;
-import com.streamsets.pipeline.stage.origin.s3.Errors;
-import com.streamsets.pipeline.stage.origin.s3.Groups;
-import com.streamsets.pipeline.stage.origin.s3.S3ArchivingOption;
-import com.streamsets.pipeline.stage.origin.s3.S3ConnectionSourceConfig;
-import com.streamsets.pipeline.stage.origin.s3.S3ErrorConfig;
-import com.streamsets.pipeline.stage.origin.s3.S3FileConfig;
-import com.streamsets.pipeline.stage.origin.s3.S3PostProcessingConfig;
-import com.streamsets.pipeline.stage.origin.s3.S3SSEConfigBean;
 
 import java.util.List;
 

aws-lib/src/test/java/com/streamsets/pipeline/stage/destination/s3/TestAmazonS3Target.java

@@ -21,35 +21,26 @@
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.S3Object;
-import com.amazonaws.services.s3.model.S3ObjectInputStream;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
 import com.streamsets.pipeline.api.EventRecord;
 import com.streamsets.pipeline.api.OnRecordError;
 import com.streamsets.pipeline.api.Record;
 import com.streamsets.pipeline.api.Stage;
 import com.streamsets.pipeline.api.service.dataformats.DataFormatGeneratorService;
-import com.streamsets.pipeline.config.CsvHeader;
-import com.streamsets.pipeline.config.CsvMode;
-import com.streamsets.pipeline.config.DataFormat;
-import com.streamsets.pipeline.config.JsonMode;
 import com.streamsets.pipeline.sdk.RecordCreator;
 import com.streamsets.pipeline.sdk.TargetRunner;
 import com.streamsets.pipeline.sdk.service.SdkJsonDataFormatGeneratorService;
 import com.streamsets.pipeline.stage.common.FakeS3;
 import com.streamsets.pipeline.stage.common.TestUtil;
-import com.streamsets.pipeline.stage.destination.lib.DataGeneratorFormatConfig;
 import com.streamsets.pipeline.stage.lib.aws.AWSConfig;
 import com.streamsets.pipeline.stage.lib.aws.AWSRegions;
 import com.streamsets.pipeline.stage.lib.aws.ProxyConfig;
 import com.streamsets.pipeline.stage.lib.aws.TransferManagerConfig;
-import org.apache.commons.io.IOUtils;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.Assume;
 import org.junit.BeforeClass;
 import org.junit.Test;
-import org.mockito.internal.util.reflection.Whitebox;
 
 import java.io.File;
 import java.io.IOException;
@@ -58,7 +49,6 @@
 import java.util.UUID;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
-import java.util.zip.GZIPInputStream;
 
 public class TestAmazonS3Target {
 

aws-lib/src/test/java/com/streamsets/pipeline/stage/destination/s3/TestAmazonS3TargetForWholeFile.java

@@ -34,8 +34,6 @@
 import com.streamsets.pipeline.api.service.dataformats.DataFormatGeneratorService;
 import com.streamsets.pipeline.api.service.dataformats.WholeFileChecksumAlgorithm;
 import com.streamsets.pipeline.config.ChecksumAlgorithm;
-import com.streamsets.pipeline.config.DataFormat;
-import com.streamsets.pipeline.config.WholeFileExistsAction;
 import com.streamsets.pipeline.lib.hashing.HashingUtil;
 import com.streamsets.pipeline.lib.io.fileref.FileRefUtil;
 import com.streamsets.pipeline.lib.io.fileref.LocalFileRef;
@@ -44,7 +42,6 @@
 import com.streamsets.pipeline.sdk.service.SdkWholeFileDataFormatGeneratorService;
 import com.streamsets.pipeline.stage.common.AmazonS3TestSuite;
 import com.streamsets.pipeline.stage.common.TestUtil;
-import com.streamsets.pipeline.stage.destination.lib.DataGeneratorFormatConfig;
 import com.streamsets.pipeline.stage.lib.aws.AWSConfig;
 import com.streamsets.pipeline.stage.lib.aws.AWSRegions;
 import com.streamsets.pipeline.stage.lib.aws.ProxyConfig;

aws-lib/src/test/java/com/streamsets/pipeline/stage/origin/s3/TestAmazonS3Source.java

@@ -27,14 +27,10 @@
 import com.streamsets.pipeline.api.BatchMaker;
 import com.streamsets.pipeline.api.EventRecord;
 import com.streamsets.pipeline.api.Field;
-import com.streamsets.pipeline.api.OnRecordError;
 import com.streamsets.pipeline.api.Record;
 import com.streamsets.pipeline.api.Stage;
 import com.streamsets.pipeline.api.StageException;
 import com.streamsets.pipeline.api.service.dataformats.DataFormatParserService;
-import com.streamsets.pipeline.config.CsvHeader;
-import com.streamsets.pipeline.config.DataFormat;
-import com.streamsets.pipeline.config.LogMode;
 import com.streamsets.pipeline.config.PostProcessingOptions;
 import com.streamsets.pipeline.lib.io.fileref.FileRefUtil;
 import com.streamsets.pipeline.sdk.SourceRunner;
@@ -49,7 +45,6 @@
 import com.streamsets.pipeline.stage.lib.aws.AWSUtil;
 import com.streamsets.pipeline.stage.lib.aws.ProxyConfig;
 import com.streamsets.pipeline.stage.origin.lib.BasicConfig;
-import com.streamsets.pipeline.stage.origin.lib.DataParserFormatConfig;
 import org.apache.commons.lang3.tuple.Pair;
 import org.junit.AfterClass;
 import org.junit.Assert;
@@ -183,54 +178,60 @@ private static void populateFakes3() throws IOException, InterruptedException {
 
   @Test
   public void testProduceFullFile() throws Exception {
-    AmazonS3Source source = createSource();
-    SourceRunner runner = new SourceRunner.Builder(AmazonS3DSource.class, source)
-      .addService(DataFormatParserService.class, new SdkJsonDataFormatParserService())
-      .addOutputLane("lane")
-      .build();
-    runner.runInit();
     try {
+      AmazonS3Source source = createSource();
+      SourceRunner runner = new SourceRunner.Builder(AmazonS3DSource.class, source)
+        .addService(DataFormatParserService.class, new SdkJsonDataFormatParserService())
+        .addOutputLane("lane")
+        .build();
+      runner.runInit();
+      try {
 
-      BatchMaker batchMaker = SourceRunner.createTestBatchMaker("lane");
-      String offset = source.produce(null, 60000, batchMaker);
-      Assert.assertNotNull(offset);
-      Assert.assertTrue(offset.contains("file1.log::-1::"));
+        BatchMaker batchMaker = SourceRunner.createTestBatchMaker("lane");
+        String offset = source.produce(null, 60000, batchMaker);
+        Assert.assertNotNull(offset);
+        Assert.assertTrue(offset.contains("file1.log::-1::"));
 
-      StageRunner.Output output = SourceRunner.getOutput(batchMaker);
-      List<Record> records = output.getRecords().get("lane");
-      Assert.assertEquals(1, records.size());
+        StageRunner.Output output = SourceRunner.getOutput(batchMaker);
+        List<Record> records = output.getRecords().get("lane");
+        Assert.assertEquals(1, records.size());
 
-      //produce records from next file
-      batchMaker = SourceRunner.createTestBatchMaker("lane");
-      offset = source.produce(offset, 60000, batchMaker);
-      Assert.assertNotNull(offset);
-      Assert.assertTrue(offset.contains("file2.log::-1::"));
+        //produce records from next file
+        batchMaker = SourceRunner.createTestBatchMaker("lane");
+        offset = source.produce(offset, 60000, batchMaker);
+        Assert.assertNotNull(offset);
+        Assert.assertTrue(offset.contains("file2.log::-1::"));
 
-      output = SourceRunner.getOutput(batchMaker);
-      records = output.getRecords().get("lane");
-      Assert.assertEquals(1, records.size());
+        output = SourceRunner.getOutput(batchMaker);
+        records = output.getRecords().get("lane");
+        Assert.assertEquals(1, records.size());
 
-      //produce records from next file
-      batchMaker = SourceRunner.createTestBatchMaker("lane");
-      offset = source.produce(offset, 60000, batchMaker);
-      Assert.assertNotNull(offset);
-      Assert.assertTrue(offset.contains("file3.log::-1::"));
+        //produce records from next file
+        batchMaker = SourceRunner.createTestBatchMaker("lane");
+        offset = source.produce(offset, 60000, batchMaker);
+        Assert.assertNotNull(offset);
+        Assert.assertTrue(offset.contains("file3.log::-1::"));
 
-      output = SourceRunner.getOutput(batchMaker);
-      records = output.getRecords().get("lane");
-      Assert.assertEquals(1, records.size());
+        output = SourceRunner.getOutput(batchMaker);
+        records = output.getRecords().get("lane");
+        Assert.assertEquals(1, records.size());
 
-      batchMaker = SourceRunner.createTestBatchMaker("lane");
-      offset = source.produce(offset, 60000, batchMaker);
-      Assert.assertNotNull(offset);
-      Assert.assertTrue(offset.contains("file3.log::-1::"));
+        batchMaker = SourceRunner.createTestBatchMaker("lane");
+        offset = source.produce(offset, 60000, batchMaker);
+        Assert.assertNotNull(offset);
+        Assert.assertTrue(offset.contains("file3.log::-1::"));
 
-      output = SourceRunner.getOutput(batchMaker);
-      records = output.getRecords().get("lane");
-      Assert.assertEquals(0, records.size());
+        output = SourceRunner.getOutput(batchMaker);
+        records = output.getRecords().get("lane");
+        Assert.assertEquals(0, records.size());
 
-    } finally {
-      runner.runDestroy();
+      } finally {
+        runner.runDestroy();
+      }
+    } catch (Exception e) {
+      System.out.println("Hoops");
+      e.printStackTrace();
+      throw  e;
     }
   }