Load a social sample file (#44)

Social files currently have four required attribute fields and can contain an arbitrary number of non mandatory attributes. The attributes are described by a header row in the sample file.

Adds functionality to parse a social sample file, persist sample attributes and post attributes to collection exercise. When a file is uploaded it is parsed and validated. If valid, it will be saved to a sample attributes table in the sample database linked by the sample unit id (UUID). When a request is made for samples the sample unit and sample attributes and put on the sample delivery queue.

We have added integration tests to exercise the upload functionality and ensure they are published to the queue with the sample attributes. 

**How to test:**
Requires changes in these branches:
ONSdigital/rm-samplesvc-api#11
ONSdigital/rm-party-service-api#11

You'll have to build them locally then point the pom file to those snapshots, this will block Travis from building until those changes are on master.

Try loading a social sample through the endpoint using the loader script:
#45

You should also try the loader script with a full example sample file, there is one available as an attachment on the story card:
![](https://github.trello.services/images/mini-trello-icon.png) [Dev task 1: INT: load a social sample file (13)](https://trello.com/c/lTUmPnuf/97-dev-task-1-int-load-a-social-sample-file-13)

Also requires these changes before being merged:
ONSdigital/rm-samplesvc-api#9
#42

pom.xml

@@ -30,13 +30,13 @@
     <dependency>
       <groupId>uk.gov.ons.ctp.product</groupId>
       <artifactId>partysvc-api</artifactId>
-      <version>10.50.2</version>
+      <version>10.50.7</version>
     </dependency>
 
     <dependency>
       <groupId>uk.gov.ons.ctp.product</groupId>
       <artifactId>samplesvc-api</artifactId>
-      <version>10.49.9</version>
+      <version>10.49.14</version>
     </dependency>
 
     <dependency>
@@ -203,6 +203,21 @@
       <version>1.4.9</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-csv</artifactId>
+      <version>1.5</version>
+    </dependency>
+    <dependency>
+      <groupId>commons-io</groupId>
+      <artifactId>commons-io</artifactId>
+      <version>2.6</version>
+    </dependency>
+    <dependency>
+      <groupId>com.vladmihalcea</groupId>
+      <artifactId>hibernate-types-5</artifactId>
+      <version>2.2.1</version>
+    </dependency>
   </dependencies>
 
   <build>

src/main/java/uk/gov/ons/ctp/response/sample/SampleBeanMapper.java

@@ -6,6 +6,7 @@
 import ma.glasnost.orika.impl.generator.EclipseJdtCompilerStrategy;
 import org.springframework.context.annotation.Primary;
 import org.springframework.stereotype.Component;
+import uk.gov.ons.ctp.response.sample.mapper.SampleUnitMapper;
 
 /**
  * The bean mapper to go from Entity objects to Presentation objects.
@@ -26,5 +27,6 @@ public void configureFactoryBuilder(DefaultMapperFactory.Builder builder) {
    */
   @Override
   protected final void configure(final MapperFactory factory) {
+    factory.registerMapper(new SampleUnitMapper());
   }
 }

src/main/java/uk/gov/ons/ctp/response/sample/domain/model/SampleAttributes.java

@@ -0,0 +1,35 @@
+package uk.gov.ons.ctp.response.sample.domain.model;
+
+
+import com.vladmihalcea.hibernate.type.json.JsonBinaryType;
+import lombok.*;
+import net.sourceforge.cobertura.CoverageIgnore;
+import org.hibernate.annotations.Type;
+import org.hibernate.annotations.TypeDef;
+
+import javax.persistence.*;
+import java.io.Serializable;
+import java.util.Map;
+import java.util.UUID;
+
+/**
+ * Domain model object.
+ */
+@CoverageIgnore
+@Entity
+@Data
+@Builder
+@NoArgsConstructor(access = AccessLevel.PUBLIC)
+@AllArgsConstructor
+@TypeDef(name = "jsonb", typeClass = JsonBinaryType.class)
+@Table(name = "sampleattributes", schema = "sample")
+public class SampleAttributes implements Serializable {
+
+    @Id
+    @Column(name = "sampleunitfk")
+    private UUID sampleUnitFK;
+
+    @Column(name = "attributes", columnDefinition = "jsonb")
+    @Type(type = "jsonb")
+    private Map<String, String> attributes;
+}

src/main/java/uk/gov/ons/ctp/response/sample/domain/model/SampleUnit.java

@@ -1,7 +1,14 @@
 package uk.gov.ons.ctp.response.sample.domain.model;
 
-import java.io.Serializable;
-import java.util.UUID;
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import net.sourceforge.cobertura.CoverageIgnore;
+import org.hibernate.annotations.GenericGenerator;
+import org.hibernate.annotations.Parameter;
+import uk.gov.ons.ctp.response.sample.representation.SampleUnitDTO;
 
 import javax.persistence.Column;
 import javax.persistence.Entity;
@@ -11,17 +18,9 @@
 import javax.persistence.GenerationType;
 import javax.persistence.Id;
 import javax.persistence.Table;
-
-import net.sourceforge.cobertura.CoverageIgnore;
-import org.hibernate.annotations.GenericGenerator;
-import org.hibernate.annotations.Parameter;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-import uk.gov.ons.ctp.response.sample.representation.SampleUnitDTO;
+import javax.persistence.Transient;
+import java.io.Serializable;
+import java.util.UUID;
 
 /**
  * Domain model object.
@@ -66,4 +65,6 @@ public class SampleUnit implements Serializable {
   @Column(name = "statefk")
   private SampleUnitDTO.SampleUnitState state;
 
+  @Transient
+  private SampleAttributes sampleAttributes;
 }

src/main/java/uk/gov/ons/ctp/response/sample/domain/repository/SampleAttributesRepository.java

@@ -0,0 +1,16 @@
+package uk.gov.ons.ctp.response.sample.domain.repository;
+
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.stereotype.Repository;
+import uk.gov.ons.ctp.response.sample.domain.model.SampleAttributes;
+
+import java.util.UUID;
+
+
+/**
+ * JPA Data Repository needed to persist Survey SampleAttributes
+ */
+@Repository
+public interface SampleAttributesRepository extends JpaRepository<SampleAttributes, UUID> {
+
+}

src/main/java/uk/gov/ons/ctp/response/sample/endpoint/SampleEndpoint.java

@@ -120,6 +120,7 @@ public SampleSummary ingest(final MultipartFile file, final String type) throws
       try {
         return Optional.of(this.sampleService.ingest(newSummary, file, type));
       } catch (Exception e) {
+        log.error("Failed to ingest sample [{}]", newSummary.getId(), e);
         return this.sampleService.failSampleSummary(newSummary, e);
       }
     };

src/main/java/uk/gov/ons/ctp/response/sample/ingest/CsvIngesterBusiness.java

@@ -10,10 +10,14 @@
 import org.springframework.stereotype.Service;
 import org.springframework.web.multipart.MultipartFile;
 import uk.gov.ons.ctp.common.error.CTPException;
+import uk.gov.ons.ctp.response.party.definition.PartyCreationRequestDTO;
 import uk.gov.ons.ctp.response.sample.domain.model.SampleSummary;
+import uk.gov.ons.ctp.response.sample.message.PartyPublisher;
+import uk.gov.ons.ctp.response.sample.party.PartyUtil;
+import uk.gov.ons.ctp.response.sample.representation.SampleUnitDTO.SampleUnitState;
 import uk.gov.ons.ctp.response.sample.service.SampleService;
 import validation.BusinessSampleUnit;
-import validation.BusinessSurveySample;
+import validation.SampleUnitBase;
 
 import javax.validation.ConstraintViolation;
 import javax.validation.Validation;
@@ -22,11 +26,11 @@
 import java.io.InputStreamReader;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
-import java.util.HashSet;
 
 @Slf4j
 @Service
@@ -67,6 +71,9 @@ public class CsvIngesterBusiness extends CsvToBean<BusinessSampleUnit> {
   @Autowired
   private SampleService sampleService;
 
+  @Autowired
+  private PartyPublisher partyPublisher;
+
   private ColumnPositionMappingStrategy<BusinessSampleUnit> columnPositionMappingStrategy;
 
   /**
@@ -91,42 +98,44 @@ public SampleSummary ingest(final SampleSummary sampleSummary, final MultipartFi
 
     CSVReader csvReader = new CSVReader(new InputStreamReader(file.getInputStream()), ':');
     String[] nextLine;
-    List<BusinessSampleUnit> samplingUnitList = new ArrayList<>();
-    int lineNumber = 0;
+    List<BusinessSampleUnit> sampleUnitList = new ArrayList<>();
     Set<String> unitRefs = new HashSet<>();
 
       while((nextLine = csvReader.readNext()) != null) {
-        lineNumber++;
-
         try {
-          BusinessSampleUnit businessSampleUnit = processLine(columnPositionMappingStrategy, nextLine);
-          List<String> namesOfInvalidColumns = validateLine(businessSampleUnit);
-
-          // If a unit ref is already registered
-          if (unitRefs.contains(businessSampleUnit.getSampleUnitRef())) {
-            log.error("This sample unit ref {} is duplicated in the file.", businessSampleUnit.getSampleUnitRef());
-            throw new CTPException(CTPException.Fault.VALIDATION_FAILED,
-                    String.format("This sample unit ref %s is duplicated in the file.", businessSampleUnit.getSampleUnitRef()));
-          }
-          unitRefs.add(businessSampleUnit.getSampleUnitRef());
+          sampleUnitList.add(parseLine(nextLine, unitRefs));
+        } catch(CTPException e){
+          String newMessage = String.format("Line %d: %s", csvReader.getRecordsRead(), e.getMessage());
+          throw new CTPException(e.getFault(), e, newMessage);
+        }
+      }
+      SampleSummary sampleSummaryWithCICount = sampleService.saveSample(sampleSummary, sampleUnitList, SampleUnitState.INIT);
+      publishToPartyQueue(sampleUnitList, sampleSummary.getId().toString());
+
+      return sampleSummaryWithCICount;
+  }
+
+  private BusinessSampleUnit parseLine(String[] nextLine, Set<String> unitRefs) throws IllegalAccessException, java.lang.reflect.InvocationTargetException, InstantiationException, java.beans.IntrospectionException, CTPException {
+      BusinessSampleUnit businessSampleUnit = processLine(columnPositionMappingStrategy, nextLine);
+      List<String> namesOfInvalidColumns = validateLine(businessSampleUnit);
+
+      // If a unit ref is already registered
+      if (unitRefs.contains(businessSampleUnit.getSampleUnitRef())) {
+        log.warn("This sample unit ref {} is duplicated in the file.", businessSampleUnit.getSampleUnitRef());
+        throw new CTPException(CTPException.Fault.VALIDATION_FAILED,
+                String.format("This sample unit ref %s is duplicated in the file.", businessSampleUnit.getSampleUnitRef()));
+      }
+      unitRefs.add(businessSampleUnit.getSampleUnitRef());
 
           if (!namesOfInvalidColumns.isEmpty()) {
             String errorMessage = String.format("Error in %s due to field(s) %s", Arrays.toString(nextLine),
                     Arrays.toString(namesOfInvalidColumns.toArray()));
-            log.error(errorMessage);
+            log.warn(errorMessage);
             throw new CTPException(CTPException.Fault.VALIDATION_FAILED, errorMessage);
           }
           businessSampleUnit.setSampleUnitType("B");
 
-          samplingUnitList.add(businessSampleUnit);
-        } catch(CTPException e){
-          String newMessage = String.format("Line %d: %s", lineNumber, e.getMessage());
-
-          throw new CTPException(e.getFault(), newMessage);
-        }
-      }
-
-      return sampleService.processSampleSummary(sampleSummary, samplingUnitList);
+      return businessSampleUnit;
   }
 
   private List<String> validateLine(BusinessSampleUnit csvLine) {
@@ -142,4 +151,13 @@ private List<String> validateLine(BusinessSampleUnit csvLine) {
     return invalidFields;
   }
 
+
+  private void publishToPartyQueue(List<? extends SampleUnitBase> samplingUnitList, String sampleSummaryId){
+    for (SampleUnitBase sampleUnitBase : samplingUnitList) {
+          PartyCreationRequestDTO party = PartyUtil.convertToParty(sampleUnitBase);
+          party.getAttributes().setSampleUnitId(sampleUnitBase.getSampleUnitId().toString());
+          party.setSampleSummaryId(sampleSummaryId);
+          partyPublisher.publish(party);
+    }
+  }
 }

src/main/java/uk/gov/ons/ctp/response/sample/ingest/CsvIngesterCensus.java

@@ -10,9 +10,9 @@
 import org.springframework.web.multipart.MultipartFile;
 import uk.gov.ons.ctp.common.error.CTPException;
 import uk.gov.ons.ctp.response.sample.domain.model.SampleSummary;
+import uk.gov.ons.ctp.response.sample.representation.SampleUnitDTO.SampleUnitState;
 import uk.gov.ons.ctp.response.sample.service.SampleService;
 import validation.CensusSampleUnit;
-import validation.CensusSurveySample;
 
 import javax.validation.ConstraintViolation;
 import javax.validation.Validation;
@@ -108,7 +108,7 @@ public SampleSummary ingest(final SampleSummary sampleSummary, final MultipartFi
 
       }
 
-      return sampleService.processSampleSummary(sampleSummary, samplingUnitList);
+      return sampleService.saveSample(sampleSummary, samplingUnitList, SampleUnitState.INIT);
   }
 
   /**