Pagopa 1011 creazione schedule per lettura table e aggiornamento file xml

pom.xml

@@ -32,7 +32,8 @@
     <commons-validator>1.7</commons-validator>
     <feign-version>12.1</feign-version>
     <postgresql-version>42.5.1</postgresql-version>
-    <starter.version>1.7.0</starter.version>
+    <azure.storage.table.version>8.6.6</azure.storage.table.version>
+    <starter.version>1.7.1</starter.version>
   </properties>
 
   <repositories>
@@ -142,6 +143,27 @@
       <version>${opencsv-version}</version>
     </dependency>
 
+    <!-- Start Azure -->
+    <dependency>
+      <groupId>com.microsoft.azure</groupId>
+      <artifactId>azure-storage</artifactId>
+      <version>${azure.storage.table.version}</version>
+    </dependency>
+    <!-- End Azure -->
+
+    <dependency>
+      <groupId>org.awaitility</groupId>
+      <artifactId>awaitility</artifactId>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.testcontainers</groupId>
+      <artifactId>junit-jupiter</artifactId>
+      <version>1.17.6</version>
+      <scope>test</scope>
+    </dependency>
+
     <!-- Iban validation -->
     <dependency>
       <groupId>commons-validator</groupId>

src/main/java/it/gov/pagopa/apiconfig/core/config/SchedulerConfig.java

@@ -0,0 +1,14 @@
+package it.gov.pagopa.apiconfig.core.config;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.scheduling.annotation.EnableAsync;
+import org.springframework.scheduling.annotation.EnableScheduling;
+
+@Configuration
+@EnableScheduling
+@EnableAsync
+@ConditionalOnProperty(name = "cron.job.schedule.enabled", matchIfMissing = true)
+public class SchedulerConfig {
+
+}

src/main/java/it/gov/pagopa/apiconfig/core/exception/AppError.java

@@ -12,6 +12,11 @@ public enum AppError {
       HttpStatus.NOT_FOUND,
       "Creditor Institution not found",
       "No Creditor Institution found with code: %s"),
+
+  CREDITOR_INSTITUTIONS_NOT_FOUND(
+      HttpStatus.NOT_FOUND,
+      "Creditor Institution not found",
+      "No Creditor Institution found inside list"),
   CREDITOR_INSTITUTION_CONFLICT(
       HttpStatus.CONFLICT,
       "Creditor Institution conflict",
@@ -210,6 +215,11 @@ public enum AppError {
       HttpStatus.CONFLICT,
       "Postal IBAN already associated with one CI",
       "The postal IBAN with code %s was already associated to one CI, this type of IBAN cannot be associated to an additional creditor institution %s"),
+
+  AZURE_STORAGE_ERROR(
+      HttpStatus.INTERNAL_SERVER_ERROR,
+      "Error with the azure table storage",
+      "Error when interacting with the azure table storage"),
 
   UNKNOWN(null, null, null);
 

src/main/java/it/gov/pagopa/apiconfig/core/scheduler/SchedulerIca.java

@@ -0,0 +1,141 @@
+package it.gov.pagopa.apiconfig.core.scheduler;
+
+import com.sun.xml.txw2.output.IndentingXMLStreamWriter;
+import it.gov.pagopa.apiconfig.core.exception.AppError;
+import it.gov.pagopa.apiconfig.core.exception.AppException;
+import it.gov.pagopa.apiconfig.core.scheduler.storage.AzureStorageInteraction;
+import it.gov.pagopa.apiconfig.starter.entity.Iban;
+import it.gov.pagopa.apiconfig.starter.entity.IbanMaster;
+import it.gov.pagopa.apiconfig.starter.entity.IcaBinaryFile;
+import it.gov.pagopa.apiconfig.starter.entity.Pa;
+import it.gov.pagopa.apiconfig.starter.repository.IbanMasterRepository;
+import it.gov.pagopa.apiconfig.starter.repository.IbanRepository;
+import it.gov.pagopa.apiconfig.starter.repository.IcaBinaryFileRepository;
+import it.gov.pagopa.apiconfig.starter.repository.PaRepository;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.scheduling.annotation.Async;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+import javax.transaction.Transactional;
+import javax.xml.stream.XMLOutputFactory;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamWriter;
+import java.io.ByteArrayOutputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+@Component
+@Slf4j
+public class SchedulerIca {
+
+  @Autowired private PaRepository paRepository;
+
+  @Autowired private IbanRepository ibanRepository;
+
+  @Autowired private IbanMasterRepository ibanMasterRepository;
+
+  @Autowired private IcaBinaryFileRepository icaBinaryFileRepository;
+
+  @Autowired private AzureStorageInteraction azureStorageInteraction;
+
+  private static final String XML_ERROR = "Error in writing XML file %s";
+
+  @Scheduled(cron = "${cron.job.schedule.expression}")
+  @Async
+  @Transactional
+  public void updateIcaFile() {
+    LocalDateTime currentDate = LocalDateTime.now(ZoneOffset.UTC).minusDays(1L);
+    Map<String, String> updatedEcFiscalCodeIcas = azureStorageInteraction
+        .getUpdatedEC(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").format(currentDate));
+    List<Pa> creditorInstitutions = paRepository
+        .findByIdDominioIn(new ArrayList<>(updatedEcFiscalCodeIcas.keySet()))
+        .orElseThrow(() -> new AppException(AppError.CREDITOR_INSTITUTIONS_NOT_FOUND));
+    creditorInstitutions.forEach(ec -> {
+      List<IbanMaster> ibanAttributeMasters = ibanMasterRepository.findByFkPa(ec.getObjId());
+      List<Iban> ibans = ibanRepository.findByObjIdIn(ibanAttributeMasters.stream()
+          .map(IbanMaster::getObjId)
+          .collect(Collectors.toList()));
+      byte[] icaBinaryFileXml = createXml(ec, ibans, updatedEcFiscalCodeIcas);
+      IcaBinaryFile icaBinaryFile = IcaBinaryFile.builder()
+          .fileContent(icaBinaryFileXml)
+          .idDominio(ec.getIdDominio())
+          .fileSize((long)icaBinaryFileXml.length)
+          .fileHash(getHash(icaBinaryFileXml))
+          .build();
+      icaBinaryFileRepository.save(icaBinaryFile);
+    });
+  }
+
+  private byte[] createXml(Pa pa, List<Iban> ibans, Map<String, String> publicationEcRelation) {
+    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+    try {
+      XMLOutputFactory xof = XMLOutputFactory.newInstance();
+      XMLStreamWriter writer = xof.createXMLStreamWriter(outputStream, "UTF-8");
+      XMLStreamWriter xtw = new IndentingXMLStreamWriter(writer);
+      xtw.writeStartDocument("UTF-8", "1.0");
+      xtw.writeStartElement("informativaContoAccredito");
+      xtw.writeNamespace("xsi", "http://www.w3.org/2001/XMLSchema-instance");
+      xtw.writeAttribute("http://www.w3.org/2001/XMLSchema-instance", "noNamespaceSchemaLocation",
+          "InformativaContoAccredito _v02.xsd");
+      writeElement(xtw, "identificativoFlusso", pa.getIdDominio() + "_" + LocalDateTime.now().toString());
+      writeElement(xtw, "identificativoDominio", pa.getIdDominio());
+      writeElement(xtw, "ragioneSociale", pa.getRagioneSociale());
+      writeElement(xtw, "dataPubblicazione", publicationEcRelation.get(pa.getIdDominio()));
+      writeElement(xtw, "dataInizioValidita", LocalDateTime.now().toString());
+      xtw.writeStartElement("contiDiAccredito");
+      ibans.forEach(iban ->
+        writeIbanElement(xtw, iban)
+      );
+      xtw.writeEndElement();
+      xtw.writeEndElement();
+      xtw.writeEndDocument();
+      xtw.flush();
+      xtw.close();
+    } catch (XMLStreamException e){
+      log.error(XML_ERROR, e.getMessage());
+    }
+    return outputStream.toByteArray();
+  }
+
+  private void writeElement(XMLStreamWriter xtw, String elementName, String elementValue){
+    try {
+      xtw.writeStartElement(elementName);
+      xtw.writeCharacters(elementValue);
+      xtw.writeEndElement();
+    } catch (XMLStreamException e){
+      log.error(XML_ERROR, e.getMessage());
+    }
+  }
+
+  private void writeIbanElement(XMLStreamWriter xtw, Iban iban){
+    try {
+      xtw.writeStartElement("infoContoDiAccreditoPair");
+      xtw.writeStartElement("ibanAccredito");
+      xtw.writeCharacters(iban.getIban());
+      xtw.writeEndElement();
+      xtw.writeEmptyElement("idBancaSeller");
+      xtw.writeEndElement();
+    } catch (XMLStreamException e){
+      log.error(XML_ERROR, e.getMessage());
+    }
+  }
+
+  private byte[] getHash(byte[] input){
+    byte[] hashedOutput = null;
+    try{
+      MessageDigest md = MessageDigest.getInstance("SHA-256");
+      hashedOutput = md.digest(input);
+    } catch(NoSuchAlgorithmException e){
+      log.error("No security algorithm was found %s", e.getMessage());
+    }
+    return hashedOutput;
+  }
+}

src/main/java/it/gov/pagopa/apiconfig/core/scheduler/entity/CreditorInstitutionIcaFile.java

@@ -0,0 +1,20 @@
+package it.gov.pagopa.apiconfig.core.scheduler.entity;
+
+import com.microsoft.azure.storage.table.TableServiceEntity;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+@Getter
+@Setter
+@NoArgsConstructor
+public class CreditorInstitutionIcaFile extends TableServiceEntity {
+  private String publicationDate;
+  public static final String ORGANIZATION_KEY = "organization";
+
+  public CreditorInstitutionIcaFile(String organizationFiscalCode, String publicationDate) {
+    this.partitionKey = ORGANIZATION_KEY;
+    this.rowKey = organizationFiscalCode;
+    this.publicationDate = publicationDate;
+  }
+}

src/main/java/it/gov/pagopa/apiconfig/core/scheduler/storage/AzureStorageInteraction.java

@@ -0,0 +1,58 @@
+package it.gov.pagopa.apiconfig.core.scheduler.storage;
+
+import com.microsoft.azure.storage.CloudStorageAccount;
+import com.microsoft.azure.storage.StorageException;
+import com.microsoft.azure.storage.table.CloudTable;
+import com.microsoft.azure.storage.table.TableOperation;
+import com.microsoft.azure.storage.table.TableQuery;
+import com.microsoft.azure.storage.table.TableServiceEntity;
+import it.gov.pagopa.apiconfig.core.exception.AppError;
+import it.gov.pagopa.apiconfig.core.exception.AppException;
+import it.gov.pagopa.apiconfig.core.scheduler.entity.CreditorInstitutionIcaFile;
+import lombok.NoArgsConstructor;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.http.HttpStatus;
+import org.springframework.stereotype.Component;
+import java.net.URISyntaxException;
+import java.security.InvalidKeyException;
+import java.util.List;
+import java.util.Map;
+import java.util.Spliterator;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+
+@Component
+@NoArgsConstructor
+public class AzureStorageInteraction {
+
+  @Value("${creditor.institution.table.connection.string}")
+  private String storageConnectionString;
+
+  @Value("${creditor.institution.update.table}")
+  private String icaTable;
+
+  public AzureStorageInteraction(String storageConnectionString, String icaTable){
+    this.storageConnectionString = storageConnectionString;
+    this.icaTable = icaTable;
+  }
+
+  public Map<String, String> getUpdatedEC(String lastUpdate) throws AppException {
+    Spliterator<CreditorInstitutionIcaFile> resultOrganizationIcaList = null;
+    try {
+      CloudTable table = CloudStorageAccount.parse(storageConnectionString).createCloudTableClient()
+          .getTableReference(this.icaTable);
+      resultOrganizationIcaList =
+          table.execute(TableQuery.from(CreditorInstitutionIcaFile.class)
+                  .where(TableQuery.generateFilterCondition("PublicationDate", TableQuery.QueryComparisons.GREATER_THAN, lastUpdate)))
+              .spliterator();
+    } catch (InvalidKeyException | URISyntaxException | StorageException e) {
+      // unexpected error
+      throw new AppException(AppError.AZURE_STORAGE_ERROR);
+    }
+    return StreamSupport.stream(resultOrganizationIcaList, false).collect(
+        Collectors.toMap(
+          TableServiceEntity::getRowKey,
+          CreditorInstitutionIcaFile::getPublicationDate)
+        );
+  }
+}

src/main/java/it/gov/pagopa/apiconfig/core/service/IbanService.java

@@ -11,13 +11,11 @@
 import it.gov.pagopa.apiconfig.starter.entity.IbanAttributeMaster;
 import it.gov.pagopa.apiconfig.starter.entity.IbanMaster;
 import it.gov.pagopa.apiconfig.starter.entity.IbanMaster.IbanStatus;
-import it.gov.pagopa.apiconfig.starter.entity.IcaBinaryFile;
 import it.gov.pagopa.apiconfig.starter.entity.Pa;
 import it.gov.pagopa.apiconfig.starter.repository.IbanAttributeMasterRepository;
 import it.gov.pagopa.apiconfig.starter.repository.IbanAttributeRepository;
 import it.gov.pagopa.apiconfig.starter.repository.IbanMasterRepository;
 import it.gov.pagopa.apiconfig.starter.repository.IbanRepository;
-import it.gov.pagopa.apiconfig.starter.repository.IcaBinaryFileRepository;
 import it.gov.pagopa.apiconfig.starter.repository.PaRepository;
 import java.time.OffsetDateTime;
 import java.time.ZoneOffset;
@@ -56,8 +54,6 @@ public class IbanService {
 
   @Autowired private IbanAttributeMasterRepository ibanAttributeMasterRepository;
 
-  @Autowired private IcaBinaryFileRepository icaBinaryFileRepository;
-
   @Autowired private ModelMapper modelMapper;
 
   public IbanEnhanced createIban(
@@ -82,12 +78,9 @@ public IbanEnhanced createIban(
                   iban.getIbanValue(),
                   existingCreditorInstitution.getIdDominio());
             });
-    // generate an empty ICA binary file and a relation between iban, CI and ICA file
-    IcaBinaryFile icaBinaryFileToBeCreated = IcaBinaryFile.builder().fileSize(0L).build();
-    icaBinaryFileToBeCreated = icaBinaryFileRepository.save(icaBinaryFileToBeCreated);
     IbanMaster ibanCIRelationToBeCreated =
         saveIbanCIRelation(
-            existingCreditorInstitution, iban, ibanToBeCreated, icaBinaryFileToBeCreated);
+            existingCreditorInstitution, iban, ibanToBeCreated);
     // generate the relation between iban and attributes
     List<IbanAttributeMaster> updatedIbanAttributes =
         saveIbanLabelRelation(iban, ibanCIRelationToBeCreated);
@@ -135,8 +128,7 @@ public IbanEnhanced updateIban(
             existingIbanMaster,
             existingCreditorInstitution,
             iban,
-            existingIban,
-            existingIbanMaster.getIcaBinaryFile());
+            existingIban);
     // remove all labels and save them again
     ibanAttributeMasterRepository.deleteAll(existingIbanMaster.getIbanAttributesMasters());
     ibanAttributeMasterRepository.flush();
@@ -262,21 +254,18 @@ private Iban saveIban(IbanEnhanced iban, Iban existingIban) {
   private IbanMaster saveIbanCIRelation(
       Pa creditorInstitution,
       IbanEnhanced iban,
-      Iban ibanToBeCreated,
-      IcaBinaryFile icaBinaryFileToBeCreated) {
+      Iban ibanToBeCreated) {
     return saveIbanCIRelation(
-        new IbanMaster(), creditorInstitution, iban, ibanToBeCreated, icaBinaryFileToBeCreated);
+        new IbanMaster(), creditorInstitution, iban, ibanToBeCreated);
   }
 
   private IbanMaster saveIbanCIRelation(
       IbanMaster ibanMaster,
       Pa creditorInstitution,
       IbanEnhanced iban,
-      Iban ibanToBeCreated,
-      IcaBinaryFile icaBinaryFileToBeCreated) {
+      Iban ibanToBeCreated) {
     ibanMaster.setFkPa(creditorInstitution.getObjId());
     ibanMaster.setFkIban(ibanToBeCreated.getObjId());
-    ibanMaster.setFkIcaBinaryFile(icaBinaryFileToBeCreated.getObjId());
     ibanMaster.setIbanStatus(iban.isActive() ? IbanStatus.ENABLED : IbanStatus.DISABLED);
     ibanMaster.setInsertedDate(
         ibanMaster.getInsertedDate() != null
@@ -285,8 +274,6 @@ private IbanMaster saveIbanCIRelation(
     ibanMaster.setValidityDate(CommonUtil.toTimestamp(iban.getValidityDate()));
     ibanMaster.setPa(creditorInstitution); // setting CI object reference
     ibanMaster.setIban(ibanToBeCreated); // setting IBAN object reference
-    ibanMaster.setIcaBinaryFile(
-        icaBinaryFileToBeCreated); // setting ICA binary file object reference
     return ibanMasterRepository.save(ibanMaster);
   }
 

src/main/resources/application-h2.properties

@@ -53,5 +53,13 @@ retry.utils.maxDelay=200
 # Nodo monitoring configuration
 service.nodo-monitoring.host=http://localhost:8587
 
+# Scheduling configuration
+cron.job.schedule.enabled=true
+cron.job.schedule.expression=*/10 * * * * *
+
+# Azurite information
+creditor.institution.update.table=ICATABLE
+creditor.institution.table.connection.string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;
+
 # IBAN ABI
 iban.abi.poste=07601