Pagopa 1011 creazione schedule per lettura table e aggiornamento file xml

pom.xml

@@ -32,6 +32,8 @@
     <commons-validator>1.7</commons-validator>
     <feign-version>12.1</feign-version>
     <postgresql-version>42.5.1</postgresql-version>
+    <azure.storage.table.version>8.6.6</azure.storage.table.version>
+    <testcontainer.version>1.17.6</testcontainer.version>
     <starter.version>1.8.0</starter.version>
   </properties>
 
@@ -142,6 +144,27 @@
       <version>${opencsv-version}</version>
     </dependency>
 
+    <!-- Start Azure -->
+    <dependency>
+      <groupId>com.microsoft.azure</groupId>
+      <artifactId>azure-storage</artifactId>
+      <version>${azure.storage.table.version}</version>
+    </dependency>
+    <!-- End Azure -->
+
+    <dependency>
+      <groupId>org.awaitility</groupId>
+      <artifactId>awaitility</artifactId>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.testcontainers</groupId>
+      <artifactId>junit-jupiter</artifactId>
+      <version>${testcontainer.version}</version>
+      <scope>test</scope>
+    </dependency>
+
     <!-- Iban validation -->
     <dependency>
       <groupId>commons-validator</groupId>

src/main/java/it/gov/pagopa/apiconfig/core/config/SchedulerConfig.java

@@ -0,0 +1,12 @@
+package it.gov.pagopa.apiconfig.core.config;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.scheduling.annotation.EnableAsync;
+import org.springframework.scheduling.annotation.EnableScheduling;
+
+@Configuration
+@EnableScheduling
+@EnableAsync
+@ConditionalOnProperty(name = "cron.job.schedule.enabled", matchIfMissing = true)
+public class SchedulerConfig {}

src/main/java/it/gov/pagopa/apiconfig/core/exception/AppError.java

@@ -12,6 +12,12 @@ public enum AppError {
       HttpStatus.NOT_FOUND,
       "Creditor Institution not found",
       "No Creditor Institution found with code: %s"),
+
+  CREDITOR_INSTITUTIONS_NOT_FOUND(
+      HttpStatus.NOT_FOUND,
+      "Creditor Institution not found",
+      "No Creditor Institution found inside list"),
+
   CREDITOR_INSTITUTION_CONFLICT(
       HttpStatus.CONFLICT,
       "Creditor Institution conflict",
@@ -212,6 +218,16 @@ public enum AppError {
       "The postal IBAN with code %s was already associated to one CI, this type of IBAN cannot be"
           + " associated to an additional creditor institution %s"),
 
+  AZURE_STORAGE_ERROR(
+      HttpStatus.INTERNAL_SERVER_ERROR,
+      "Error with the azure table storage",
+      "Error when interacting with the azure table storage"),
+
+  ICA_XML_ERROR(
+      HttpStatus.INTERNAL_SERVER_ERROR,
+      "Error when writing the ica binary file",
+      "Error when writing the ica binary xml file"),
+
   UNKNOWN(null, null, null);
 
   public final HttpStatus httpStatus;

src/main/java/it/gov/pagopa/apiconfig/core/scheduler/SchedulerIca.java

@@ -0,0 +1,146 @@
+package it.gov.pagopa.apiconfig.core.scheduler;
+
+import com.sun.xml.txw2.output.IndentingXMLStreamWriter;
+import it.gov.pagopa.apiconfig.core.exception.AppError;
+import it.gov.pagopa.apiconfig.core.exception.AppException;
+import it.gov.pagopa.apiconfig.core.scheduler.storage.AzureStorageInteraction;
+import it.gov.pagopa.apiconfig.starter.entity.Iban;
+import it.gov.pagopa.apiconfig.starter.entity.IbanMaster;
+import it.gov.pagopa.apiconfig.starter.entity.IcaBinaryFile;
+import it.gov.pagopa.apiconfig.starter.entity.Pa;
+import it.gov.pagopa.apiconfig.starter.repository.IbanMasterRepository;
+import it.gov.pagopa.apiconfig.starter.repository.IbanRepository;
+import it.gov.pagopa.apiconfig.starter.repository.IcaBinaryFileRepository;
+import it.gov.pagopa.apiconfig.starter.repository.PaRepository;
+import java.io.ByteArrayOutputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import javax.transaction.Transactional;
+import javax.xml.stream.XMLOutputFactory;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamWriter;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.scheduling.annotation.Async;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+@Component
+@Slf4j
+public class SchedulerIca {
+
+  @Autowired private PaRepository paRepository;
+
+  @Autowired private IbanRepository ibanRepository;
+
+  @Autowired private IbanMasterRepository ibanMasterRepository;
+
+  @Autowired private IcaBinaryFileRepository icaBinaryFileRepository;
+
+  @Autowired private AzureStorageInteraction azureStorageInteraction;
+
+  @Scheduled(cron = "${cron.job.schedule.expression}")
+  @Async
+  @Transactional
+  public void updateIcaFile() {
+    LocalDateTime previousExecution = LocalDateTime.now(ZoneOffset.UTC).minusDays(1L);
+    Map<String, String> updatedEcFiscalCodeIcas =
+        azureStorageInteraction.getUpdatedEC(
+            DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").format(previousExecution));
+    List<Pa> creditorInstitutions =
+        paRepository
+            .findByIdDominioIn(new ArrayList<>(updatedEcFiscalCodeIcas.keySet()))
+            .orElseThrow(() -> new AppException(AppError.CREDITOR_INSTITUTIONS_NOT_FOUND));
+    creditorInstitutions.forEach(
+        ec -> {
+          List<IbanMaster> ibanAttributeMasters = ibanMasterRepository.findByFkPa(ec.getObjId());
+          List<Iban> ibans =
+              ibanRepository.findByObjIdIn(
+                  ibanAttributeMasters.stream()
+                      .map(IbanMaster::getObjId)
+                      .collect(Collectors.toList()));
+          byte[] icaBinaryFileXml = createXml(ec, ibans, updatedEcFiscalCodeIcas);
+          IcaBinaryFile icaBinaryFile =
+              IcaBinaryFile.builder()
+                  .fileContent(icaBinaryFileXml)
+                  .idDominio(ec.getIdDominio())
+                  .fileSize((long) icaBinaryFileXml.length)
+                  .fileHash(getHash(icaBinaryFileXml))
+                  .build();
+          icaBinaryFileRepository.save(icaBinaryFile);
+        });
+  }
+
+  private byte[] createXml(Pa pa, List<Iban> ibans, Map<String, String> publicationEcRelation) {
+    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+    try {
+      XMLOutputFactory xof = XMLOutputFactory.newInstance();
+      XMLStreamWriter writer = xof.createXMLStreamWriter(outputStream, "UTF-8");
+      XMLStreamWriter xtw = new IndentingXMLStreamWriter(writer);
+      xtw.writeStartDocument("UTF-8", "1.0");
+      xtw.writeStartElement("informativaContoAccredito");
+      xtw.writeNamespace("xsi", "http://www.w3.org/2001/XMLSchema-instance");
+      xtw.writeAttribute(
+          "http://www.w3.org/2001/XMLSchema-instance",
+          "noNamespaceSchemaLocation",
+          "InformativaContoAccredito _v02.xsd");
+      writeElement(
+          xtw, "identificativoFlusso", pa.getIdDominio() + "_" + LocalDateTime.now().toString());
+      writeElement(xtw, "identificativoDominio", pa.getIdDominio());
+      writeElement(xtw, "ragioneSociale", pa.getRagioneSociale());
+      writeElement(xtw, "dataPubblicazione", publicationEcRelation.get(pa.getIdDominio()));
+      writeElement(xtw, "dataInizioValidita", LocalDateTime.now().toString());
+      xtw.writeStartElement("contiDiAccredito");
+      ibans.forEach(iban -> writeIbanElement(xtw, iban));
+      xtw.writeEndElement();
+      xtw.writeEndElement();
+      xtw.writeEndDocument();
+      xtw.flush();
+      xtw.close();
+    } catch (XMLStreamException e) {
+      throw new AppException(AppError.ICA_XML_ERROR);
+    }
+    return outputStream.toByteArray();
+  }
+
+  private void writeElement(XMLStreamWriter xtw, String elementName, String elementValue) {
+    try {
+      xtw.writeStartElement(elementName);
+      xtw.writeCharacters(elementValue);
+      xtw.writeEndElement();
+    } catch (XMLStreamException e) {
+      throw new AppException(AppError.ICA_XML_ERROR);
+    }
+  }
+
+  private void writeIbanElement(XMLStreamWriter xtw, Iban iban) {
+    try {
+      xtw.writeStartElement("infoContoDiAccreditoPair");
+      xtw.writeStartElement("ibanAccredito");
+      xtw.writeCharacters(iban.getIban());
+      xtw.writeEndElement();
+      xtw.writeEmptyElement("idBancaSeller");
+      xtw.writeEndElement();
+    } catch (XMLStreamException e) {
+      throw new AppException(AppError.ICA_XML_ERROR);
+    }
+  }
+
+  private byte[] getHash(byte[] input) {
+    byte[] hashedOutput = null;
+    try {
+      MessageDigest md = MessageDigest.getInstance("SHA-256");
+      hashedOutput = md.digest(input);
+    } catch (NoSuchAlgorithmException e) {
+      throw new AppException(AppError.INTERNAL_SERVER_ERROR, e);
+    }
+    return hashedOutput;
+  }
+}

src/main/java/it/gov/pagopa/apiconfig/core/scheduler/entity/CreditorInstitutionIcaFile.java

@@ -0,0 +1,20 @@
+package it.gov.pagopa.apiconfig.core.scheduler.entity;
+
+import com.microsoft.azure.storage.table.TableServiceEntity;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+@Getter
+@Setter
+@NoArgsConstructor
+public class CreditorInstitutionIcaFile extends TableServiceEntity {
+  private String publicationDate;
+  public static final String ORGANIZATION_KEY = "organization";
+
+  public CreditorInstitutionIcaFile(String organizationFiscalCode, String publicationDate) {
+    this.partitionKey = ORGANIZATION_KEY;
+    this.rowKey = organizationFiscalCode;
+    this.publicationDate = publicationDate;
+  }
+}

src/main/java/it/gov/pagopa/apiconfig/core/scheduler/storage/AzureStorageInteraction.java

@@ -0,0 +1,62 @@
+package it.gov.pagopa.apiconfig.core.scheduler.storage;
+
+import com.microsoft.azure.storage.CloudStorageAccount;
+import com.microsoft.azure.storage.StorageException;
+import com.microsoft.azure.storage.table.CloudTable;
+import com.microsoft.azure.storage.table.TableQuery;
+import com.microsoft.azure.storage.table.TableServiceEntity;
+import it.gov.pagopa.apiconfig.core.exception.AppError;
+import it.gov.pagopa.apiconfig.core.exception.AppException;
+import it.gov.pagopa.apiconfig.core.scheduler.entity.CreditorInstitutionIcaFile;
+import java.net.URISyntaxException;
+import java.security.InvalidKeyException;
+import java.util.Map;
+import java.util.Spliterator;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+import lombok.NoArgsConstructor;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+@Component
+@NoArgsConstructor
+public class AzureStorageInteraction {
+
+  @Value("${creditor.institution.table.connection.string}")
+  private String storageConnectionString;
+
+  @Value("${creditor.institution.update.table}")
+  private String icaTable;
+
+  public AzureStorageInteraction(String storageConnectionString, String icaTable) {
+    this.storageConnectionString = storageConnectionString;
+    this.icaTable = icaTable;
+  }
+
+  public Map<String, String> getUpdatedEC(String lastUpdate) throws AppException {
+    Spliterator<CreditorInstitutionIcaFile> resultOrganizationIcaList = null;
+    try {
+      CloudTable table =
+          CloudStorageAccount.parse(storageConnectionString)
+              .createCloudTableClient()
+              .getTableReference(this.icaTable);
+      resultOrganizationIcaList =
+          table
+              .execute(
+                  TableQuery.from(CreditorInstitutionIcaFile.class)
+                      .where(
+                          TableQuery.generateFilterCondition(
+                              "PublicationDate",
+                              TableQuery.QueryComparisons.GREATER_THAN,
+                              lastUpdate)))
+              .spliterator();
+    } catch (InvalidKeyException | URISyntaxException | StorageException e) {
+      // unexpected error
+      throw new AppException(AppError.AZURE_STORAGE_ERROR);
+    }
+    return StreamSupport.stream(resultOrganizationIcaList, false)
+        .collect(
+            Collectors.toMap(
+                TableServiceEntity::getRowKey, CreditorInstitutionIcaFile::getPublicationDate));
+  }
+}

src/main/resources/application-h2.properties

@@ -53,5 +53,13 @@ retry.utils.maxDelay=200
 # Nodo monitoring configuration
 service.nodo-monitoring.host=http://localhost:8587
 
+# Scheduling configuration
+cron.job.schedule.enabled=false
+cron.job.schedule.expression=*/10 * * * * *
+
+# Azurite information
+creditor.institution.update.table=ICATABLE
+creditor.institution.table.connection.string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;
+
 # IBAN ABI
 iban.abi.poste=07601

src/main/resources/application-local.properties

@@ -33,5 +33,13 @@ retry.utils.maxDelay=200
 # Nodo monitoring configuration
 service.nodo-monitoring.host=http://localhost:8587
 
+# Scheduling configuration
+cron.job.schedule.enabled=false
+cron.job.schedule.expression=*/10 * * * * *
+
+# Azurite information
+creditor.institution.update.table=ICATABLE
+creditor.institution.table.connection.string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;
+
 # IBAN ABI
 iban.abi.poste=07601

src/main/resources/application-pg-local.properties

@@ -54,5 +54,13 @@ retry.utils.maxDelay=200
 # Nodo monitoring configuration
 service.nodo-monitoring.host=http://localhost:8587
 
+# Scheduling configuration
+cron.job.schedule.enabled=false
+cron.job.schedule.expression=*/10 * * * * *
+
+# Azurite information
+creditor.institution.update.table=ICATABLE
+creditor.institution.table.connection.string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;
+
 # IBAN ABI
 iban.abi.poste=07601

src/main/resources/application.properties

@@ -75,5 +75,13 @@ retry.utils.maxDelay=${AFM_UTILS_RETRY_MAX_DELAY}
 # Nodo monitoring configuration
 service.nodo-monitoring.host=${NODO_MONITORING_HOST}
 
+# Scheduling configuration
+cron.job.schedule.enabled=false
+cron.job.schedule.expression=0 0 0 * * *
+
+# Azurite information
+creditor.institution.update.table=ICATABLE
+creditor.institution.table.connection.string=placeholder
+
 # IBAN ABI
 iban.abi.poste=07601

src/test/java/it/gov/pagopa/apiconfig/TestUtil.java

@@ -1044,6 +1044,7 @@ private static IbanEnhanced mockIbanEnhancedBuilder(
   public static IcaBinaryFile getMockIcaBinaryFile() {
     return IcaBinaryFile.builder()
         .objId(10L)
+        .idDominio("00168480242")
         .fileContent(new byte[] {1, 10, 20, 30, 40})
         .fileHash(new byte[] {100, 100})
         .fileSize(5L)