Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Pagopa 1011 creazione schedule per lettura table e aggiornamento file xml #226

Merged
Show file tree
Hide file tree
Changes from 15 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
b3303f9
feat: adding cron to populate ica file PAGOPA-1011
FedericoRuzzier Jun 16, 2023
14eb94d
PAGOPA-1011 finalizing junit tests
FedericoRuzzier Jun 19, 2023
30b39ee
PAGOPA-1011 removing unused variable
FedericoRuzzier Jun 19, 2023
f1d9000
PAGOPA-1011 updating all profiles
FedericoRuzzier Jun 20, 2023
da23b86
PAGOPA-1011 removing unused thread
FedericoRuzzier Jun 20, 2023
3f0637a
PAGOPA-1011 remove unused version from pom
FedericoRuzzier Jun 20, 2023
8b25de5
Merge branch 'main' into PAGOPA-1011-creazione-function-per-lettura-e…
FedericoRuzzier Jun 20, 2023
1d1e4fb
PAGOPA-1011 removing ica binary file from iban master
FedericoRuzzier Jun 20, 2023
c89e52c
PAGOPA-1011 adding test
FedericoRuzzier Jun 20, 2023
4e91c4b
PAGOPA-1011 minor fix
FedericoRuzzier Jun 20, 2023
536b12e
Merge branch 'main' into PAGOPA-1011-creazione-function-per-lettura-e…
FedericoRuzzier Jun 21, 2023
c8735ce
PAGOPA-1011 removing unused field from test
FedericoRuzzier Jun 21, 2023
597d4f0
PAGOPA-1011 updating starter version and removing duplicate
FedericoRuzzier Jun 21, 2023
1e78a8a
PAGOPA-1011 solving code smells and security error
FedericoRuzzier Jun 21, 2023
61bb5e1
PAGOPA-1011 adding test to table storage integration
FedericoRuzzier Jun 21, 2023
a6c9eb8
PAGOPA-1011 removing vulnerability and code smells
FedericoRuzzier Jun 21, 2023
9328c6d
Google Java format
Jun 21, 2023
af6be00
PAGOPA-1011 adding version testcontainer
FedericoRuzzier Jun 21, 2023
db973ba
PAGOPA-1011 minor modifications
FedericoRuzzier Jun 21, 2023
4070fc1
PAGOPA-1011 final changes to excpetion and disabling cron
FedericoRuzzier Jun 22, 2023
2e7a4c6
Merge branch 'main' into PAGOPA-1011-creazione-function-per-lettura-e…
FedericoRuzzier Jun 22, 2023
446c1f4
PAGOPA-1011 renaming var
FedericoRuzzier Jun 22, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 23 additions & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@
<commons-validator>1.7</commons-validator>
<feign-version>12.1</feign-version>
<postgresql-version>42.5.1</postgresql-version>
<starter.version>1.7.0</starter.version>
<azure.storage.table.version>8.6.6</azure.storage.table.version>
<starter.version>1.7.1</starter.version>
</properties>

<repositories>
Expand Down Expand Up @@ -142,6 +143,27 @@
<version>${opencsv-version}</version>
</dependency>

<!-- Start Azure -->
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-storage</artifactId>
<version>${azure.storage.table.version}</version>
</dependency>
<!-- End Azure -->

<dependency>
<groupId>org.awaitility</groupId>
<artifactId>awaitility</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>junit-jupiter</artifactId>
<version>1.17.6</version>
<scope>test</scope>
</dependency>

<!-- Iban validation -->
<dependency>
<groupId>commons-validator</groupId>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package it.gov.pagopa.apiconfig.core.config;

import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.annotation.EnableScheduling;

@Configuration
@EnableScheduling
@EnableAsync
@ConditionalOnProperty(name = "cron.job.schedule.enabled", matchIfMissing = true)
public class SchedulerConfig {

}
10 changes: 10 additions & 0 deletions src/main/java/it/gov/pagopa/apiconfig/core/exception/AppError.java
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,11 @@ public enum AppError {
HttpStatus.NOT_FOUND,
"Creditor Institution not found",
"No Creditor Institution found with code: %s"),

CREDITOR_INSTITUTIONS_NOT_FOUND(
HttpStatus.NOT_FOUND,
"Creditor Institution not found",
"No Creditor Institution found inside list"),
CREDITOR_INSTITUTION_CONFLICT(
HttpStatus.CONFLICT,
"Creditor Institution conflict",
Expand Down Expand Up @@ -210,6 +215,11 @@ public enum AppError {
HttpStatus.CONFLICT,
"Postal IBAN already associated with one CI",
"The postal IBAN with code %s was already associated to one CI, this type of IBAN cannot be associated to an additional creditor institution %s"),

AZURE_STORAGE_ERROR(
HttpStatus.INTERNAL_SERVER_ERROR,
"Error with the azure table storage",
"Error when interacting with the azure table storage"),

UNKNOWN(null, null, null);

Expand Down
141 changes: 141 additions & 0 deletions src/main/java/it/gov/pagopa/apiconfig/core/scheduler/SchedulerIca.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
package it.gov.pagopa.apiconfig.core.scheduler;

import com.sun.xml.txw2.output.IndentingXMLStreamWriter;
import it.gov.pagopa.apiconfig.core.exception.AppError;
import it.gov.pagopa.apiconfig.core.exception.AppException;
import it.gov.pagopa.apiconfig.core.scheduler.storage.AzureStorageInteraction;
import it.gov.pagopa.apiconfig.starter.entity.Iban;
import it.gov.pagopa.apiconfig.starter.entity.IbanMaster;
import it.gov.pagopa.apiconfig.starter.entity.IcaBinaryFile;
import it.gov.pagopa.apiconfig.starter.entity.Pa;
import it.gov.pagopa.apiconfig.starter.repository.IbanMasterRepository;
import it.gov.pagopa.apiconfig.starter.repository.IbanRepository;
import it.gov.pagopa.apiconfig.starter.repository.IcaBinaryFileRepository;
import it.gov.pagopa.apiconfig.starter.repository.PaRepository;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import javax.transaction.Transactional;
import javax.xml.stream.XMLOutputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamWriter;
import java.io.ByteArrayOutputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

@Component
@Slf4j
public class SchedulerIca {

@Autowired private PaRepository paRepository;

@Autowired private IbanRepository ibanRepository;

@Autowired private IbanMasterRepository ibanMasterRepository;

@Autowired private IcaBinaryFileRepository icaBinaryFileRepository;

@Autowired private AzureStorageInteraction azureStorageInteraction;

private static final String XML_ERROR = "Error in writing XML file %s";

@Scheduled(cron = "${cron.job.schedule.expression}")
@Async
@Transactional
public void updateIcaFile() {
LocalDateTime currentDate = LocalDateTime.now(ZoneOffset.UTC).minusDays(1L);
Map<String, String> updatedEcFiscalCodeIcas = azureStorageInteraction
.getUpdatedEC(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").format(currentDate));
List<Pa> creditorInstitutions = paRepository
.findByIdDominioIn(new ArrayList<>(updatedEcFiscalCodeIcas.keySet()))
.orElseThrow(() -> new AppException(AppError.CREDITOR_INSTITUTIONS_NOT_FOUND));
creditorInstitutions.forEach(ec -> {
List<IbanMaster> ibanAttributeMasters = ibanMasterRepository.findByFkPa(ec.getObjId());
List<Iban> ibans = ibanRepository.findByObjIdIn(ibanAttributeMasters.stream()
.map(IbanMaster::getObjId)
.collect(Collectors.toList()));
byte[] icaBinaryFileXml = createXml(ec, ibans, updatedEcFiscalCodeIcas);
IcaBinaryFile icaBinaryFile = IcaBinaryFile.builder()
.fileContent(icaBinaryFileXml)
.idDominio(ec.getIdDominio())
.fileSize((long)icaBinaryFileXml.length)
.fileHash(getHash(icaBinaryFileXml))
.build();
icaBinaryFileRepository.save(icaBinaryFile);
});
}

private byte[] createXml(Pa pa, List<Iban> ibans, Map<String, String> publicationEcRelation) {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
try {
XMLOutputFactory xof = XMLOutputFactory.newInstance();
XMLStreamWriter writer = xof.createXMLStreamWriter(outputStream, "UTF-8");
XMLStreamWriter xtw = new IndentingXMLStreamWriter(writer);
xtw.writeStartDocument("UTF-8", "1.0");
xtw.writeStartElement("informativaContoAccredito");
xtw.writeNamespace("xsi", "http://www.w3.org/2001/XMLSchema-instance");
xtw.writeAttribute("http://www.w3.org/2001/XMLSchema-instance", "noNamespaceSchemaLocation",
"InformativaContoAccredito _v02.xsd");
writeElement(xtw, "identificativoFlusso", pa.getIdDominio() + "_" + LocalDateTime.now().toString());
writeElement(xtw, "identificativoDominio", pa.getIdDominio());
writeElement(xtw, "ragioneSociale", pa.getRagioneSociale());
writeElement(xtw, "dataPubblicazione", publicationEcRelation.get(pa.getIdDominio()));
writeElement(xtw, "dataInizioValidita", LocalDateTime.now().toString());
xtw.writeStartElement("contiDiAccredito");
ibans.forEach(iban ->
writeIbanElement(xtw, iban)
);
xtw.writeEndElement();
xtw.writeEndElement();
xtw.writeEndDocument();
xtw.flush();
xtw.close();
} catch (XMLStreamException e){
log.error(XML_ERROR, e.getMessage());
}
return outputStream.toByteArray();
}

private void writeElement(XMLStreamWriter xtw, String elementName, String elementValue){
try {
xtw.writeStartElement(elementName);
xtw.writeCharacters(elementValue);
xtw.writeEndElement();
} catch (XMLStreamException e){
log.error(XML_ERROR, e.getMessage());
FedericoRuzzier marked this conversation as resolved.
Show resolved Hide resolved
}
}

private void writeIbanElement(XMLStreamWriter xtw, Iban iban){
try {
xtw.writeStartElement("infoContoDiAccreditoPair");
xtw.writeStartElement("ibanAccredito");
xtw.writeCharacters(iban.getIban());
xtw.writeEndElement();
xtw.writeEmptyElement("idBancaSeller");
xtw.writeEndElement();
} catch (XMLStreamException e){
log.error(XML_ERROR, e.getMessage());
}
}

private byte[] getHash(byte[] input){
byte[] hashedOutput = null;
try{
MessageDigest md = MessageDigest.getInstance("SHA-256");
hashedOutput = md.digest(input);
} catch(NoSuchAlgorithmException e){
log.error("No security algorithm was found %s", e.getMessage());
}
return hashedOutput;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
package it.gov.pagopa.apiconfig.core.scheduler.entity;

import com.microsoft.azure.storage.table.TableServiceEntity;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;

@Getter
@Setter
@NoArgsConstructor
public class CreditorInstitutionIcaFile extends TableServiceEntity {
private String publicationDate;
public static final String ORGANIZATION_KEY = "organization";

public CreditorInstitutionIcaFile(String organizationFiscalCode, String publicationDate) {
this.partitionKey = ORGANIZATION_KEY;
this.rowKey = organizationFiscalCode;
this.publicationDate = publicationDate;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
package it.gov.pagopa.apiconfig.core.scheduler.storage;

import com.microsoft.azure.storage.CloudStorageAccount;
import com.microsoft.azure.storage.StorageException;
import com.microsoft.azure.storage.table.CloudTable;
import com.microsoft.azure.storage.table.TableOperation;
import com.microsoft.azure.storage.table.TableQuery;
import com.microsoft.azure.storage.table.TableServiceEntity;
import it.gov.pagopa.apiconfig.core.exception.AppError;
import it.gov.pagopa.apiconfig.core.exception.AppException;
import it.gov.pagopa.apiconfig.core.scheduler.entity.CreditorInstitutionIcaFile;
import lombok.NoArgsConstructor;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Component;
import java.net.URISyntaxException;
import java.security.InvalidKeyException;
import java.util.List;
import java.util.Map;
import java.util.Spliterator;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;

@Component
@NoArgsConstructor
public class AzureStorageInteraction {

@Value("${creditor.institution.table.connection.string}")
private String storageConnectionString;

@Value("${creditor.institution.update.table}")
private String icaTable;

public AzureStorageInteraction(String storageConnectionString, String icaTable){
this.storageConnectionString = storageConnectionString;
this.icaTable = icaTable;
}

public Map<String, String> getUpdatedEC(String lastUpdate) throws AppException {
Spliterator<CreditorInstitutionIcaFile> resultOrganizationIcaList = null;
try {
CloudTable table = CloudStorageAccount.parse(storageConnectionString).createCloudTableClient()
.getTableReference(this.icaTable);
resultOrganizationIcaList =
table.execute(TableQuery.from(CreditorInstitutionIcaFile.class)
.where(TableQuery.generateFilterCondition("PublicationDate", TableQuery.QueryComparisons.GREATER_THAN, lastUpdate)))
.spliterator();
} catch (InvalidKeyException | URISyntaxException | StorageException e) {
// unexpected error
throw new AppException(AppError.AZURE_STORAGE_ERROR);
}
return StreamSupport.stream(resultOrganizationIcaList, false).collect(
Collectors.toMap(
TableServiceEntity::getRowKey,
CreditorInstitutionIcaFile::getPublicationDate)
);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,11 @@
import it.gov.pagopa.apiconfig.starter.entity.IbanAttributeMaster;
import it.gov.pagopa.apiconfig.starter.entity.IbanMaster;
import it.gov.pagopa.apiconfig.starter.entity.IbanMaster.IbanStatus;
import it.gov.pagopa.apiconfig.starter.entity.IcaBinaryFile;
import it.gov.pagopa.apiconfig.starter.entity.Pa;
import it.gov.pagopa.apiconfig.starter.repository.IbanAttributeMasterRepository;
import it.gov.pagopa.apiconfig.starter.repository.IbanAttributeRepository;
import it.gov.pagopa.apiconfig.starter.repository.IbanMasterRepository;
import it.gov.pagopa.apiconfig.starter.repository.IbanRepository;
import it.gov.pagopa.apiconfig.starter.repository.IcaBinaryFileRepository;
import it.gov.pagopa.apiconfig.starter.repository.PaRepository;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
Expand Down Expand Up @@ -56,8 +54,6 @@ public class IbanService {

@Autowired private IbanAttributeMasterRepository ibanAttributeMasterRepository;

@Autowired private IcaBinaryFileRepository icaBinaryFileRepository;

@Autowired private ModelMapper modelMapper;

public IbanEnhanced createIban(
Expand All @@ -82,12 +78,9 @@ public IbanEnhanced createIban(
iban.getIbanValue(),
existingCreditorInstitution.getIdDominio());
});
// generate an empty ICA binary file and a relation between iban, CI and ICA file
IcaBinaryFile icaBinaryFileToBeCreated = IcaBinaryFile.builder().fileSize(0L).build();
icaBinaryFileToBeCreated = icaBinaryFileRepository.save(icaBinaryFileToBeCreated);
IbanMaster ibanCIRelationToBeCreated =
saveIbanCIRelation(
existingCreditorInstitution, iban, ibanToBeCreated, icaBinaryFileToBeCreated);
existingCreditorInstitution, iban, ibanToBeCreated);
// generate the relation between iban and attributes
List<IbanAttributeMaster> updatedIbanAttributes =
saveIbanLabelRelation(iban, ibanCIRelationToBeCreated);
Expand Down Expand Up @@ -135,8 +128,7 @@ public IbanEnhanced updateIban(
existingIbanMaster,
existingCreditorInstitution,
iban,
existingIban,
existingIbanMaster.getIcaBinaryFile());
existingIban);
// remove all labels and save them again
ibanAttributeMasterRepository.deleteAll(existingIbanMaster.getIbanAttributesMasters());
ibanAttributeMasterRepository.flush();
Expand Down Expand Up @@ -262,21 +254,18 @@ private Iban saveIban(IbanEnhanced iban, Iban existingIban) {
private IbanMaster saveIbanCIRelation(
Pa creditorInstitution,
IbanEnhanced iban,
Iban ibanToBeCreated,
IcaBinaryFile icaBinaryFileToBeCreated) {
Iban ibanToBeCreated) {
return saveIbanCIRelation(
new IbanMaster(), creditorInstitution, iban, ibanToBeCreated, icaBinaryFileToBeCreated);
new IbanMaster(), creditorInstitution, iban, ibanToBeCreated);
}

private IbanMaster saveIbanCIRelation(
IbanMaster ibanMaster,
Pa creditorInstitution,
IbanEnhanced iban,
Iban ibanToBeCreated,
IcaBinaryFile icaBinaryFileToBeCreated) {
Iban ibanToBeCreated) {
ibanMaster.setFkPa(creditorInstitution.getObjId());
ibanMaster.setFkIban(ibanToBeCreated.getObjId());
ibanMaster.setFkIcaBinaryFile(icaBinaryFileToBeCreated.getObjId());
ibanMaster.setIbanStatus(iban.isActive() ? IbanStatus.ENABLED : IbanStatus.DISABLED);
ibanMaster.setInsertedDate(
ibanMaster.getInsertedDate() != null
Expand All @@ -285,8 +274,6 @@ private IbanMaster saveIbanCIRelation(
ibanMaster.setValidityDate(CommonUtil.toTimestamp(iban.getValidityDate()));
ibanMaster.setPa(creditorInstitution); // setting CI object reference
ibanMaster.setIban(ibanToBeCreated); // setting IBAN object reference
ibanMaster.setIcaBinaryFile(
icaBinaryFileToBeCreated); // setting ICA binary file object reference
return ibanMasterRepository.save(ibanMaster);
}

Expand Down
8 changes: 8 additions & 0 deletions src/main/resources/application-h2.properties
Original file line number Diff line number Diff line change
Expand Up @@ -53,5 +53,13 @@ retry.utils.maxDelay=200
# Nodo monitoring configuration
service.nodo-monitoring.host=http://localhost:8587

# Scheduling configuration
cron.job.schedule.enabled=true
FedericoRuzzier marked this conversation as resolved.
Show resolved Hide resolved
cron.job.schedule.expression=*/10 * * * * *

# Azurite information
creditor.institution.update.table=ICATABLE
creditor.institution.table.connection.string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;

# IBAN ABI
iban.abi.poste=07601
Loading