Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: Included detailed info in reports about KOs on payment flows #161

Merged
merged 10 commits into from
Oct 27, 2024
Merged

feat: Included detailed info in reports about KOs on payment flows #161

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
9cd8546
feat: add details about KO primitives
andrea-deri Oct 25, 2024
6210146
feat: add massive extraction for simple migration operations
andrea-deri Oct 25, 2024
127a673
Pin actions/setup-python action to 65d7f2d (#158)
renovate-pagopa[bot] Oct 25, 2024
65f390e
feat: update on massive extractor
andrea-deri Oct 25, 2024
1ba8c1b
fix: set five days as upper bound
andrea-deri Oct 25, 2024
bdab6c4
feat: add report massive generation as GitHub action
andrea-deri Oct 25, 2024
d418872
fix: changed retrieve of week dates
andrea-deri Oct 25, 2024
e227ef8
fix: add actualization script
andrea-deri Oct 25, 2024
0cd4827
fix: update on report actualization script
andrea-deri Oct 25, 2024
6bd1b23
Merge branch 'main' into feat-report-details-ko-primitives
andrea-deri Oct 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 20 additions & 1 deletion .github/workflows/07_report_generation.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ on:
- generate
- send
- both
- massive_generation
default: both


Expand Down Expand Up @@ -187,7 +188,7 @@ jobs:
subscription-id: ${{ secrets.SUBSCRIPTION_ID }}

- name: Setup Python environment
uses: actions/setup-python@v4
uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4
with:
python-version: '3.11'

Expand All @@ -214,6 +215,24 @@ jobs:
REPORT_APICONFIG_CACHE_SUBKEY="${{ secrets.REPORT_APICONFIG_CACHE_SUBKEY }}"
python run_extraction.py

- name: Massively generates reports, up to 5 days after the date
if: ${{ env.operation == 'massive_generation' }}
run: |
cd ./scripts/report-generation
export REPORT_ENV="${{ env.environment }}" \
REPORT_TYPE="${{ env.report_type }}" \
REPORT_DATE="${{ env.report_date }}" \
REPORT_SLACK_WEBHOOK_URL="${{ secrets.REPORT_SLACK_WEBHOOK_URL }}" \
REPORT_DATAEXPLORER_URL="${{ vars.REPORT_DATAEXPLORER_URL }}" \
REPORT_DATAEXPLORER_CLIENT_ID="${{ secrets.REPORT_DATAEXPLORER_CLIENT_ID }}" \
REPORT_DATAEXPLORER_CLIENT_SECRET="${{ secrets.REPORT_DATAEXPLORER_CLIENT_SECRET }}" \
REPORT_DATAEXPLORER_TENANT_ID="${{ secrets.TENANT_ID }}" \
REPORT_DATABASE_URL="${{ vars.REPORT_DATABASE_URL }}" \
REPORT_DATABASE_KEY="${{ secrets.REPORT_DATABASE_KEY }}" \
REPORT_DATABASE_REGION="${{ vars.REPORT_DATABASE_REGION }}" \
REPORT_APICONFIG_CACHE_SUBKEY="${{ secrets.REPORT_APICONFIG_CACHE_SUBKEY }}"
python run_massive_extraction.py

- name: Send report
if: ${{ env.operation == 'both' || env.operation == 'send' }}
run: |
Expand Down
17 changes: 16 additions & 1 deletion scripts/report-generation/datastructs/entities.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,14 @@ def __init__(self,
"total_no_carts": trigger_primitive.no_carts_total,
"carts_completed": trigger_primitive.carts_completed,
"no_carts_completed": trigger_primitive.no_carts_completed,
Constants.TRIGGER_PRIMITIVE_NOT_COMPLETED_MACROTAG: {
Constants.TRIGGER_PRIMITIVE_NOT_COMPLETED_RPT_TIMEOUT: trigger_primitive.not_completed_rpt_timeout,
Constants.TRIGGER_PRIMITIVE_NOT_COMPLETED_REDIRECT: trigger_primitive.not_completed_redirect,
Constants.TRIGGER_PRIMITIVE_NOT_COMPLETED_RECEIPT_KO: trigger_primitive.not_completed_receipt_ko,
Constants.TRIGGER_PRIMITIVE_NOT_COMPLETED_PAYMENTTOKEN_TIMEOUT: trigger_primitive.not_completed_paymenttoken_timeout,
Constants.TRIGGER_PRIMITIVE_NOT_COMPLETED_ECOMMERCE_TIMEOUT: trigger_primitive.not_completed_ecommerce_timeout,
Constants.TRIGGER_PRIMITIVE_NOT_COMPLETED_NO_STATE: trigger_primitive.not_completed_no_state,
}
},
Constants.COMPLETED_MACROTAG: {
Constants.COMPLETED_OK_RECEIPT_TOTAL: completed_payments.closed_as_ok,
Expand Down Expand Up @@ -54,6 +62,7 @@ def from_db_item(item):
item_ci_info = item["creditor_institution_info"]
payments_info = item["payments"]
trigger_primitive_info = payments_info["trigger_primitives"]
trigger_primitive_not_completed = trigger_primitive_info[Constants.TRIGGER_PRIMITIVE_NOT_COMPLETED_MACROTAG]
completed_payment_info = payments_info[Constants.COMPLETED_MACROTAG]
not_completed_payment_info = payments_info[Constants.NOT_COMPLETED_MACROTAG]
rejected_payments_info = not_completed_payment_info[Constants.NOT_COMPLETED_REJECTED]
Expand All @@ -67,7 +76,13 @@ def from_db_item(item):
trigger_primitives = TriggerPrimitiveReportInfo(carts_total=trigger_primitive_info["total_carts"],
no_carts_total=trigger_primitive_info["total_no_carts"],
carts_completed=trigger_primitive_info["carts_completed"],
no_carts_completed=trigger_primitive_info["no_carts_completed"])
no_carts_completed=trigger_primitive_info["no_carts_completed"],
not_completed_rpt_timeout=trigger_primitive_not_completed[Constants.TRIGGER_PRIMITIVE_NOT_COMPLETED_RPT_TIMEOUT],
not_completed_redirect=trigger_primitive_not_completed[Constants.TRIGGER_PRIMITIVE_NOT_COMPLETED_REDIRECT],
not_completed_receipt_ko=trigger_primitive_not_completed[Constants.TRIGGER_PRIMITIVE_NOT_COMPLETED_RECEIPT_KO],
not_completed_ecommerce_timeout=trigger_primitive_not_completed[Constants.TRIGGER_PRIMITIVE_NOT_COMPLETED_ECOMMERCE_TIMEOUT],
not_completed_paymenttoken_timeout=trigger_primitive_not_completed[Constants.TRIGGER_PRIMITIVE_NOT_COMPLETED_PAYMENTTOKEN_TIMEOUT],
not_completed_no_state=trigger_primitive_not_completed[Constants.TRIGGER_PRIMITIVE_NOT_COMPLETED_NO_STATE])
completed_payment_info = CompletedPaymentsReportInfo(closed_as_ok=completed_payment_info[Constants.COMPLETED_OK_RECEIPT_TOTAL],
closed_as_ko=completed_payment_info[Constants.COMPLETED_KO_RECEIPT_TOTAL],
with_ok_receipts_only_sent_after_retry=completed_payment_info[Constants.COMPLETED_OK_RECEIPT_SENT_BY_RETRY],
Expand Down
52 changes: 49 additions & 3 deletions scripts/report-generation/datastructs/report.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,28 +40,57 @@ def extract_from_report_entity(report_entity):

class TriggerPrimitiveReportInfo:

def __init__(self, carts_completed=0, carts_total=0, no_carts_completed=0, no_carts_total=0):
def __init__(self,
carts_completed=0,
carts_total=0,
no_carts_completed=0,
no_carts_total=0,
not_completed_rpt_timeout=0,
not_completed_redirect=0,
not_completed_receipt_ko=0,
not_completed_paymenttoken_timeout=0,
not_completed_ecommerce_timeout=0,
not_completed_no_state=0):
self.carts_completed = carts_completed
self.carts_total = carts_total
self.no_carts_completed = no_carts_completed
self.no_carts_total = no_carts_total
self.not_completed_rpt_timeout = not_completed_rpt_timeout
self.not_completed_redirect = not_completed_redirect
self.not_completed_receipt_ko = not_completed_receipt_ko
self.not_completed_paymenttoken_timeout = not_completed_paymenttoken_timeout
self.not_completed_ecommerce_timeout = not_completed_ecommerce_timeout
self.not_completed_no_state = not_completed_no_state


def merge(self, report_info):
self.carts_completed += report_info.carts_completed
self.carts_total += report_info.carts_total
self.no_carts_completed += report_info.no_carts_completed
self.no_carts_total += report_info.no_carts_total
self.not_completed_rpt_timeout += report_info.not_completed_rpt_timeout
self.not_completed_redirect += report_info.not_completed_redirect
self.not_completed_receipt_ko += report_info.not_completed_receipt_ko
self.not_completed_paymenttoken_timeout += report_info.not_completed_paymenttoken_timeout
self.not_completed_ecommerce_timeout += report_info.not_completed_ecommerce_timeout
self.not_completed_no_state += report_info.not_completed_no_state


def extract_from_report_entity(report_entity):
report_data = report_entity.get_map()
payments_info = report_data["payments"]
trigger_primitive_info = payments_info["trigger_primitives"]
trigger_primitive_not_completed_info = trigger_primitive_info[Constants.TRIGGER_PRIMITIVE_NOT_COMPLETED_MACROTAG]
return TriggerPrimitiveReportInfo(carts_total=trigger_primitive_info["total_carts"],
no_carts_total=trigger_primitive_info["total_no_carts"],
carts_completed=trigger_primitive_info["carts_completed"],
no_carts_completed=trigger_primitive_info["no_carts_completed"])
no_carts_completed=trigger_primitive_info["no_carts_completed"],
not_completed_rpt_timeout=trigger_primitive_not_completed_info[Constants.TRIGGER_PRIMITIVE_NOT_COMPLETED_RPT_TIMEOUT],
not_completed_redirect=trigger_primitive_not_completed_info[Constants.TRIGGER_PRIMITIVE_NOT_COMPLETED_REDIRECT],
not_completed_receipt_ko=trigger_primitive_not_completed_info[Constants.TRIGGER_PRIMITIVE_NOT_COMPLETED_RECEIPT_KO],
not_completed_paymenttoken_timeout=trigger_primitive_not_completed_info[Constants.TRIGGER_PRIMITIVE_NOT_COMPLETED_PAYMENTTOKEN_TIMEOUT],
not_completed_ecommerce_timeout=trigger_primitive_not_completed_info[Constants.TRIGGER_PRIMITIVE_NOT_COMPLETED_ECOMMERCE_TIMEOUT],
not_completed_no_state=trigger_primitive_not_completed_info[Constants.TRIGGER_PRIMITIVE_NOT_COMPLETED_NO_STATE])

# ============================================================================

Expand Down Expand Up @@ -196,6 +225,22 @@ def extract_detail_for_payments(self,
total_triggered = carts_triggered + no_carts_triggered
percentage_carts_triggered = Utility.safe_divide(carts_triggered * 100, total_triggered)
percentage_no_carts_triggered = Utility.safe_divide(no_carts_triggered * 100, total_triggered)

# Divided by trigger primitives not completed on D-WISP
triggered_error_rpt_timeout = trigger_primitive_info.not_completed_rpt_timeout
triggered_error_redirect = trigger_primitive_info.not_completed_redirect
triggered_error_receipt_ko = trigger_primitive_info.not_completed_receipt_ko
triggered_error_ecommerce_timeout = trigger_primitive_info.not_completed_ecommerce_timeout
triggered_error_paymenttoken_timeout = trigger_primitive_info.not_completed_paymenttoken_timeout
triggered_error_nostate = trigger_primitive_info.not_completed_no_state
total_non_completed_triggered = triggered_error_rpt_timeout + triggered_error_redirect + triggered_error_receipt_ko + triggered_error_ecommerce_timeout + triggered_error_paymenttoken_timeout + triggered_error_nostate
percentage_error_rpt_timeout = Utility.safe_divide(triggered_error_rpt_timeout * 100, total_triggered)
percentage_error_redirect = Utility.safe_divide(triggered_error_redirect * 100, total_triggered)
percentage_error_receipt_ko = Utility.safe_divide(triggered_error_receipt_ko * 100, total_triggered)
percentage_error_ecommerce_timeout = Utility.safe_divide(triggered_error_ecommerce_timeout * 100, total_triggered)
percentage_error_paymenttoken_timeout = Utility.safe_divide(triggered_error_paymenttoken_timeout * 100, total_triggered)
percentage_error_nostate = Utility.safe_divide(triggered_error_nostate * 100, total_triggered)
percentage_total_non_completed_triggered = Utility.safe_divide(total_non_completed_triggered * 100, total_triggered)

#
completed_carts_triggered = trigger_primitive_info.carts_completed
Expand Down Expand Up @@ -250,7 +295,8 @@ def extract_detail_for_payments(self,

self.payments = f'''
\n*:zap: Numero totale di pagamenti innescati:* `{total_triggered}` \n di cui _nodoInviaRPT_: `{no_carts_triggered}` (`{percentage_no_carts_triggered:.2f}%`) \n di cui _nodoInviaCarrelloRPT_: `{carts_triggered}` (`{percentage_carts_triggered:.2f}%`)
\n*:moneybag: Numero totale di pagamenti completati:* `{total_completed_triggered}/{total_triggered}` (`{percentage_total_completed_triggered:.2f}%`) \n di cui _nodoInviaRPT_: `{completed_no_carts_triggered}` (`{percentage_completed_no_carts_triggered:.2f}%`) \n di cui _nodoInviaCarrelloRPT_: `{completed_carts_triggered}` (`{percentage_completed_carts_triggered:.2f}%`)
\n*:moneybag::done: Numero totale di pagamenti completati:* `{total_completed_triggered}/{total_triggered}` (`{percentage_total_completed_triggered:.2f}%`) \n di cui _nodoInviaRPT_: `{completed_no_carts_triggered}` (`{percentage_completed_no_carts_triggered:.2f}%`) \n di cui _nodoInviaCarrelloRPT_: `{completed_carts_triggered}` (`{percentage_completed_carts_triggered:.2f}%`)
\n*:moneybag::no_entry_sign: Numero totale di pagamenti non completati:* `{total_non_completed_triggered}/{total_triggered}` (`{percentage_total_non_completed_triggered:.2f}%`) \n di cui _KO nel flusso del Nodo dei Pagamenti_: `{triggered_error_receipt_ko}/{total_non_completed_triggered}` (`{percentage_error_receipt_ko:.2f}%`) \n di cui _timeout per mancata redirect da primitiva di innesco_: `{triggered_error_rpt_timeout}/{total_non_completed_triggered}` (`{percentage_error_rpt_timeout:.2f}%`) \n di cui _problema di conversione in NMU_: `{triggered_error_redirect}/{total_non_completed_triggered}` (`{percentage_error_redirect:.2f}%`) \n di cui _timeout per mancata chiusura esito pagamento_: `{triggered_error_paymenttoken_timeout}/{total_non_completed_triggered}` (`{percentage_error_paymenttoken_timeout:.2f}%`) \n di cui _timeout per abbandono o attesa da Checkout_: `{triggered_error_ecommerce_timeout}/{total_non_completed_triggered}` (`{percentage_error_ecommerce_timeout:.2f}%`) \n di cui _per errore imprevisto (nessuna ricevuta inviata)_: `{triggered_error_nostate}/{total_non_completed_triggered}` (`{percentage_error_nostate:.2f}%`)
\n*:envelope::done: Numero totale di ricevute inviate ed accettate dall'ente:* `{total_completed_receipts}/{total_receipts}` (`{percentage_total_completed_receipt:.2f}%`) \n di cui per esito pagamento _OK_: `{completed_ok_receipts}` (`{percentage_completed_ok_receipts:.2f}%`) \n di cui per esito pagamento _KO_: `{completed_ko_receipts}` (`{percentage_completed_ko_receipts:.2f}%`)
\n*:envelope::no_entry_sign: Numero totale di ricevute inviate e rifiutate dall'ente:* `{total_refused_receipts}/{total_receipts}` (`{percentage_total_refused_receipts:.2f}%`) \n di cui per esito pagamento _OK_: `{refused_ok_receipts}` (`{percentage_refused_ok_receipts:.2f}%`) \n di cui per esito pagamento _KO_: `{refused_ko_receipts}` (`{percentage_refused_ko_receipts:.2f}%`)
\n*:envelope::repeat: Numero totale di ricevute con invio rischedulato:* `{total_rescheduled_receipts}/{total_receipts}` (`{percentage_total_rescheduled_receipts:.2f}%`) \n di cui per esito pagamento _OK_: `{rescheduled_ok_receipts}` (`{percentage_rescheduled_ok_receipts:.2f}%`) \n di cui per esito pagamento _KO_: `{rescheduled_ko_receipts}` (`{percentage_rescheduled_ko_receipts:.2f}%`)
Expand Down
31 changes: 16 additions & 15 deletions scripts/report-generation/logic/clients.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,23 +76,24 @@ def store_report(self, report: Report, type):
if numeric_data is None:
logging.info(f"\t[WARN ][WispDismantDB ] No numeric data generated. No data will be persisted for report [{type}] of date [{report.date}]")
else:
try:
entity = ReportEntity(id=Utility.get_report_id(report.date, type),
date=Utility.get_report_date(report.date, type),
total_payments_on_ndp=numeric_data.total_payments_on_ndp,
creditor_institutions=numeric_data.creditor_institutions,
trigger_primitive=numeric_data.trigger_primitives,
completed_payments=numeric_data.completed_payments,
not_completed_payments=numeric_data.not_completed_payments)

self.__report_container.upsert_item(body=entity.get_map())

except exceptions.CosmosHttpResponseError as ex:
logging.error(f"\t[ERROR][WispDismantDB ] Error during persisting report in '{Constants.REPORTS_CONTAINER_NAME}' container. {ex.message}")
raise ex
entity = ReportEntity(id=Utility.get_report_id(report.date, type),
date=Utility.get_report_date(report.date, type),
total_payments_on_ndp=numeric_data.total_payments_on_ndp,
creditor_institutions=numeric_data.creditor_institutions,
trigger_primitive=numeric_data.trigger_primitives,
completed_payments=numeric_data.completed_payments,
not_completed_payments=numeric_data.not_completed_payments)
self.persist_report(entity)


def persist_report(self, entity):
try:
self.__report_container.upsert_item(body=entity.get_map())
except exceptions.CosmosHttpResponseError as ex:
logging.error(f"\t[ERROR][WispDismantDB ] Error during persisting report in '{Constants.REPORTS_CONTAINER_NAME}' container. {ex.message}")
raise ex



def retrieve_report(self, date, type) -> ReportEntity:
report_id = Utility.get_report_id(date, type)
report_date = Utility.get_report_date(date, type)
Expand Down
Loading
Loading