From 184ff7960878d8b00dd6454525dafe00dfc86fb8 Mon Sep 17 00:00:00 2001 From: Radek Buczkowski Date: Sun, 12 May 2024 19:18:47 +0200 Subject: [PATCH] feat: Power BI history details, table lists with DAX, plus new functionalities --- docs/power_bi/README.md | 239 +++- docs/power_bi/user_permissions.png | Bin 0 -> 48535 bytes src/spetlr/power_bi/PowerBi.py | 1017 +++++++++++--- src/spetlr/power_bi/SparkPandasDataFrame.py | 314 +++++ tests/local/power_bi/test_power_bi.py | 1165 ++++++++++++++--- .../power_bi/test_spark_pandas_dataframe.py | 142 ++ 6 files changed, 2475 insertions(+), 402 deletions(-) create mode 100644 docs/power_bi/user_permissions.png create mode 100644 src/spetlr/power_bi/SparkPandasDataFrame.py create mode 100644 tests/local/power_bi/test_spark_pandas_dataframe.py diff --git a/docs/power_bi/README.md b/docs/power_bi/README.md index 1d5a3a80..92af531b 100644 --- a/docs/power_bi/README.md +++ b/docs/power_bi/README.md @@ -1,9 +1,11 @@ # PowerBi and PowerBiClient classes -The `PowerBi` and `PowerBiClient` classes contain logic for refreshing -PowerBI datasets and for checking if the last dataset refresh completed -successfully. +The `PowerBi` and `PowerBiClient` classes contain logic for refreshing PowerBI +datasets, and for checking if the last refresh of an entire dataset or its +selected tables completed successfully. The logic can also be used to show +refresh histories of datasets, and to list dataset tables with their recent +refresh times. The same data can also be returned as a Spark data frame. For easier PowerBI credential handling (service principal or AD user), the first parameter to the `PowerBi` constructor must be a `PowerBiClient` @@ -11,36 +13,47 @@ class object. ## PowerBI Permissions -To enable PowerBI API access in your PowerBI, you need to enable the setting -"Service principals can use Fabric APIs" in Fabric (see the screen-shot). -Additionally, you need to specify the user group that should have access -to the API. +To allow access to the PowerBI API, you need to enable the setting +"Service principals can use Fabric APIs" in the Admin Portal in Fabric +(see the screen-shot). There, you also need to specify the user group that +should have access to the API. ![Power BI admin settings](./admin_settings.png) Apart from this, each PowerBI dataset should have a user or service principal attached, that is part of this user group. +Additionally, to access individual tables and their refresh times in PowerBI, +the class must be able to execute DAX queries. This requires additional +permissions. The "Dataset Execute Queries REST API" option, found under +"Integration settings" in the Admin Portal, must also be enabled. + +The same user or service principal must have dataset read and build +permissions in each individual dataset: + +![Power BI admin settings](./user_permissions.png) + ## Links [Register an App and give the needed permissions. A very good how-to-guide can be found here.](https://www.sqlshack.com/how-to-access-power-bi-rest-apis-programmatically/) - [How to Refresh a Power BI Dataset with Python.](https://pbi-guy.com/2022/01/07/refresh-a-power-bi-dataset-with-python/) ### API documentation: -[Datasets - Refresh Dataset In Group](https://learn.microsoft.com/en-us/rest/api/power-bi/datasets/refresh-dataset-in-group) - -[Datasets - Get Refresh History In Group](https://learn.microsoft.com/en-us/rest/api/power-bi/datasets/get-refresh-history-in-group) - +[Get Workspaces](https://learn.microsoft.com/en-us/rest/api/power-bi/groups/get-groups) +[Get Datasets](https://learn.microsoft.com/en-us/rest/api/power-bi/datasets/get-datasets-in-group) +[Trigger A Dataset Refresh](https://learn.microsoft.com/en-us/rest/api/power-bi/datasets/refresh-dataset-in-group) +[Get Refresh History](https://learn.microsoft.com/en-us/rest/api/power-bi/datasets/get-refresh-history-in-group) +[Get Refresh History Details](https://learn.microsoft.com/en-us/rest/api/power-bi/datasets/get-refresh-execution-details-in-group) +[Execute DAX Queries](https://learn.microsoft.com/en-us/rest/api/power-bi/datasets/execute-queries-in-group) # Usage of PowerBi and PowerBiClient classes ## Step 1: Create PowerBI credentials -The client ID, client secret, and tenant ID values should be stored in a key vault, -and loaded from the key vault or Databricks secret scope. +The client ID, client secret, and tenant ID values should be stored +in a key vault, and loaded from the key vault or Databricks secret scope. ```python # example PowerBiClient credentials object @@ -79,6 +92,19 @@ Available workspaces: +----+--------------------------------------+----------------+ ``` +To get additional information about each workspace, use the +show_workspaces() or the get_workspaces() method instead. +The first method shows a list of workspaces, and the second returns +a Spark data frame, with the list of workspaces. + +```python +# example listing of available workspaces +from spetlr.power_bi.PowerBi import PowerBi + +client = MyPowerBiClient() +PowerBi(client).show_workspaces() +``` + ## Step 3: List available datasets If no dataset parameter is specified, a list of available datasets @@ -109,13 +135,44 @@ Available datasets: +----+--------------------------------------+----------------+ ``` +To get additional information about each dataset, use the +show_datasets() or the get_datasets() method. +The first method shows a list of datasets, and the second returns +a Spark data frame, with the list of datasets. + +If you don't specify any workspace, datasets from all workspaces +will be collected! + +```python +# example listing of available workspaces +from spetlr.power_bi.PowerBi import PowerBi + +client = MyPowerBiClient() +PowerBi(client, workspace_name="Finance").show_datasets() + +# alternatively: +PowerBi(client, workspace_id="614850c2-3a5c-4d2d-bcaa-d3f20f32a2e0").show_datasets() + +# alternatively: +PowerBi(client).show_datasets() + +``` + ## Step 4: Check the status and time of the last refresh of a given dataset The check() method can be used to check the status and time of the last -refresh of a dataset. An exception will be cast if the last refresh failed, -or if the last refresh finished more the given number of minutes ago. -The number of minutes can be specified in the optional -"max_minutes_after_last_refresh" parameter (default is 12 hours). +refresh of an entire dataset, or of individual dataset tables. An exception +will be cast if the last refresh failed, or if the last refresh finished more +than the given number of minutes ago. The number of minutes can be specified +in the optional "max_minutes_after_last_refresh" parameter +(default is 12 hours). + +If you want to check only selected tables in the dataset, you can +specify the optional "table_names" parameter with a list of table names. +If the list is not empty, only the selected tables will be checked, +and the table that was refreshed first will be used for checking. +To only show the list of tables, specify an empty array: + table_names=[] You can also specify the optional "local_timezone_name" parameter to show the last refresh time of the PowerBI dataset in a local time zone. @@ -156,16 +213,27 @@ at 2024-02-01 10:15 (local time) ! ## Step 5: Start a new refresh of a given dataset without waiting The start_refresh() method starts a new refresh of the given PowerBI -dataset asynchronously. You need to call the check() method after waiting -for some sufficient time (e.g. from a separate monitoring job) to verify -if the refresh succeeded. +dataset asynchronously. To verify if the refresh succeeded, you need to +call the check() method after waiting some sufficiently long time +(e.g. from a separate monitoring job). If you want to refresh only selected tables in the dataset, you can specify the optional "table_names" parameter with a list of table names. If the list is not empty, only the selected tables will be refreshed. -(Note: It is not possible to list available tables programmatically -using the PowerBI API, like you can do with workspaces and datasets. -You have to check the table names visually in PowerBI.) +To only show the list of tables, specify an empty array: + table_names=[] + +If you set the optional "mail_on_failure" or "mail_on_completion" +parameters to True, and e-mail will be sent to the dataset owner when +the refresh fails or completes respectively. This is only supported for +regular Azure AD users. Service principals cannot send emails! + +Additionally, you can set the optional "number_of_retries" parameter to +specify the number of retries on transient errors when calling refresh(). +The "number_of_retries" parameter only works with enhanced API requests +(i.e. when the "table_names" parameter is also specified), and it will +be ignored otherwise. +Default is 0 (no retries). E.g. 1 means two attempts in total. All parameters can only be specified in the constructor. @@ -209,12 +277,18 @@ If you want to refresh only selected tables in the dataset, you can specify the optional "table_names" parameter with a list of table names. If the list is not empty, only selected tables will be refreshed (and the previous refresh time will be ignored). +To only show the list of tables, specify an empty array: + table_names=[] Additionally, you can set the optional "number_of_retries" parameter to specify the number of retries on transient errors when calling refresh(). Default is 0 (no retries). E.g. 1 means two attempts in total. It is used only when the "timeout_in_seconds" parameter allows it, so you need to set the "timeout_in_seconds" parameter high enough. +The "number_of_retries" parameter is handled in a loop in this class, +and unlike in the start_refresh() method, it will work both with normal +refreshes (i.e. when "table_names" is not specified) and with enhanced +refreshes (i.e. when "table_names" is specified). You can also specify the optional "local_timezone_name" parameter to show the last refresh time of the PowerBI dataset in a local time zone. @@ -264,6 +338,13 @@ entries for each dataset, depending on the number of refreshes in the last 3 day The most recent 60 are kept if they are all less than 3 days old. Entries more than 3 days old are deleted when there are more than 20 entries." +If you don't specify any dataset and/or workspace, the history across all +datasets/workspaces will be collected. The datasets must be refreshable +and workspaces cannot be read-only to be included in the combined list. +To exclude specific PowerBI creators from the list, specify the optional +"exclude_creators" parameter, e.g.: + exclude_creators=["amelia@contoso.com"] + You can also specify the optional "local_timezone_name" parameter to convert refresh times in the data frame to a local timezone. Depending on the parameter, the names of the time columns in the data frame will have the suffix @@ -297,15 +378,119 @@ df = PowerBi(client, dataset_id="b1f0a07e-e348-402c-a2b2-11f3e31181ce", local_timezone_name="Europe/Copenhagen").get_history() +df.display() +``` + +The "RefreshType" column has the following meaning: + +RefreshType | | +--- | --- +OnDemand | The refresh was triggered interactively through the Power BI portal. +OnDemandTraining | The refresh was triggered interactively through the Power BI portal with automatic aggregations training. +Scheduled | The refresh was triggered by a dataset refresh schedule setting. +ViaApi | The refresh was triggered by an API call, e.g. by using this class without the "table_names" parameter specified. +ViaEnhancedApi | The refresh was triggered by an enhanced API call, e.g. by using this class with the "table_names" parameter specified. +ViaXmlaEndpoint | The refresh was triggered through Power BI public XMLA endpoint. + +Only "ViaApi" and "ViaEnhancedApi" refreshes can be triggered by this class. +"ViaApi" are refreshes without the "table_names" parameter specified, +and "ViaEnhancedApi" are refreshes with the "table_names" parameter specified. + +To see what tables were specified with each completed refresh marked as +"ViaEnhancedApi", you can use the show_history_details() and get_history_details() +methods, as shown below. They work in the same fashion and have the same parameters +as the show_history() and get_history() methods. +You can then use the "RequestId" column in the "get_history" +and "get_history_details" datasets to join them together. + +```python +# example show and get refresh history +from spetlr.power_bi.PowerBi import PowerBi + +client = MyPowerBiClient() +PowerBi(client, + workspace_name="Finance", + dataset_name="Invoicing", + local_timezone_name="Europe/Copenhagen").show_history_details() + +PowerBi(client, + workspace_name="Finance", + dataset_name="Invoicing", + local_timezone_name="Europe/Copenhagen").show_history_details() + +# alternatively: +df = PowerBi(client, + workspace_id="614850c2-3a5c-4d2d-bcaa-d3f20f32a2e0", + dataset_id="b1f0a07e-e348-402c-a2b2-11f3e31181ce", + local_timezone_name="Europe/Copenhagen").get_history_details() + +df = PowerBi(client, + workspace_id="614850c2-3a5c-4d2d-bcaa-d3f20f32a2e0", + dataset_id="b1f0a07e-e348-402c-a2b2-11f3e31181ce", + local_timezone_name="Europe/Copenhagen").get_history_details() + +df.display() +``` + +## Step 8: Show and get the tables in a given dataset + +The show_tables() and get_tables() methods can be used to show and get +the list of tables used in a given dataset and their last refresh time. +The show_tables() method displays a Pandas data frame with the list of tables, +and the get_tables() method returns the actual data frame converted to +a Spark data frame. + +If you don't specify any dataset and/or workspace, all tables across all +datasets/workspaces will be collected. Datasets requiring an effective +identity will be automatically skipped from the list (effective +identity is not supported by this class). +To exclude specific PowerBI creators from the list, specify the optional +"exclude_creators" parameter, e.g. + exclude_creators=["amelia@contoso.com"] + This can prevent "Skipped unauthorized" warnings. + +You can also specify the optional "local_timezone_name" parameter to convert +table refresh times to a local timezone. Depending on the parameter, +the names of the time columns in the data frame will have the suffix +"Utc" or "Local". + +All above parameters can only be specified in the constructor. + +```python +# example show and get the table list +from spetlr.power_bi.PowerBi import PowerBi + +client = MyPowerBiClient() +PowerBi(client, + workspace_name="Finance", + dataset_name="Invoicing", + local_timezone_name="Europe/Copenhagen").show_tables() + +PowerBi(client, + workspace_name="Finance", + dataset_name="Invoicing", + local_timezone_name="Europe/Copenhagen").show_tables() + +# alternatively: +df = PowerBi(client, + workspace_id="614850c2-3a5c-4d2d-bcaa-d3f20f32a2e0", + dataset_id="b1f0a07e-e348-402c-a2b2-11f3e31181ce", + local_timezone_name="Europe/Copenhagen").get_tables() + +df = PowerBi(client, + workspace_id="614850c2-3a5c-4d2d-bcaa-d3f20f32a2e0", + dataset_id="b1f0a07e-e348-402c-a2b2-11f3e31181ce", + local_timezone_name="Europe/Copenhagen").get_tables() + df.display() ``` # Testing -Due to license restrictions, testing requires a valid PowerBI license. -Because of this, testing must be executed manually in each project -that uses spetlr to refresh datasets. +Due to license restrictions, integration testing requires a valid +PowerBI license. Because of this, integration testing of this class +must be executed manually in each project that uses spetlr. Recommended integration tests should include all above examples, i.e. listing of workspaces and datasets, checking a refresh, and possibly diff --git a/docs/power_bi/user_permissions.png b/docs/power_bi/user_permissions.png new file mode 100644 index 0000000000000000000000000000000000000000..4453a074791ff25e2f97c7879c39fa8f0690143a GIT binary patch literal 48535 zcmY(KWmsEH*S2vf+Cp&%Rw%`aTL^9~MG6#$Vg+sp6nB^4?(R~GySoLK;_mJm>*F9t$YcX6vu9?_I6AfPg3je+*%u!CUGTL{{KWh<4v) z#1YCyDfZznP)xm8?~ zrkV7{zmydAFaHKCp*h{+Bl#fAlk^Ca&wt}Jjva}4QA{_;VP5*qXm_4l_94i4Wc6U<85 z!otF^BW)>E{5`+T!ha5PAae*04}ZV>_b9-`*!W61Pf!2mh?A2OhQc}j+JJd=54N=Q zQemMpT9#<{^q9?*tFM%HYz-I}^h>%(HkR&ND7s=Y6+i-mJskPYe^m_`Cq`Ei{dfOG zrgqP&YRJ@)l$2Brc{l6(-r$~1hVF;cQm&(;edM@kM5v2a>B#En+4!gI+`9JkbyP?G z`Ql3OM6tBf2Sf)GT~bo8lr>HBm!jSTJu%19Y$1%&ckrOD9BZU$%{`q)?)4NMxw8^3 zggf0X)7jiOIbAvbqj|G7Yoa{=#bw!N8aFz9##AOsUspmxqFJ+_AW*RKU9uX~0%seo zj3>QK=gaSZxD8vX#J9)hU!|pa_V%tP-+r9b9wL*E-q~L%=Mi_4EgvyXbT!*9lPZdv zC7V>hZ|asf#8wxg%O$GeJsldExG75<&3M=z7$eXt<*f-lqNY%^0^vd(iY_dqwAhZL zY|Sh@m=?6Wq@~@)7amKw@rJf*9lpg)$u!wJ_B6=`%L_Kbs8uE0!3@nBqhz~6-)^_= z-PUj}O+v=CJe{)XjxU~~Cz{7qc1_p|eQ#3%fs@ z{e0#>P2}uho*B|2TS!$F^57cJ7)RIoXHcQX*8qgu_|`qmJ)bEL=k35@@gDY~U}_fp zx3w-E7m(T$o))aYt#`IPLpHwDi&9p!{FA|UQZkTe-6no>e8~d-duK;gUi&?zuZ(OG zf_|SlIh)8mZLU@AXBL=eZ?q0$Gqj*1=|c6%cWWey?&7Hfqi(;O-o>%8<5uN%MbpV> zHJyg#M97iPsxT_B`YB<#{^!Q^N5XU@bX3$j)52VTEJJv4;IG%q^9a2RxdUn*arOv^ zjUGaSvT<=!R8fnYAkR|;AF79JJv7_mPy%z$w`+6)##B4=?@chi@0EWqsIZ*pG@3cz z>`x7oVWmrO$3~itdM*!@C`C;6ynr?^y%BfKjE;_;nE$vz%JSeIj;A6(L-nDt-PS1g$GYrA?UL57S)+Vog#(uO%a zz=1usJYb$P^ji#bqF%hIx19UWb6HNt8Q#5Cn=f#eJs5b+(fG2t~7%HXT9 z&K{8s+I%K9)jYZPA3r6%#`4d5TGBErDA7G7GVw$6mn~_k&T8`y_hBmmzzkAhOiTTz zXie?if@oc>h8WIv-4QIqWl6zC_|$#3M(Nuya2d4`bAJ9u;X!qPK-V;TKz7-Qhs>xI zlQY<(iZKG$FMEb;N)AXVuHB9u9&b>_>iUo1#>rpVV_6i*Ay>QpjOOt)-#M#n{){&@-2kp@3f6Qny+&3w}|=sw(+m5Ur!k z7$@;x%L(S*nM<+W^4GrI2U|+SDag0;?_uJW=hZ2tXjgSeg0s^z>-;kY?$1#2gOkzu1n#%eT8z~FqN_gk^W z5UWnZxL#_+Um2NEuKoYMnWl_+%!?2+qi@&mcE5p6$+-(eF^f4&CfBdHTHN*VwY^<= zI&p$HwOyvKb7kJk-*p3?@%6l(93EP+;?)sAE6N%YR^cOCKfWB=G{00;fXolHKW`t` z8@;T&)AP9ghkIHc=76Vlia+OZyG(4={^Uruc1@Li72KYQ?{I-f!EN#a&kh&E_LiY# zPFrtrt=sN@B@sHI!=%J2%qD2|GHc&>S{WgMjCt*X>hT{cPtS+N+KGmwK$dXckK3@M+PJBzTzZ1;d$6>Uv6Co z-*maO{TWurtM7i^`@5gg$$xt&F*VU?5nNBYPU*B{bA#nK;G~urn%binLY)1qy$YL| zFE)rj31+?v@g~z^9rt-`ak=bTeZJwdixs(uj`Sb%II%P6#^JMMsH2ex^k=P`>k`z~ z*7tPY+%L@JP0s(C7?`N`f!U&^VJ7GS*M?7ayQ5gSGLgKx14uY**DY6*UwJVR zcz;KF9*_c^EAp}Cf<2~%!3*5gp^QZ@iz1U1SZ?NcrnFalP4qGzd)`S*W7oaJ`eu`j zP~+{vbEU@1QK4?nhhyTKdDBxfzSz78hU2Zl*l%NDq*F#(W@Qq`@}1%x8GI|w-A6|9 zXP)~a^U|6~9vyyI0!J$3SL3;-0}^P>Q@mA4X6+YDxfefsL*Dc1?PXC;%cLd?AvBX% zQ)OKV9GjGVX7e0{)xoaX&}wxza|J-|cfYkmot7OE!MZdc7%VSV;mu}Wcr~Xl6oYai zbC|F1ee-z#zkW4{>SH3ewd$Q1}-^?-Fok$XA*r>iM&qt=MjdSzS{ro>1%O z$9rB;#ti2i;dl~!2AEU(!}8g*isF1Dwe9VSn{^V8#Si|+v+izFkq1rk@shG8XiJm2 zzQ@P__%7&eWq7JTK!)9}q_j?Fs1BXV`SauK%Y&D}h801#KPJJ(6SDL`obXPPQ5U+% zU1!C1f;O4Yei-tGLDhI-oNKu`OMfq{L2 z@I`@X{h*{A7Y}$Rnj~auMIRAw03YDkfV&R#?vv(ID}PaKQf*yxf-8K^lbDNrm!YtcyqoI>5zS~@3tNz1ufkUrY1e6|t!SB)jBwtC zfxhjkA+~DwzKFI@^S3?~i&o9^Kyki0QTr_wbi7dl1^&*1Eo9Oi4F8~GVA7`F3T$#r zQ>37Fc-sMUU*n;qBYFzY^9)(LTL0-l9jxb9Gp71O_R<1%6dBKchbS$#gi+YXEfOZn zX3E#A&D}@$ZCB5~(0fBmSg51NsImfVvO(!Et$f2Rji-o5#7G4QrBVzU95c+qHg#e7 z+#-sx;f;*@nX{vG1%9);vmsPuH*-dKmnh@CNCryOGO~eEK_k;&T^4*4fc3!yjAU{V zR6K&fhb21$_#%LldAG&b-3g%}s|z;s8X6^_a$@`RR(=>HD4G z3<9>hMO0}bbb$SD=dt2bOrajn4St{Jd6Qk;VWQgjO4Gljd@b}rWV%tY8Fx|a=2rm8 zn-cves3i!tmwTKo;bpY@LmO;DKPYkd{BX*LL=Qls%dE^9ZGYTm=M{0rUl+4%y`FKb z@kx{cNL-{9bB?}TDZUU!h&(%2oA}Ln4kUt}Myx%JZrt}I09CFe04lZ`oVdP9`|I;I zj0N2u)V%0^zm)5S^FH!L_btTQL59>hWN8lu&9#S#{zaq%qRM-Oh97@Q?&B8tBBV~m zGVqXWrS@xmd&j1dq*%Kr&Q&(4qx7vp!OIzm5TZv$N$dmj3QH? zZ@t!@4CLu0TUtLvi+a91h>&Pb)B}H_^!YwRL}}QL(1BrhO8b@lRMj^Z<3E+p@H;|< zw&PTWrTb8P`x3OR8hZyYtjWrWVvx70o%cdR*_sAmv2C=Ot^*=3kC$NbUJaGiBIVp+ zf`Frag1R~#%`l4iC`cg|=7vj*I_|#s;77!&_0I0O-_re$#@P393FvwUXZWWBZGZE= z9*X|V2vPLzeatG9oSpCBZ_H;wB9*^8gNRsSWdwxF_=j0z=b;;jO0s~URUb8to!`K- zbQw632ER@mKg6CugC7`Ol86HvB>TDf;7 zS^)w7BQ;Gk&?CosN^F13Z`_A}7#bW5RCX2mUiwWc&hFUUZ%BFmv$^2iitKKq_G6+z zY%0UW`#`mPybsg;;W*(23)AL1!MZn*LtT)u$BKDd=wRVo;p*2!HZGy0@lWViC;<)p z5(c5rZvS$u^?5`XevHom^@$&@;J_B5Q7?&@bQIImsDz9pdJq>|6sboq@aJNPDdhHM zSL9`Wl(SGi+0e8xiYH@lw*C3~EL8uos|9pTibn~HmV#}wdkH*VmwSO%*cKb4i@-YX zPlvMfr}26)hxK<@?C=e{nF~(O<6OL}o7etu9t@2hDcZY@{!v{eTR223k~s74iv(DQ zM)cEbuZllp>2yJHp>~md4he5a9L}*2JQP(OJYF`s6$T>#T#Gl7{Z%=JuM`t#A00hIq=fr@O(y^~LcRMYf)T?( zmR+4y!GX$V`4HN*-i=uY5_w$5>xR!vMz_gLo(k%yi4C#L+3jnLCYiQugM9F54VX1J zdX+Px-nSpIQJHEjc`cywX-h7sWDKdFU=wS~z5MK}J}gG}ehq;gBga&zFF`94^sXY4 zdjT5z{#3HZn>Y}?(6urE6oBHAH9q?vUN)#87a8yOZn{&j_DEzZrL%}WY$LJzbsw?l z`@qVwy)R_BK@kI)IQ4pQqr=BQ$NP~L?hJ+z67CuJ?#fW$MWPe##*+~s)lEqgy}Jcg z+{Lxcw>g^}mQ?>}3Gc=gVtG7$c`kQxtU2|?Vyif2+cswe{Hpx0Bk>uAZyk*u(kGDn zfxCQ9pDG6(;9f!5y_cqP8P;ahW_vS?tb^yeTHa*Mth$1JbV2rwrviRD-|FH8^ny-y4R zf5>bzl)sOH*hocO>cW!1taV~}&Y?AloO6B2rBoCvSOt_wE^j>Er#v8C2DWW906t8;jT*K8JNZC(BP>LlAuHO5q{m^>z}ZOshJm%HPI*8%V|2^h2Lh4l zF}#DPb*n}CQ9li8pn%3BJ}nh8eP1JmDGv*VJ-&u}6t=MhsEkMZCxamRw0nmSc4&C_ zj~0+e5@N6ogO4ZfJ(G-}Eel%^3LZ{QSGYPk#uX|RB~}V%b~OtT{mmIBjp;XMPRo62 zVOb>7t9KevSLoWjSJh8l$T^hmJjmj{C})PFz)SfD`b1C2oH^bN*M!-_heRDiJWL?1 z@jf5?8>8aN{Agw8{5vsyggUm^MJ?q3vJwR5gTRe0$vM}Pl%3h;_nW^ZP|}aI zru_=XNDlU}ato@CbfFvPtyP%~t5u$sNg>W-!T_xh$pQ{T&SQ+a)S7rAV%`TWHn3?{ z^{`wSGv=b6yrmMlo4u^S%B{-y>P%VZ0-nND!oh1ZzGY?-RzJ@z6IApzbr#Rs2!z7>NZe#8WiE zvlG|YS;LSEkN#>(CJgc3vJmlUs6|xa3EM1j=L1bHubHR)uhq%|60W{@mn%}i)R+sgF;h&mj)kDRj_eS|r%(vSyqFfedaVzq&+AXTcDO51oSjnM4qbEj zGJ5#susBbK^AGW7;yfhVMp#qGW8BB3jgn z6Zd$z;O9Q_ovF~Z?IAJO5PLP1AG5Yq2YgwA_NcCtciiPe{jFh91^Ff(Pe>*he-cZ4 z$B;9AqFjV#RZ;1whtRqlsk#2wJn8QGNBg#^z#*>s1FG$(64!V(=T_2IEYBmt-KHB$ zYw?MnT%;jHvfADaj!Al$IDp;#54} zlPSTx#Y19Mz56!5))xCA+%k$fyV0$~xAf%=GOqg5yyc+{llxys=dbp2vN+8J;!!8p zH)D3cEP4ob5(p}}P%qDw^WLpZy`QDHq6QR=66!IELas2OQ))^A){uj79@f|iL1?&| zY3FQwp9lT{Jm(ub6<3tK$sm++-cQzT6hTuE)AUY@RQJ`l2m2f#PAaQ93AnktC>hIq zLXM(qm2D$ZRS`LJct204TGgKz`^4n#vaMdCa@M3avr63SQx3WMQYC8MrW2CGNm|<( zDQ4<}SIJ-ab?k_6vZ+yBOU>TZ(OIwyZUe>=4tmZ_b6kma^(fKQj$0BVqh?mDGLwcmgM> zE;>JIn}N(=Ek=eLA>;H&3Qq|toePb(I9jHgKBhX7wq{*rn##S%`*#;TgK)a~@l1Cp zr#!<9Ye?;E$;gF}UP%c+cDHs}qka>AfAK%Ii#eveksk-K&n&oADe&cN#qU(ly5a`h z$VTBZ_VNi3A z_Ob2YpK};o0^)Q=cV$&{lBeO5$ZLt;Vt{x9p`y<#s7Tbh?vUN z)L$=>$)&)nJ(xE^8iIFmaZ&I3e`1dB!$3Dt<@4tO*N!>|4j#Jqq! zCZLyDCc4jW^+wH0bp5@HZ1MQp)$-U;TL*X#Tz%_cEwzylNN&2n-0iRVo>K<1m!K{y zWwm+2?S;lOv;Nhn?DF(8gjeT0$^Us%Mbp|`v1r%*ZKTR>?Ma5?@28*Pm%Ii~Z8o)s zDYl{)iXu-ny%X7d_=n*AJObzKH0b^87*A|Er*JW5kB6X0Rk!f$O$_DiD~xlw+~FZl zjgGFg>a-hhJuEeKSH}5>-h-9?#Sx@y6HO^8)AFM#_itF%<8U(9$ewbp^t{{Vzt8^? z;s3U|Q!9}D*H{L)r=}B)oO@d}kY6LTLMZYnXrS4r5d)DdOe_>*^Msd*nmD%-8(qqVN z4P5%^CTvi571VoftN6vf^u9xB?54O=Tv0UQ^K0LID&pFD)VRHJqy2Ze4k%G@f%ap-lW&l03$G?bCP1zS7qjcp@9y zc5wCUN?ujUQQqJ7qf^98^tkbs1zRKtW_AbPh@YMXE4tG@UL>|-ztXq-dNfNy&B9pn zsNKPqq7f-`Qe7+&w0-`Kbu~O_4R(0+V~RveWE)j62VUI4-ce)5giBE>pS2!EdjfJ2 zoBM@_7HroLrOmW{*>O?RlXd*o3Qi-dLstI;j=zcpXE%%15RaRA#xM$j$+j$v`{Rbq zi1iL+yqd_`(5xGqe>X|&ocdl**9D6FaOT_H6HJh5f?!h9VaDYGV$HS4;y}3)HnN%3 zbqYYmBJy`?J=EomaBqIj^22>YCmm*%z%24(kJ%+~i(S7B2)sJ$HaHunF_;Z*PN^rM z-M=nB1Z*F4U9tlKE3V9=t*u|}*dx6b&zJ1R=2O5}8O|HQM5~o_6$}n2>+RO)?kxpi zmb2KhY8qSAG5BsXNzmax=YFA;w~lZSSn=KI9t#$f!!*}IGZLojkf#n{@7Si?u(dCY|>|<|Z+SdcgyOy0+m%Hi~l(dX3Aag4k zjM@G?`yGp0o9O{%MY%jRzq;Wy3x$HIiTwUxVNZn_ku2 zRa8M$I1OmdNTbPKPNT}%M&5L3P~~H`F>^D;c74ynhwrMRBW5C&*IXP>n)J{4VyrH% zI=w}dIRA#;8~TpdE!hseBDxaAo?1Avf*5oM2Jo%AS0{TL=i2Ap>{C&?e%T8~$gVWI zb6Xa$+ex~`n9mNi2V!>RNQDlw9c)o)BC&IcCNOh8j2i?3+<^fx8p7|nMNjyp%1271%YmZ*~T zpRSBNd*OoW#HDe4>G9qYgE7t%Tw4_xb??DvUOM_%lEcaECOpTX}vNtmP|Pqu;0 z?Ul3`y{N$x792JV2la%~kyMoM@h?*H)v?md3XU2-{~AuJczJ#x_bJx-aK`vw+@kyC zNH^@T+}gi>zaW)tP}gzZD%gIA!lR=*Q|IA_DAd^)&3x*}Od6 zk|R#Z;9m)?*yvo*LnMREZV%Mf;hYT%4|MgLzCkv@29e5TTG=3S`8F(%96-fTD+s(D zuAYc(;jpai{qaBEWJ5twZ%*K}IC9Vo7gUa$Y_~(q9vIe>j3O!>_2Ejw4SrVmw1`0$ zO7-zSb$uRNemJ@7dX*Z^Aylpdqr&@KN5GSY&A;sbAUZSxbzxk;hAX(cUfH47g@jkj zGwhI8I?nW~oIY)LxA*b3@a>XOXo^7f)6Jr`J_xQztmh}{aSFhaZE{7<*NP+27o%mU z%(bNl&SJi@Rq|#Lj6K6?Hb$h#bHhHI=95x(V0T?t{&nW^&MK6YXvdRz~(mYO^mnt$Ng<7I+ZUtt|BI?K5V+ zaq55md#Vm2pMXP+o@f3x6zTShrQ_ow6Y`-u#1W~w%3*eak2?F>NSyI?mDi@}4m0ya zcH{KtN-22wlf;a?mf%X*J}Y}yoZ$Nb@&9!tCo!{fAiJeFEcbD^`)$_gCk|Sr#SIEk2)-HwC?)m23>iH;6>Uz9c=2$wL7k z!L8jdrJ#ROGo|p0+z}cffhRmY5GhEqIi)DF+~;!>1`t^ZN<25Pp%yY8z`wj8wFc6` zM+VKPGTQNmv-nS-dAIVpf%p>;!MO-NoHoY0al`8XfSE|U-D|#;3YP|Wzd=&1T9}|X zxTg<=ULo6#*ww#A-wfr9z&B5&^Oy4O`01_(#g2iuGh6jtCFL!Kcz?Sifwb6Ru*@(! zv+1G<={^XBt1SP)UT)VOM{d`+Yy@1d)SfedP;T6%S>hGLSIdP4u7msvD?fqiz@-wu zna_6q0@6Pf5@ytb$1!My2Opf_8)YU*=VL^)1hnRS5b2kn@2ZP52w7-5P<999cRB7S zHnXL_WY!aC4j|DQ_=tUBF+>pi&45u1Jv8e4nTn1wXy`{#-Uj=R?!6@~LJPXR^627s z;#NMQ$8J0kin7L=6j}aqsVC-PJ7&4k&ITK!&5hNT`GjYi-xGYU!?K6y&{ey!);a_l z{VStWw&o+lP_nKp))q5F$|!XIa0)DdbvTzS=+TE|B?xP!oL}ow& zMe&`I2yea-`%f%9Z#d5Xumo3CfAG0hVv(OwZ(?H|hLZz$mz`th+Ml{ZOwO}8P5z^w zOYic2_RG2D1)czTd4y1j4wLo<%fd3av{k9(@p>9vMtElZiE~S@nDqpWZhAH&1LP_r z(eWq`-0zvM z+Yq+PkCdt@PRp`E&65xZQqxc#5Hp2Rkke`^H*x4CvWMe^2J+u<04BAUNgg{@2pt!~~oZJtme9vc% zV;-YrmYK3{aD5yc+G8y9<7_+rg!qGb8{2laj5(DaIu&{-n)p0?d>pX^m%hWI5Mcc{ zHT18knQb_Qi*GntNW?0qI zyDrT+b+WvQsUjW-{0}HWMIX1VqvT%zGb6|6K!y^`-Bg(Ta;Mhp3u)_G?C$n>z9-JH z*G=MeaCN>6A9M#1qRB*(e0M*Zkm>$2Hw)isy(w?Mk&e?RJKSPsvH|8|PUA5}gdTSm zT+gmj*V&9!JXkeDz;!FU!{M8cH;XF6H^&>J#(g#W2KQC(*WsHs?9`rPX1jcjg$PKS z;6X^PQJtwh?t;TFnV(UZ^0W+Ir9X43Q2nUKJ5ge+Sm6>azoIhH+eHj$g_;rK?=d1I zJVf@-0kh~ydzL~1Du!;oMCjVo2tg?9-mCzufZjgd6ja{~i}Kc)IvRZBmA-g&J#&X# zKM8B-hBCqh7_rawoFr-74;>GO9r$9{uybH>>_?0N7PWjB6*@JW^~BeCdRaK zm``$qr0ScA1f`Z{&it(b-fmDROrH^c_u)^38wZ%d&Z^|YbMru_U zUB9K!zR=ER(;N!qGfIVp$D^q+Y2ofLg!%>A56S;Ng-_LVeOvrS<0$rU+I6BrZp18&Bv^{CxPONYV&BRh! zqKv5dwxhFd*i`jhYts^VnIPGzDK#PWbo5Nb%k@sh0-3CF2EHfs4xr>wm{UoaXBFWR zt*F`9rn~GU*HuhaHL-tMeBbeI_1$0Uk2s%H|N38na<$Bs7CC4BC4dk2iYO`zQc_;Z zmlswbb)IBcclN1}PsaVov>!m;fAL082GkYv@O_b&9=LmHc#&>(=8q_W2i%Wlud-mMr=J@V zqN%aXNQ1nWCA_A;l1%;nGhJL-Is>m(%tEK<2i9NzH(%but#^#w8E-64N(W+v>2gUn z=){0MwLfaUM5@Adx?E;R%|(WM*ek*y25%>%cyz8DiNPtzU-F_~cWyqGV4!L@eCffntmSDs6tnM-Lr>e|m6A#Y~=3 zt}M8Ot?0_eSu-fn&THReuNfJ&p+HxaFL>f!?Z9~W@AQdwhZfzFTZNSPoKBXA*g|iM zyxTveGaCo+_1zvdT%;4X;ppMy5#aAeR^9&+7Tm{Z6UuGLKzZqb`v{Au&QK21+Fdu; z_3kFUv^#E(en(vi3za>8?JYJfbi>Lx3aJZ6I;XAL43X)}e4PH`sKbFo`FQ`{i__SP zaqr+*?ks&y1w1^k7yR@1Y}eU6xnU1S>R--}Dy|q6GralWv#VXh+O`(1QVHo-G(?z9 zRCTE?F6DYGzj00vhasl4k0^&*o2qbLqO=KmNOy({+r@RL2m|9PQ5)m%3)$LhlS zv)o5y=g;*($yltecILa+TZJ`W{R*K7%;Jx#W@F(|q$)*J!&*!e+fQQPfJXTkHnKPe z?_iLR{Hr6{11b6cb6MMC&7W(^$$0iLdr+3bLpE9#Q!`fIKV;nc2kVO+6=i4&Tfv?A z-WWCEIlVQ1{?@uNV%8$13CrfoTzgp9${P=yKOc>20K#{}3T;~3SziCw9u`Jxk2hax z6;?wy2nBm-HyyHKeiM6v=1VvvpTIrdY@aBLd17^&aE^hq^UdF2?BwypH zJT$wUCdRx~d8$Q?xr>q=`1Zu@rnrJ|dE)0{+zm5>HdHjGgj^j0la27p%{b;=22#{L z=oy5H(RHuIXh=)<2?pip6Vt7w*#zWL3kz5XUe-`o5lkR@sUc9xIpY6l9~WtRl)EJ*pi5UvBRBE8?-+&k+xV=|I1~UI6JH`t0Hpdn$8ytg*_+(vNtm&|C8!=6%_I>8j_H0`)H?{{I>O5&M%X zYuyM+v{WWlVp=WSw8zj>xJR`dN(K7VY9sfMMA=Z zzCNkula($ej+};+9vliSVU!pqtJ#bny)lq}N8esFcz~g6;L-b!6h-l8+htt&CBj8q zf-i9Y+_uSji)_2$o%hBEQfPuNH2xsigAnj0oFj@+);ffC@v3GsrB3Za2lJQ2RJ(L& zq7syOzVQ#DVNNTB++U_s@Z|fvQ2Db9U^3!u6{;EE`ViuvvnI7Bl1Y4mFQ<7EyEZ2D z_WI3MabzRWAi{hpwTq~ZgY%i_xP;jDv^7)QDS$SmU-aIbV!(4g&(S&g#MctJO4;$v z7lMlBzoxS;LFv6j4m@i8a+Jr#p*Hdchh!>g(HvCA^))*W|FZR4_Vrg+x5&6Za;WEr zWce5lJ@t`WzxpQmoG=V#{%RV`bAom-3_5*L4d6c5gp*DW!L?89McvVIL!rODxBHnT zXW9E`;@=@AAo)+qSNx!lfrV@v+#JnMjOMf27heBj>z zA$q@Mh&uh(Y}En~$X^mFb=LGgk(KHTvnkd-tk-Gz@W9WTtvzdZH2syXSmyO7y_+R@ zwN18MK$%%5zk9@YxF-wP6-3MZhkO~@Vk^l?xSegi-W9-$H>d(Y1jAQb+owWqn4(Sj zSRvC1jV3HsiGC{H%=*bD{k`IkEQO!6_3W!tM=>Mg{9U-e+l+){J{_5^7Z+w!xiQei-fV=j z<0=Br{lS=2u6xKL-B87q_4C2kSl=#m{(UMdNgeQecvc403HFIH>5|b9$l{;* zj4Zia=78}6Nx6Q-O{QEGdbAl2#{AxnR}X|AIaz2OyWG^6&WM4#XZAz#*RN;w@vw+S zBh5cN$gyXug1|y~ub7X=?VY)&0JW)y>d(a4Y^lVZ2b&E&Scep`E+LisnHU{zUpzWx zmNneFvbZ?9$bg=@TWEBtDd*wEAoG&TZaOcK?^T|SXqux^ zhJZ8N?z=!if3od;LNg6_x(cvp)eM~Y)N9+6fm$i@60gtTuEYgrq*cCRR^j(j$H$OD zi7XY&rrYd~isslrnh$~hRRofdgn4FzV${<{D`hlwoP}(VdhnYU^3L|8rE`Q_Pm#OY z7Ej60)-d~c)r1h_BbrZ|m|qE`w4=OaMhx&CIN7Oomc0b3$$=C&k~ z#!6;9z{*6-IeINu&cF|c7rP(eB|c|aJ;I77%Rxv_}O4<@NzG{t%qj{Jf-c zCWYE4dh9L0g@^x9Q|WNBf0iXOu6Rih^QXzw1qs0w03Hu)SwfII(qmo~?{SJVR`t zr9#Ah=ZXE@NbZoU57`bu^^#9uAPZsK$g_3ck1vQ*@9 zrqN5p4()K>w?6rmou@pk?z*Hr^hSrYtodYN&NQ|gZcytknxN|rx5<8aI`xVKJ5w(- zM!&_#(t~Zn&11FmkX-@K$Ggw2S7w(=dqPLqZSW#+1|)^7<$T3K@0-rT39>QL$?$r< zpX1wwFlM?>4*1=Co~#@78f<>b*uCm~Gy0bx<>cS9M z>DO2Z7nJ6J1r#1laOucDTLPj5T3LNHdK_mfV8oyXgzvR%gfOQ(L~>4lUH#5(bC3(Z zA}}fL_d`d4k)p$*Y$*AEXVr#ZwBL_dTT;cyta3f?(czF65xAYg-K%HKt50l8HvN>m z5WYoAu0!j-uttkX&A)AnRF^QMo-oVcOk=Xp&tTFddj zjC#r5v-)1G->%@GaTq90M6g0b5G*$)Y{Nk5&*}B-E~LIWjNLpvf7G`-mPvfle4A0> zz8_O@86iZ5_;T~&N#55d=h*XBCQ!@|;t$!#)2_4EOtK%#JCm%rP0EeHhEUi??Hd?y zRu&^SCr7bB;n?}ZQQe|d9lplER;5EXu4O8>c}%+c1X@e-Yw5Wl4RXAs7g_U0PC0#$ zVz+}(*75l!^V=BCxrdI>C5xDco}6{#VY!`FCwhpkGrr4%_w~b~_*RIFgov1G^J$wK zKirWEZgzk(*Ljs0zcy12J-ZPe132F1eHatirJ&0~#siKIT*A%Typ69|u?~WNqbx&d zvi00A@;re7sa*3xiB|65-@1Aa{$JNB%(Z}5?sWVS?ooHXl^Js zlfdmyH|uaxTysM^#6Okza9p2x4KPC5vmS`&#=zJpq z?xIzr8i@{|K4BuZ)9A4M4@VdY|3?ZU^#3HUoKHfGIUG*@R&WaXhJPyB;@Zi8h+{f=9%mAbknzU*<_iFMY~$xRO$c2Ce-{IfB9H-l1qDk3fO1A?VyhAu z(1)V_g1B*5$_Jt#Bt9!6a0-X*2D;eo26W>J4_^UD78`80KWWEB0v~1R6X+~A#H+T> zr==i$zKFu>{sdt<8nP^g415Hd*OO=r53i+fyKpg05(=65Thh(l1Cwb}${Ss|E`U%3E)$8u|D^ETl%^T$hnHeqqH%aE*~i`(~f`Rd765d%X%mn2uUoZmX5HwiQAYx%a2Tg2w+JWFgme4p> z5-O5ViCC?lL(swjWV8z>Du+6L_J@8GgE9VM>cm`HCLPDYUV|_4yV&q7z!zkge=Ll` zs!RK;%u=`_-o1sQOcZSJ{ZiFy%xz_=rNQewR6hyb8~Y;aqYcKVh>UC0UC6bAO=T|= z(!JQHZb&4CkmTu(smP0oMx!qt&5Y+l2|zQ-yDj#Z#rfdfe6)0j0@4bBIuX;;;QO_a z@!Kd2SGmGdCal3pD_D$YA<*3WCM&k8XP^C|v7VRrnyx1Ryhj~{=Yuf-f}Dx;dk?@0gZNh+KbxH6<-SPP=FcmM0WtsXEL;nU;+h^j* zmGqB)^lv6kw~tg@v8q&d9T{elx~n$W$h5`^9BN(O^DVpE@pI$Oj7CQV;*wF^BsDa< zu+*>ExFvC#{?MQzJGTCLzt+XXEGg%PF_gew&B~bBRfS`y-9faek_n4*LD5R8$&6Ts z`kP#E6iZ>F$VSMG(Eo}NO|T?R8UX6jh}5B_+##8K)n_%Q4ceQuB5ciqXF((QHaQcQ zy&V=3y%XT|pgkwV#(j{N?i3A(Vxe+OIxP5B6>Mchn~%99eZrB+fh|Mk9#A=zNlORBpYL z4>yvBVwUA6WL2V{m$z-V!i#@0;S<}jjai@!pY^FnlV+RRp~Y|kjl-tk25y}GI9jXT z-Ft3;(RkVUKAQ#TL{`g7{m1XK$3acS?`Pi%IeOx^__5x|<_sjE=y#YQ6jUe#p_rWnvDLJr z`g77^{w!}8=ogtcoHgZvbyb$JPhH=a*aO3JhyOGnlsB(0{GQEg?ryzZC1;Sipd=zj z&%oLJHZ9jdH-01Q4dfqcg3Gf$ z>t`B@Z}euJ>mlI;MP(WzZu;Sb(Nqm{w9SZ~$oIKyUgKSU*V;LC~bu*Mov$wA>4R)$NtXn&+P$b|45kwEZN8aLcT6x@Ch{U?> z$9(9!D)z}AB&GC(c&z^$4nB9L#CO?>TE3}KH2O;3^RD6DufE>Swf$I823jBfSWNbC zet*KfRzR&M-`E zdi&}-bL6@y;xKX>%R*LXfw*2rxT~CEDUyO?61KTAhfa|uVQ6Wg>7QR0?`8pCg)r_X^Sm{001 zF>6ArKN9in?@2KVYE0|!E&Z0L_JQ%A=249f*z2Kl>sR1ZWv$*<_zFk~U?kbrAPbC6 zsz&gyfL28EF3X%+hoGMTO2jBb5=dQQ#HN9B(k@%nbU>kl4Kdl5Nj>9A$jzk>7WnnZ zsL}tU>%79@{NgyTW{5I`sL^{JE$Zmeq6Ucw20;)+8{OzFx@d_m2#M$|dheol(d(#V zbRzck-`!{TV)w$``!w zYBhIUp}9!@)O6IjOqO#FjW6qZqh`A zwduu}0qm|;2TlJ~I;^VdnHVF|C(d1QVTI%M-`t*qkTSnh&G|9dbH?!*r}E${>vq@X zMB~@v;xk`m&`6-X-X{7;J1zA098XlH@9w9VKgH2U1nq-d#F2-Cm=X|(;RkgigT=%$ zouIPd^M~I{l-w>W6$Tr;A4I}%+A>(oI2_K-uWwTFmP2#A-TYho-g??Fy8E!!HjI+s zUiui6o`1J%{ie1}6NL5e$5#mvwn9`4N2!m=M0-j!gZ-xPM?57f-6E*Y&~QiET7d4u zPuszSQ)+25ES$^WZyUe3iBw-Xx2N(9fRrv|G_B5(G*1Gx6(4Z^hCB~w5h-~zDo?FI znpAp?X$#G-R~YAIY+;FT@O;h7%t#x>NXy0|0Z-vcWais@VJWr(L*O+TTPW@~Qo$jH zj${VOwnW@jr8!1@S|)u@(auz&W@c1oFq6a!GU1K>FJd#NYCOlP!sn|YVFaIOjb#}L ze{LssqEXl?P&!rifNXI&ve^y03py8aenn7!dZ&I@5r^Q-01{QT&3ajK)X92u zGOblQmSShuaIlsK9)*W!O4U?dF$8#~#6>r~gtYWp5*rca<`%62?sGo`dO8Na?fnpN z_fmBA9UAx4)`1bO8-zN|4wH5Ar#p~-tfES0e^=1Ne5tmPp!I~IPAF0g7q3j>_bIZJ zZD20^X4sH?fp2AVAek#@?p3I7f<}$iV4w_o^Kn^zmGqfcj!GbyP@G_$@MlfKMEuwJ zYI2yVmsfVIue$EJBzNwcdN-m)mvvXWsHX8{V2FWK)BM zj4}5~4R^nC@4=NTgMkwoFYxk%Ws3h|m(a-ZfVw<9nYiMOdhbj0rlqcD^TVVUUSPk< zGQlvaDj&ap&JOXl%7v}o%7v8{0z0m?^3wHnhV z!?aY=%8Kh0@hPq;uyLKjK@8 zGk`J@Y_b2rp(HT}_w}J2D2tD9$sA9o(WD)3Ke>p}cWo$V_AbWv3`RrrV|ey7(YSB-?PC&1B@3=gi*XI7K~LeJ zOfKKqsfa#n<1p6ZYiNlSeHzI_AUW!Ld0Hq-m*urZJ!q}VXH@%N%gfOd)edqnavJ2T z3sHwOi~Kb1cg!v^jG%E^>qEi}G<8h8zG9)(PA-s^seZVDaHP+Aj95qZ-QR5#phuOk ztshbH8&kCa;!T)!He&klWonL3@0LzKKJ(*I zj+5;X$8%O+=>Vb1M3Diqh(s-kxe7RP9OZv^;lDbj!tm4!t{yi)QjUnBv)k2aLF+>d z;9SJ<>y&7qQ!oit{u?=UzY|yIFshQG?_43lFFR%Ti>yEF@XRa@$lSgT|e#B0uR3GXSH~(vgy6sLnU2Uc=yiTYE zq`6t+IY5-0t{j_LNMCU{KfbOSmfW*vE+Kj#r|e>kRK-dRT6Wr)nY5EP@Lo&Ff4*Af+0wBnhWCB4y&(DDBM9NCw1xUDH*R7$7!nM&{ zMTg_2>)qm20DDm@(?;tF=>I?S`awJGu5CO%z=aN#G1S66yUJoPwrkaOG0%3_5ot;w zZ29U$`Ad0c7*SOvFEhm*ATOCYm(Jo{=32K&%jU%J?*nWPmHSYx!`WLFWHmqnftCB! z1KauzBgf{)K>xp4hrw;xW4U>&-Rz_YNDao%#sUgD?;kP;1hRm#1E{J4$K&z_8;%A> zg4@z=L=8p1ER`WA@(8cjYA-l>;xo2)3b;~XLGK?2ggR{<-mxM&!ZdwGNTrB1Vro{ z5JP{M=o)t%$88yI!~E=Xq%RgdOi%uHey}W0mofXGU_2{k(L+e9J|AQ9Kli7&MqeDO z-u-^K-WylvZ7b$xA%>iNj@Yr59<}ysY9JeW*%KRJ5;-W-joV#6{mh+N%@gL&JAcy2Mz?ghH8J zrkIavSNDheS_js+?!J6w4zf##|5a2?XlVD_90ac=yYShl=mwyyv>IiSU`Gkc40og$ zIgnDG5I7g#RZmHncT86doucH7#d%)w#?|uUulK}85%_e@R*tjgL2sWeI{>Qh^nbGbWXbxZ_(4PNs)n;Lz z|4k)l1_V8oPOxLo1#4%qopTaF6gV0Z5s}BCthNmFQm)$ots^t$@oh(^1$v`@nz-jP zs12ym+afUf$;ufdc01ken-3fI(t49CKtpb-X?}TeS1)|%N&Eq}j=iQ1hv*nMDRgpd zdtqX+C~QB#{%nTAvXJpo9Cc+(ZH7gs*v;#^S^P<1mVy&Tn`fVBJm}J;enJ2II8Dt0 zyX|oU&yUB88by0$tujyUF%%)NafNUS?+ zm1<(oER6iz7SCKO99R^kMwpArfBYHcNe7@dLXd0pl@@mQ---nVi?_mG=a;hfrP>65t_d_C02m!5&7oe3{L?v{`Y66Ul66CUD>um?X zlW_Dl+&{&Je%n&Zw%)Hx63#!-? z)M9mLL3v>5r{Dx-l7lYudXx4)5c_dzyIy7l$4*eku;#j|4xJe9T;_nlVPK z6`0ah&PXE+39A>B5!0Ri1H>_0KV9yB+Y6$i=x*a01pJXIOPn}as@mOg zoIf7a4wKAzzRyEZ%IM;j$qw_EFi|rmEo~;mD|EVkh6&?IVydQ9TBXA z&uEWf&ctCj&h+>0GVTt~^k#l$C@<&kdz?(pV)bJw!wd}Fy8(lix8U5W5t#)%yhk~j zeN1t@`CZ}K(N{jpH?SKs%rZl_LE-^S6C{|P1{x3{{BHKy9m~kekU>$}pzwTsSbz@d zyya%`DC~Y0GtJ%;;vivdHG3Bb_HBdOt3J$0t=M>eCx6=Yogt2VJIH$n`0S4>3)Os?|Ez!N$JJ;u zEn=x?DuxH2_0y~TtkP3t7yCiBn*RKfW;utDqsf7_4QHgwg=xJ zwf*12Qqf9TeSPlo2*z|bTp2oPoSrw??_A&*T&q`Dr)Uz%djgE2;1+GIFFG+>M0>X? z`}U`_YC;}tK*96RAp5n(dYMRDXQb!}sw23Y$D;A@0K37UF=@ED^gRIri(zzm5??hn zHVb2#Pt-hb|8!5opeE6$rja2LY>>>8srd)=U=U={ejY@B)PIRAM6PshLWD;0z6ho)quSoC;%GW?q6Yddl6 z#S?;+12hw+Yp9=Y=IjGQXJd?dJ!N|Q>m6bCD!GQ9sLJZ_&{7b#cTIVHW%mu>*qNW5 zjH;jE?}U0%Yj61aU^oY8VdZ4ngQ*qKUKo*oF~N*2|F8qsk!Yu2ogM6+)QUWrn8qA; zf*-U+CehLcXd9^|HrrYCRwqpd%3p=BYpsi<_9|;oNceMqXZGLMjLV<^I;Gqy`Q?xP zUD9jJi%Xj=QGP^|Ycno#a7`>!3ErxJ;Q?R>)?l_z2wvZDb*@LeKN4tDzS zqaAW~mTmg#zypln5sCzDOQuh*!xFv2Gxtr51>&#aXpIMW%30VK_IctAXfk2}YWzN7 z@txUw5)(i%QMm~S=Hos8N&pEz;LXAa?}0Wm8BkCy;b1>ZAi0|x@p~ug(xNP}8)8;& z>JX$c3^3@VI=>=38F+;<1Ld{;jHEr4+NW_~5b`WK>j(Dm=C?!4|IW(%tZ+0IGhPUe zi7;TWYWin+EO{w!*dv&Bff)Bity>S-);^($+)2n)u)`^)_4~pMm&h6d6XP?Iv0qL- zN;f-VP1M(9)IE{^0tLKgb-)ClWeORtsz1AmUw;yOWP|Nsp6gn!ot=wq5)Js@84_Jo zPMcW=Y}W4f+Fxiroia4vvaCe@0k+OI3~h3ZzE+`EVa+6~j(#V!cGzIO9%7@231$1Q zgaCJw96HN6a{jsH7I0@QEjq##(mdu%R-X#>QW-$ydouXAC8=2O0{&T9D=eO}xV_y2 z0y>tiSl~&YqX4`5(YettbO`k}QBzr4zu*1rgK=bi?|2gXwT;(gFi2@HjJ;e0QQpk1 zj7yTtYN7}NS;?H$dR*a-kJkQtL-gcCB44Ar-cOYv#cvig8e?!MS>Ayy4H)|Ka|H=M z9+zbwA3ODY0ELfR+uJc3A#rGa))tY$k5;WB)L)HC-(muK_jvsy&Lg*-ACgHNhV9D} zy_F2{#A?p3y6Yb&zKNG{4A#Cdz;bDE#M7V{PX_fto?~-w%V&*%#&e;`dd5O>$bT}S zBZ*CiFH^1q4y9Q+(s8==G8H6lxpbv%5iQZD>ROf0$&$7ytK)&zJMLZ}8dkQ(&Ecet zIx7$|!Eds(5A0%+1lFbYGBUZWebdHXNeoU%oK4aT3gs>b6LN>^iNT2Rm82qp-H#XI zuElxBYzex4hz!%-KL1p%4;&*o zk?)TFF7bWd>V! z@%hZu9&Sm$nNfY^pTzAwo`&aI7xdhNQ+o+Nx1v`7eU%+m+4fdm@?d$64+q*WX{T#g(;~yS?_``0TWAt`dTofa>c! zS*N{Cet({sS6nyM+sw;3_Y&5)@%Zl=7IK3(XyE$S?r8(Yg}vClhq_Szmb8iJ`+?HM zr{t19g`0?aeUB^Si2q%x7XQyJW|Y2BW>yi=56;+Dg)_eb@fJlzHT zp5El`Txfq#H4F=d42mP@D>w@fX;%GKoO^x+0;?WLU%#~QEFmgU?iu>%LSp))^jEMI zX;Ki~Xt#)}2If(jKRL|tvdC#3pChc?o{f(mAw+wYbaPspWU!PaQkJHsqh(s)&7G@T zFB0d2bttv1n6b=h!fC)BI*#B}sE4ndaQ8nBY&k!>wkv6f+=x2m{JmY^++J80g7?ky zMS)}p`ufelO9gHN2MMt0H@;0D?l*Sof{JiS0dARka@>TP-M4K%EQS31Av=>5b1}BX z2Rv-+DzOibB#?{1*?H+<;$d^|aHWlzE2r%bJhzAe{m4Sj0lhc%LI;5}hNZ;WFfDea z8W%3#dY4VMiL{K-H2U19(h5d{Uip%3X(I0&xI87g-N3$S+LV#w&Aw^bJ-5x_XkBXm z?5#6gIM6CzKsmul>n?I(c{rWM-lP=|> z^|}0tVl`m#f;bOEjE`*~5)#uJ4Zb<5_jp|ixBYyI!Ztrdv*qTt3`Y!;8E}!H;vu5KyI} zU5GMLopKD=+U37F%x%5WD=ldd7Lue*OZW0i87;7^3)>0N{VDiCovE>Br$Y0Ku<#%6 zkwtSKJiZ`n>kW0eFrGN_ z_cBE6X9AQnl^1@%i{1XZ0F}~tgC=>6Of-aHVC4SJkcIq#Z_}-w`*4Id(*KejihKqr z*HRQ!N41uPK)^o^0`(Jfc}ksW~MLiztCl$)cZ|J6vJ{SZ2D~ z@#{+Kg8l?QCjH7CHf?`%j3)ZB#!vr%H+5KkFnJ-=tUk(pmv(PoyA6ul=R5%-#ql{H zv5B6fAh5gnu`=kRkNz>L-g2=!eo)lcKge|e5F*QEyB@S;Kt={JTh2-x=NLZl@a5G7 zIbSg4)!kbjdLeslerM)+V-{Gg4Q(E~mme~(Y!k>Yg#?%D*CUS#MXsFB1Yryx9f!(S z&AV3Ge|zhBt{pqa_edl0xmk|1FJM^*b}tNbT;y24jHS0zZ8`5d6h*;QjH?Qiig5{j zpQwhlb+G?@^YYpaem70F@xx@)CVR15$oRG^c=gaa%f{Qw+IjA2%1?QdYAaR5UHM6h zQTY0Wlz_doC!DjHua(VHilFTGDj$B+ZkUJH@$)Az^UlUm8-g;u89RSZHj*RI`Wdtj z^^g^9LtEM8{n4ZM_&E!#x$!9Tczc)QDni{H-EN;=G^e85Wg{c8vjJA ztq$VXB#rER00Qp~2SZvb+FhO?I>%-PvKSh&x(bW6kSAnFM|h~5jJYLYr-d19#v{6z z(~YP)T&`m*r6#Ee8po)*@RgF|tyfSvKih{AeuZlQ$h4_hfFy`DyRHtcD1IN-?Yu|j zp8UaZLy4NfDAblHTQ8MPIGJSV2{#5~&lL`s$#YA*_+C6Cl*asSGBt0gRXUGOet~Fx z2d8XE-hX{<2k|dZc2WIz1!b7tNCgcIFPM7m@i;zQjVZM^VUmN2czthrkSX>Ht)RgU zF?HH1sk1loTIZpsqi514J4J21g~x}_^wPqZRQ->LI<8Y1xojY|C&AD^}xe$$r$OsPr1Zq^sd(Drp|bs zSa9E+JKnvs>@9A)x{{pBu?PTO3)gU6-S4Qp;Ddx4#H+!l1HLo$kpD4F9t4 zt{84~f*G`Pc^ULk@dpHp|oPInTIKn&nFvqxOaLcmDy!+6xh*3Omi#lz^C$IYWJ|qHn&++Hz(8 zL~5(#^BzO5^n)acbhPN+5RZd>RR|hVjceTDkn(yfDS#IvJndCU7KztW_%TBFD+_07 z@!I`DS()QOVxrvB5H6$Wj%Asr^Gts#r({!(6!YCA*7fj(skbTd=Ee`c?6HM0>pm^k z^4Sf84Aj#a@i;QLer0{lZ&AJH3h!Pj!OSeU+aE40R9fyWpqo_ z|7qD=eBC{|OYW@J|gZ2mN`zB#hOKviO_) zJZwU0R@78nM-!#JbqIGf_(%3NWhdge_`%e4Ud8)jO}a&TSr0iQxQ9D~N3CC6=}&^6 zWUc%U+PU-R(z9{HG^@Nwk6f9EI@NE` zR(m7IUeb!!B2&6~nyjuJm@xx<^j<-!pf0P%*5_Kbwtp1S$@?iX!1MCfQza%xuEG$^ z`J%aswwrJO8RbpW5VJq8=)(1Ws(adqERZMAp(mu0&P{7b+ObYC?m4A%v>0mT?dlg_p2>v#&C7rcz<3GO;3Ytmn*rRM} zI+|hjM6bNj8s5Wcf$$);2A$c8wEFvq`$-LSfFTG;sS1^_!qOjQg}_fxX`U*-l0-9& znd*muB(Bx{suAL{)_>lv^wzOJj@8-o?{RM@fb!`C)Hf=iPrS7i3b9%7O=DOQF=53P zL5(xzUnHXd+b68`_P}xVzTNqpYbq}^i2sn1?+y9CWq_`{3Y_<9PsxAi893(2{aL0A zbm1l#M`N9Hkt6f(0E_F+qr{z*3VVQj?w9BB8VbOu3p8D>MVTYV3sz&L4)HGjxW>gW zp+A{ayj24D${>4WQU_GZ$ym7D}MZcYP6<&fMQ~N@bmzURME;*_UI+@0gx63K7>Rq?PWbdv< z?vwd7aPTran4WR@YS<&%UOHc@W@_)V48r?$t-ayIVw&WJ8h*b*iqVYz-bZE@w|ndzctjh&g*{$AvY;w`Dh*~IU|y$e*+sj?<#=P< zIsryySaDg){Cp@c=9fwO9s*S`mAI4He|#8X*}qPEdG~;tyX&&MhkAutHNJ467G}SJ ze)>6Hlb@7NFau@C1K*dbQuN8li0foI8hLy*f;PGog^m75I1W)!u@kDKdIzxF?_mu0 zt&uR>iEoCIqiwdjm_$(Qp|>F<%}FN_vHn!^E!_arPJI@gujN*wi&(h#!(KA&3a*!8*NkHCO_v5{_oomJ}t6FN7kw z!=D@!@);uyvh>Yy#-RZx#sRR1jmb9hd49UfV{MDWNe z585SiBkU##_FbTvZLbZ(`;e$9S|JKW_pz7*pJ1&Il$_!csAw=Od9D+kINj?k3p6?( z0Kb|Lko9XF(ov68qNzyvn@|c1><54!C|d7y0V1NYyh<~+=NXYYf-XFO0`L;^D85=e z0A@vecOrXdft;-adJ7>y2Wc6OdUgvSb*b;8zZr_b`{EH$ z)No1vobFKjMS9Nvyed0X(4yz{Ey)eQSG_?M%HDEmq=|4v(>xt$dRFV7FMTp3yjtde zTRf#Gc~woF382a|rc_MFK>mN%8A=xW#tg7-Lkune`WB$Wc(z}E@{*(F3{DM7{WS#u zi^lR)#70Ox9^vqP4x2@+X(7ShEaIkhCZv`q<#W6_J ze`v}o|Fy17kQlfTI15(iXr@#l%!yhg}W*u`F!?zUcS$f?6RG>MV}4ajkt?377m5 z{4HjL*23tf8!BHRviYm`U?=37Wldef?A8;W#E)Kbg2QJ56u{06q^n8IfHc9t1`+K+TuT297%_A7el&!h0c-+iwqvTM~`fCf>TCgIN08DopYD`3lre{aqGBn;-J1JZ}~ z8-!?p*j!H}Lzc)PD)cAOAK*?IHdfrFFaQ0Pz=y=p4gdS)lp_xTG^UDF_Z3nObsiTg1$oyL>H=m1XXzk4y+eWFbEdMr)@w$5!> zN+lwE{()XkRZpd2f6CFS0w|o~*+G*UeDI;^gni$U@WgcxnqdkbEF1 zPAp+P!9rU%A@BZ6C<72&AqYO0#N_Un;m#aP2emuZnVD%huR&dn_%B^6YT;NAdkK~? zQuM^uclfUJrung<;&U8FHI2S1lEq5ya{Achq_?#3M=*3;1E^C-4KA=d|=!NOX~G$6i&svh3Yb|P^Ti(#MmsrmZRAzkoLssdrNQ~U|CP;%mr3uzzM61*&dRX z^fI%J4&1dQ1eEkA_x-V-+0-g{SVo_*s}&av+B$=UJfG3f&TD1A6~4BWWyJ}U&qt3_ zn_h#M=K-XU{XUAyW8SuKs|9m5R;Gp^b4VVl4E5fk15dHULC5Kphp2nv76WtKAO6X6 z1;}qz*E?kM(R|U69DeYeoZ)q3 zCmc6su=A<*AIg7wpIke|?*28R&bClDHJwTYs2Zw&d;=MeCt48444t?ai}U#vR2q`( zPd3_D%6}b?R$H-U2Wo&6qZETf3CS2fndf}bkAF-L%v=!5Q|q(%oaLNp~!V^UaTn#IwbM$+7V0fYVD!CSj~I zE>o}PWcr{SLaj$eH!@KO5v-o*sz3}Y|#&8qN zCq6&QHIuz^)!4qi~53RYpXdG!8; zXAbtKHiDj;9A2E?dPe9;-28B#>+sr&k`%G7SNJ*}lE+NXYGy@$BUnh6ZfUz@?DDN zsNXV<`dw#=q8RgzKKlT0s~oTBA+Rh*5lB^gUUZiw&e<$lA~g@O1!iI#>26ZrAlM}6sUG;8`@|*W6wV&9X;ZJc=h=LKLEu4 zsTuc?fCbXw?ayWzO!8^@in_iRBM?Hl%)a}<8>@-gfAckc>DugE-ZB{R6aSO2m)?5K`7e~-6$u&SCe77VRzH*9rfHU6ygAHZ$KE(|(KBwqq2|A~^cK@`JGON>?<@L8(NY^-SL*;R>m93-;crs;lkk7vhQWv( zMRzT=Y?6B_;u}fu>v81zr113Rc-idWjEwk*MYI*9?ZFl?PVdY7fzJx=Ev6q&M%yOV z_K7kliJn=?9lVgjFxaB7j=t}cA7HK>kXQ;Mx;rW3V!h74i~wU;$XWIDYk9_y5G+n+ zsoFA6H(;2;YImq>M`P?1DdRZk2g?+TQ<+G2MKXxkIhpd8 z_KgCn(v}*D7E+RYFJI z>8GxJCu&RO;Qy*e_vTst_l$v^mZyF6fF_LPj$yBy!gv#SSYTklRHAoQaQk~|dyn2` z3{QstQWC0akd=e}0r`EgzI(!Yw5K{~!Q(okXvJFSqwo+Cp1JEEKaV_14n~j5=t&#% zBp*HeDV;_a!CY2)`Z7T`@1~dd8!e0fQ_?raa6#fg%$ruzq#CWr%*SNA7F5zsHnUcH z2Z{)jM3o?8vdEUm@uR<=c!(yCY2->}@cf)uv$X`0j}xaAH>2PLGBEr}CMk?sC*rP@ zZ;t@H0qSZ;bF-eqEEpYm`f7r|uR0^+dU}t|RH)^HKkg?zjPF^@sx1+-atSTWH@Xs# z^<3wCnJJ{zM0GX4KU>@}H}hu^z9|*}Xa{@XSue7@jfvI=nh4r%?i_Euf55%U-%%QSP>;YI1NZi9Mzr6VnU(^S&f z>zNfW2g=BC;QAA=?Cf3DBU~VeZHtI3bP2d|Irtf(@Bd>auf{iiU`K{%Gk$AIw=_fJ zMMLU^*LUNGkg{K3uE=p+eeU!JabHDpbzug13w^OM4DC-O@+2bo8#*4oh};YKELCn^ z;{{ha^f#k=3V;0zdAQKI;U(NT&=>J7%E2<}V2A%e4LcbKp34AA0wx|eZc^nzNJ*zV z8GA_k*{vet&(hlOsj~NkEFdD{a5_&jDp9!2ue~Loj!7@qjvxOjaJIcPImZDuasm_J z1j)bn%~Yw64O$j#ak+|g+6NlZnRQF-gs-XlKUgVJNjT|hCT%&2-Njf4OZ_!q>B3hp`8&V09f265EIowSZEq#s%~r=4_8*W4Fvw*^GJ z;XVf~;fUjtX{jT;iS1#09%iZ1$jb98j%OT~p0sV@OxNVUD0UbdYdReP?{35$ zQhaDSm@dC-2ZaJaKxX@v&6L$?wXwW(VIGl{_NR1yH%GXu5ShAm6IGt|}VG~np zu<6B{Qa@__K~Pg zgp4(B_!_Td$-=hj5Pz9O9Y^B%H*bo^-5&dXAy3*Yp=rVs356u?>SVOqZh%(|>SgQn z=pAWRLo=Id`w~-U@n8!n)2mH@JAV%^fYDHg82uyJd3mnZvHaUHu1(sYhypD zx%AePC~Gt}Ug>r9{DRHcn!%sV;b}OxI`Q`K?{5A$~qTkXLii7 zJ8?65VTLch2WLaGqbUZbD2usKKMRTl`I_If@iMF!51Yhv>e>5G-TW|-=TBVLfs2%| zM@PKr5}O!ND{we!Fy+S+wa6e7`Ick?p+OgtgYpga=xe^E4(7}s>T;Cl zCk^MPQ?`pSS{#ioWtx<$G1Q{a440h*^UfYW^Zm+~=Fy?anlmrW_t9Znm!AucWge8K zPgKCJjy2M#sz%$7vBDj{3{~4YjozBq@_14re zRz}EgNSfJqzxF*$B`+K}+MBKUiJqD6C&-HQ@%2Hq=BJZ(7EQg32Hyq>`6zHa^W}-? zziJ|8<*d3|vnUYK&)mB?02-nBj4%HH_Y_nn9dwG(fM1q4NE)KHaOkN+Y_bq(*% zAN}Uq-Dr-eekr<(aF(+*#{YK|lgx41&_pPIQ62Y7d!myY*LE^>>tzFEe1dN0)Bqaj za#U>>{Vc?^8@wZ!>Ddovdzm)h%+A0O*Jk?Wlgtkdx_CO#6D$=!^?;dA!p@%C)1$vI zu53H}Zg)E!fjE}mYb!b9=)&xOOHOF6BVvEai$6}zN!IPC8+O6HMo70hPnF|S@R%&fY7(pCSli!W$zfIGQ`w4^#-d__b1JP@WaIB@? z9`FTl5zBr18-{aK zw_cESiisF(Tr5u&Tv2hRqJ@{&$O53}{i&nbUgPzn6fXm&y^xFcV-}I7(rm$%I!Vsn z3T(u1l)P7>=TnRJ8-s*!u=}-CVNC_3GqPV)} z!hTJuq$BapTba?b9>AER`9E7(2InWk&$hsovmL>Sd|Z2B%kwK%HPNI;J$cR5fqnqH z)&ISPD$Nnpa;QA}Vh&mNTyB8CK|apETaWE1Ru)7{jeX=#M;SxY*dT}^Uav^_hi5T> zgYM}Th|~$(!O&Jy4U>PHP(ek_*(3@KVtt$b7Z`GCEiOyEzb{eMx%Ro|>qWxTjxJ#v_H5p9F7%2@M4_ zURz#CW5M;Zd{m8+>049$V;Ae_JpU1LJ1dl5o4VQ4W+zw!<&Hs+it7xt3DRU#F|bx0s?&C6+utftj6W0WO87uq z5rx|mQlP0}Kd|zfF^)(QJ7mM_x9IsLe)R1r{t~Zy9@$8w4_;i$x8KZ^Ip~J3*~@h+ zFL$aRy8!Z}$})Q6)dIN)|Bh8|B_k70Us{#)fUWVU61_!x@$&JzXT(@n0iir2sURH3 z?_n99;&Ku367KyAVG)cbe&XRi`-Qv?J;b{`d$}L{DhPve<1i7C+hM#*Vv>E7BH6!w zfJZ-?G?gcNDbr`U&eM;7S%=ybF)Vui==x585XbaZWg@U4a3!Ms(m6x~RXDAJd5Ry$ z7ZPqq4%zu}KGfC)OKxGbEoFu9vot<8OSfYpCduWhd0Qmq7#8|I7a#kxoCb}N25@-( zc&Yy`Xyl8u6mK-Eb!+7Y1$op10YOGw3L$N=0hT;mRkNHiTyxZbM{RLZp%yzmPS4wQ z55CtJ`HIXW6iXPeWSCV6E&*q!!Z^-XzBfYvXb*F3gNwjc&{j6z-#?KOkfLXo3dY#@(&wX2Sy0o+(4r zeVEPPpVWROh(B8Bof;(Ag_O#9Kh6h71iVx>GgeAU;>ZRi!Jnt_T>T^(z?*D;Js}HV zDtx)u4_>7Mg5`5Su#}R|H^86j%RlQ=Z%eku_}pYK^$bf}=Gb!9K2fu?Q0A)NY5sd* z=nIy$9?96jG_DtEdIQs6g;%%0P}SP=Zp|9O<^CJ(Db@`{Y!|o`O~(s8GwhYr)|on=$jACH=o8Cnw3G#G z2`tIF3^{I<`R(I5#<~hwGCrlpnH>CMPAtrDZzH@}MoDqz3!Z1$Ol2J^52kovzF~nf zD)2l6CLjTFET}LZBun1eX5-VnG3UL9zbe*x`MxcQzwjTcBub-jz zbP47!J`q<`Dwz?;8(pqGfs)on{iPzRl~+xhB%w93Aif$AUniUx*Rk zNa(Gn?;>P!q6qyx8J_tBIzbE-WtWzB!-Ue>%(t`o#?sq*AE>BK{wWayT4qB7 z>?4s#R~)GBp3sJng89mURrn1 z69?8|OBE2drlh>XH~KStM5xU5b0eLswtT%T5jbO%7MyW}6+3_RF{3&Sy03`Xo=J;3 zs4hyvOna?lLc4 zNC%4kAWCX?9|ssyY3j4z+&^hk??1@5An@w5s5|u_pFgp+uS>`rCCxX8sf);jN3J-) zf7?Hpdt-((&qlW^cP$ruxS;uD2et_Z$1|x?st&roOoQw(8Mf~+<}B9*|Jvm()Qcxh zX5+v7ABCNDRFv_zujy_CL}ciaQb0gpVCa%gX#@lWL{hp-x>35jrKO}p=|&o)yVLs} z@4f4;d)7JY$DcSe@5DR!?!7v&2>IS8-!jtkDr~UwILA_4yi2;?eCrs<6UECq1 zd(F4GUbe~c@SOCV^-?a5P3HXCzA68tNvk!OlXC@nd^l#$qUtVy z1brxBN!&EUaGFMw1*NCJg2? z;K2Swe&g0T*WNGb*bEg!eSNdJo_xyNl}lDiN-f8N$emskaBV1?>VC>>*!+%2TmHVU zgfv!ug~^IghXS-Rj@05QCgl9t^(AjSbX2?uixXJUZA(aDR?)9@cc=WApJn9N*GAUO zY?0MtKm(Il&S8|JXYuIHQw8s^h4gJxrT-u1eBIq3Oj;4*F*T<0FE zO@+N()`GFN`SXK^;>_t>QvZpL>teeh#Cf`w%Qx^#r&9nrdY<`waRr<5drvg|$Ky9vG^mI&64Ur{1Y(>?AJD*t>M zn1<2gvbc+B$*9EtiAlUX*>1_?DKEl>2>&u1?9ba~q0bTUOkO@5><+ZIY`STtowc_p z(|e^axe;nQUc;0A#~;AeFLTuDs#t#H{m6Nd&!$($Rf=(MlDgEjM`f<#i8ES5U-}`O zp1k57N#rU#c$$+(qQ_uvh9I%*&mc=}PVHv{%P}ohbe+LV8M8*MLJ~={_^ISYo{CgG ztp2={qSi+V1DFXXFgMOed2yPG!X^vi1=0HKAt{TM*)(ocPBHC2pX!aQDjNANCd3yUCL+&)XG_#?8mM+0Y!5c&tL7MLUkM$bN zC*r?bf_~hipD3)$*fFRl7ss0*g7Hwx3Wx`|UV(uD^}7FTX<5palm!N8gvp0K!0@Hhf_wji+Q#Pr+y@BPYG^Ykii@^kcP#clKn z)?8O4w?n{o{Bwfxqag1-uIeCLI*KPw)c{GY)-ZMWeocsq@2w;MpCh`C_6KCP;SAY4 z>NI@i^2R^O&u!*vD);N=gWwFu#g0Ji{o-mJf~L)#^Bc(=A@*h>Lh+finZVVT1WOi;ZRMJn_rtykr_EkHfA z0wQNGkr*P};C5HrNZd|ug=e~l9d#y6C#6U&zu-z3Wz81T2Vac}baos=!@DYzdeum_> z<_f;F$Pf4ioh!jqi_MQc(zbKSn(J3KNVFQyupR|lhtBnq3MO2dg)Z+Z$l_Scvk+XG zp~x766cA>PR0LNn>*n@VHI}6gp$i5acVIkQZI%zm8{orZI-aq-o;QwkKfo4n4@3;y z=grLflCn|}m`WOK8@4@h3WvG6Pi=`6gy`ZcEIVQT-t~>H`j9=}$D#k^wPEn*k?mr||6_aVz-e6lwo@*Mr8*Ip-UPj9Weat7ZRAe6J(qc}=7xh;<`V-DSL* z;tS}%{10{{u5~|W(^mEo$anDFfrOO<@KKR{Bc}?@9T#Xz{OZSf(zbSLexy#HR=oi9Csz0dk6!Z^F~+Td0L*Aa6W*|;(61KYmo zr~`XHE0_1%c?%pCBFmI!Y6)L0?(jp!F8u}H?X?!XgJ)TK{Mh$^HJFJ4eGhyqPfPbf z97q~0B40&4|8$`D%@geKzoYVW1{t9s@K*TOp$_HNA7p{G28m5fTc@+Jo9T0G#KQ>eTpyOP<0D-CNM6md%!dJ8}5}^gk_3lJbU;6E%YLPY2k{7XKE(z5!n? zZ{G0F)h^xole!Bo+9P66me^N4u)n_f_Ky7en>Ni8(i2dhLEu>(x|h~DX)eNs*8^I6 z9(b`mA42?uY+72hn5r40(O->-Oymf|_#}{AZQ!6}hP*U5_kogP4*+>rmLB$(!UJRr z_G==Xl~VafF?QgMx>DOJa@{S~n>`>>s~67Z*a<9T<^!g+|4sV^VKd|Pz8I-YmyhtJ z$o;J2UBka~rYvC*Gx(w>Fq5DB6kTyb<%4I}Fi0>L15Pawjp zJJROKU1L}!#QE&(ZiF4+0vNsla2)Q=n4hL_I+3nyNW2X%FAZJg)}q@kdz{&PZ-?n% zf8Eo{WhLQj)A`tMTgS1LVUl=RBOH#=QcxFmf=AAx&Ik)A7=F}#ry~MaXxy;`o%Ffwgns*)5y=z^oUDXuX9jjt}s#R1;_WFPT5Q8&0)eu6L$h$~yjpj~?&4EbPp2se{N9^jAiVIY6Z=9otI*u}*TDqXK$q1o25^bi2 z*brY&WVunR4#i&$V4JI_>i0{veUYIHd5v;V4_Fi*K@MYHGv$<_1EBWj*m^aZRMUQs zd({nDlgx6#Uvz!9A<8;te1g{!er~_i`ca~V2!u%sBfta+1OFtU+0n1YuwPsX9P_5# z*WRVu#UY$-eiEMK{A0--N$a`vDIPt15-x|u$4+~(O2XZ`gKQ|ZVaiRt6dbilz!^qsQnn)^nG>8>0njzXFGFaelBB;BKMiX}4$-BdYoWO!~xo_(|U$^Js0~4|HXm_QNz%|aZNmRs-MC13) zjX2}9`GXV7dqhLL=@3qq`^@tx^78euXlUJ)5AX2`(MDyIgP1V?k9W|ck*v&H;IN^5 zphZ}Jra;Og$ErH~D}{3Le8C&r0ll|?+Y&b-%X{L*yRO2@hbz04{UWuF4m0_ERK$`| z-TEM(b{`k=nqnG{`=3$4`1{Vvk0;~Jo09k1rGz8ae`4RhK}a>OQbr65K=m^hBgZQN z-CU|hDnaK`{a600Y*@C@0weujNlZH$A=i4)80AaA&w1CRt(RI)NUc5Ka1cPbWn%T& zb0QdThd!+}Ux-S=h|crZCsY<}*VV>l*_9u5kiJ>cjjDvxGPpx=V2xzO(_RT0&Fm9S zL)1?G*v2S@X#{QVbGWr6l>oKFNtM6SSC(Kezq-&F+}S;#@|$7=luPTZ!z!vbt2Mq4 zl6)7ck^HkJ)mE> z+|cGuX?;G>2wvkVi^*4qHL8dhzGR8fXlpQwaC$3!b|X`kBU&O)4})IN1krtud1o(Q z-T7{{6$-l^%X3$jof0<_jd@)h=XL5&`(|Brp-8x*;?Dw>I zDo@_6kg_`OBn`8ISu>6$%4`V_^>>D0jmy6HUFq^xJVQl$gGF&mgZgEscaH49w6t(C z6FNRu9#hbXtsYZmv6It&HC`5ZWO7;-I)VO$V10XM zcMK}JPZ)aU2Co0ceAzDbFlUB4Rg?vkU+rm-*QW7n^&_6sXjK$ggsV$&#WLyJZD4L` z2m+t7hfZ8~Iv~z8~Z5Nx(WU!la z*Z!q$F(k(LEZX=F3-7Pp+GxeQGacN41WN^J6=6%PgiO?6z0p;*YdJR$$T3i(b6Hy8 z7KI;F=?X}^6Vs8(3=e5~DZ`ibRI~`vk14s~AsihpCUq#0WNYGEL;JG8*tOweTe?GE zNd%Huu0sw50MQpw{lQ^O+B|bzBC9Hx>wLN8!h6k=Q!5Rg_!z`!5jY;LnvEBrTFyP9A(swhwBp)BI%RO#rCZyBrx3z_-1;D%1bEhj(m#!#G&1$Y*t z^cEgVNq10;KNWL$tWOZ%YaY`1cG;F}$^$JKZe}9VS9CtIIZcuK>A*%w)cHLzSo!sb zNG2^RxNW>xvfjn{re z(pJN9vZiAw+mvD+q)?fY(yI>Wz$V(ut(T_gF;)76S)YGD3gFRq-ENsqsN$V#dZ|vO zCcJO5QHz>at(QVXJ6HHDCn|e7no{Y|>;CF%ccPNrLD-n~Q(RnJ1s1FH2OaAzr2RKO zyI7%f6%Ur!9#mFr@rjWd?3sn36_I+@G{yV7Ye_1UZVSAL-K*kQZ6iCoy9$i@I41%( z0z#N&I<>{(vYl~x6$MZjD$4ZC8IH-77N2wKFY;YpH{4kg_Al9}>JmP%YBn*{EWNL- zNOKkF$jXK!=eAa`;6q(lvY5Maxr_VxyEEJ~i)xc9mfR!k>eN(H-*RDKHsPe*#}2G( zPdc~j$t)A_<(#i*bu0_}R&VGDT0T1?tU`jMSev%iwLVLJ`AX;ZVWDU0ye2mGNG|X8 zw>i($pXAhKzL9SJ^7ZpT_KV@Zqb(BC-i|lTE_Mx|)`ahN&8+&PGd7YkhB*Sw&Way4 z2L{mLYFi;v`$qz~L+Mw34PI2UFjD2lvmq&~bJC6acbg|0gK(V(a1Re8BuW|9XqFR= zH~=nNisH@8CHtCY$%p<5;D0aYHXp(Os}fc&fcew`7wH+e?>os4_gUZq)4C8LyxNsA z+|Wf+Jk1K1um-1X3~U#Dv$BsrGgXnJOICR1&q;@0s4 z`74ly&j3BBO?4ZnZxcDV=xyNMq=Vm$yvM$zq^{BQ*C=^bC1~oyz&2e9S*MoYPQqy? zJ(EpzpHyn9r=Wym9?MgN&R9I*t|dHmk!=)L(b%;n>shWi)aZoa9JV^;AU7E`WdSD# z1|N`(<+ZMxeP?`kbv#8i!{E7t4AXa+ny6@IkWbWx_JQppSRyP9$)rPE%2VFk82N@- zK5YkkAhy8HIPnYK zyXaR-lnmtaDThlXn(;FQw5_%D>;l;#a~PQaBszW3G`}t?5p0`}rN&=ajATBynks#6 zw<UD}`O*JBEOIXnw;CtPH)ufCJdR=Xui4^lLGOPC7_U@m06u z0uW7Wez-bXSqY(w!~h)!EC7Mb+wLxBum3$ZR<#Wt?A!j+#c{AGryvprnmw2G-NTx( z_Ab_7tww?Ws2%y&JD-+lj3&!9u)p!X8XtP5>;iC*>{KUfE7ib}pu40@R@VkDyQd&G z7hv6o@9+o+MYzT2GI-xo-1G$(0*)KrxNq4OT&d9s=Ul>lkk|1HFfX`JxD(FCgD}|; z#H@O@u2CLVu3<)UIRjltCETQARo>Wh19v}Lf1t29eEM_i>oP{@D+MF69~!5j2o3M= zK3fS-|JxIbE0k5m?7Ld-kCHnkfJwjRaU>YQb*&;d#+vieZMZnm5N{iPL+!kzS*h-+IF2cq{a}*^|O_e2x>=fp7%v70_36nzY>f)o1Fqu+~ZfKa# zR5aIKi-^jB4#GEuWVk`-b-K{(z!7z36!2#b&j^DeT@&&)AIGMXcpjVuDO6nRD)_E5 zt#E-37>W+hNkj0FFp`@3{Nr6e!lKUv1NfbG)G5$=D5|?j=N{UT@p;@YwwB;KB7a?v z?ApLvQupN(+6QerwzY5P^|=pLti;5H}k>sbFhEY>65DehBxBnQX8>YlOj zlhT9{xi-tYny{h7Jf}o*1@2qfbr6H55?BOZc{mRC%VuD3x1UC<_39o5t&v?$R3OG| zjM9^K3SXZ$&1@H7zh)>Tz#Rc`6ta{3)YLzH!Xco+0vSTJpIlEd^U zky=wY{D-ayw(P9fL#=yBxU>$?lohi&)8cM^^8>4 z(WL1NPBDst{hdK7P&^N7qjJmmx| z>5N+VJAniv{=ZQ6%vt+s{}@pYVZWQ{tIrT zUuX5h1RDh=ID$!-lhiO5@$|?|CfL*?r6|hr++;3e$KilRbemBoh*i{0B5{o2qY^B! zu2SDvh*}zHKUNWc?5L2X#MumZDWzl6NTY@DgkILk6QKAy&@=Kb5@VxB;n+rT5ugU3 zQ4{wOb+nrpeecv%#*THvU~_54kiooX;|M{BJ`|1)> zV4b@Z{R85fXr1>;kUqn!mm<&L0N8O5%v@$*g(ov8rZhb5{yr~^1b==}rkeNY`#F!y z1E~tmF>mgx`$ljF*H+uS4UaxHVmrE-9V%AzZ4SqYg1Ek8($#+CqQ@ptC+HK{y=v+! zZtpRJ@aAX*34Mcr6s4CoM!KDxR99-OD#nJ2K3j4?OcZ_7sD8=oJG*@|o#V=9V|*Jw zYibf9%(@8cDjJSv1yWvmABE^A5(vh6z${;P|Hm3g$-TkOU=ZBJQrF@(f_YVmqA z!`41NPk*8_7z0jXot2=RkyuC_)(0&~=Mem_;ywh}=}!%w>TXheX!vnL!{WH(kAq~h zc_$u#WBfcZN#FUaYf3aHmB9$_&}B4((ItrPTJx^G$VTs2+hv;R*VM*`^x!NIj7@48 z+SI|^Qf=R2drG@nQ#{Y~rl0#NRrh_bx+H>R{$tOhx8_f2JgoyT$aNiIxk7hatgcC7 z?wiI^!{Y8kdv5CmRgj5bVcRfUpFK7$a}~E34&NJ}G0Or0wM-#PNVVujLPb+wPQ_zt z#{uGFGZe$AL8Nt!q!XI=!dU?_$Llp4;+)6!?J0S`;aeQ0ENLv%A45Www2td4mF$b4 zVlf;GCCvvh5shFo@~oDLjXKJGF6dK#=Mks`t%Sd6v^-l|BX6PFzviji2;Db6iqpbX zCJ4C|%41QULicSLzND8X*plhToY2CXDw1tIW7RiZ&vMhKKY_jxwa#^zB=b9-yQ)M} z!oZj9lF@%APmb!#zes36OZ5)c&?Ryu{GjZ#wJ7X9q6v(zhC(Ap4-#eYe@`M&*XJWN}D2X8#)0GPrah3 zlIe^@M7dM@%vdifGF9mN3N&{1wT;guhht}tiXh7LDFZGonKG?1e9DGclJ;_OkOc<~ zDuj%TI`Uk=xWFw{o1e*+EtbWd$W&M4GN^l*+4%`)BiB&CLs;f~w^gQR&^3Ngeq8l! zQVT>lg5NB+cV6N*x?vCk^QKum2eYBwk3qSfP2>t)<)s^*%oDwqG?sycU!ay5Oc+O2 z=5Zyd@p+_VHjm^=!S<}JGNeSu<| z;OQxRun{(pI9`A+8ydZnq?gBAA7t-T8%iz9UF;2I zd4rhNGsqQB_qcL=LCwVA>r)bRVU9=rSIT&jTr)pfm~PdFmTnzkdIIt31L}UJB`j)X zdQwYW0=xf8ESLYWG?J3~ulNmcEqzBOG0qvjGpBL{*iBqhP*6sk8WJe9JukMB{;@mM z*Qb3M{crUAUv0Yo@i4qFL$P5)sYB>LQa;`~XZ9roekTj$Gcvx}lHGs5|383`9Lhd# zwFUR+{{Nm8(t_r(Go23R*Yz>|vo7HFyEr~lG3yxIn*V>~{$BsXk4{~e z4FYmXQlL<0{BD3qsNoI_%}W4iV&G4EQp^BgBR2o>`TtKE;s1O`C(^r+Mz5bjaO6pk zz~{3|4{rEJQ+n!i#Wh>TJ!1QMedJ{p!we%Kmm9~;2BHqxGMVubb_uBbOqgTz45SyL zrg}0zfGUA;bVmCLKMf^pp+cwj84vhDKS<&PoemswB5^ zoyo`ykMzCXc_9zkU_w#A%bT5!9jTiZ$7yevo{!zJ25mEZ=qk_ANWTXPB;$?%OvZUF zm|+`OQvYX=2dYe$8E}Q3V*ef(6ohemLhRmoPwrY$960-hM=*%Xzg6S1p)yYL^@X63;?mM{B{t`S*2~M9;^OgR5=T+n5 z9dRs~$P=vY%t(6!Lqp$?RzYxHnxq9?_KhKEyXbg??g}U9Q_$8w-jPdpV#rKl*D^%g zKP<<82YL<@D5R}@nFZ~{>aS!N6G~pcOHi5SOpGx8HGINLm=PFmZ(&m^fDM`;&uAG! z4w4HKsH>ki>M^am1X2H;|_#raN%<&e2hKe#{%(a3DK`9s_C@L#psF@o+Iy zR#zbK9jW!;ps+1xRR5GIPCtl7Ul|v)T&wJy9Htj4kEku%#MpRjSapYzz{~Joh7`M4DMkJQ-?Pgjr5L~C(1n6gbZ{`l zjbHE6!ublop+rnu0WZ`d1u2_0!HpfqBuYp{YiovnSf&z1!XX*zd8Kk9+nXn3R{P!? zG^^}>q~FCr2t?F%!|@U&r1W~O=hbE^F_>yz2EvKXsm^njD_}16;e&Xdvop#E;4M4? zEKuHNFkw_+X+i(Gcf+s!6fod1!F8&scC(8LT;SRPFHk^j^mYc!mpXh9?^B>T_R7M( zoXKRqZhf1>xe%`_n(w6B5{fjax!9x644s})UfIXeh4eMtI3mHK9A#{kd?D211xolL zx3j~0VYLP6ZpGaq=S51hWC??_yYn+DTR3U@zW)d6Kl$WP{~glh>yNF!elE58rt_%W zU_DuTOz<9YtygmkRF2s4NgTM~G9~Pue7JBlrvtF?I~tDHto6t^bROk*2iEbpnSw4x zCHo`#zST9Q%s)TBnppayc~3vxXcrss{Nr-_ zW3-o??_9qStnAHx>*xp!dmYq7&2o~jltH+gLb-2XdH!NLUm*pupG>~1OECZFt-gP$ z;-2_47{n@wY=@Du^H2if442@DsL0RMSFHI;VNO581a7;}L~W9Lk+WNVdGn<#13^SB zx~%TmSb7*oQ=d>Vz?Y`LHH4Tc*445zpdVthP=%J^bz^zjLj~&VEhr}@9IXGamfofV z>&=yo>sSXXP$7hLdll*J|FRR!r;6XAu6=~3&cUuN35ZgZ!|fKcT2fz#f08&-0tSyU zNHc#ssr3^f2RQuu^v(!J{=q=Z_^ z%l{q8zddXd*#qAX(*!V(6KIgCKoxu1%h$-Er0|Srt7Er*g=k;$+ z;jJ>bXO)oF^AfB$)BgtDCEmz&PQ8ZbV2BT00m1`pz|+A6^Ud4@V>ZO|&%`{=!i#F7 zetQ$XWumV!yGq4s1;8BPOT(Q$`y%mLUp8<|2V0+fTL_S-Y3Wp(d6#0d_ZfI_Ln>{; z4*^Wx?pUF+8%@@-1D`@z$QpbQ#-KdqLJE*M!r`5>5I@c51#Fn-UUx3H^HN!@t==Q@KH^8}ym@Qy7ZK?`Eiws`cEf zo!O7#ivyz$e_jA7z_HPauS7=G(3VxX3qSa_-~s)F$JH{r$YWp#dCdxsJbW$ac}({W zNGo)Ny{pmsS7Hp^w)66X9U7FpZ^l;&b0WEjn!(`w^#QQ*HKgHlwKkWb3N|@JsPX-9 zP^>L%k?0XXWyHny?f;p3NVS$D6^Y}IhJ)jfjb6*9c-ONP^P(kK8p%KUi#5SIh3#uz zTcnCU1NO=F+DYj&I$Cl(axNYm3f_r_Y9C2--?QCbzO9sX+RyJ#RQZfm=oyCaY4n=; zNTRl+Vs59{d1l^M&){uVy?+2q>U)V2QdC?_N zBlvK?6Y!vHy1K;0a@!jr_*4*TAyaB)2+YOXTJ}}RhH(#R;u-MjLjR>zKqtlRs)MZ6 z2w$X+(Bh{XMKPmbV>$UQDY&4p#?ExCtEHakm8=CRrP*~d%3}*f8P6f#r5WwA6bU8y2 zUr8k5Q@SW4)@6Dq|CptR&}?jp_pzaxJVbtKrh}L8D{$=4p7kMMo;`mXa0M4K2$pR8 zj>IRSmb@v@TRN)|mRC71s1nZL7hQx<$}(+@&xDst>k3DZ@E;n7p^)40M||om2=$nQ ze`~uC@r_;S{i;Thp9sHo(WdP6Ty94+_R=%ziwrkFfUGn$K;O;}kUw4|T0}JMo1JU0 zGqJH#i7aEG(#8o(pY4mGORm@PM@tfx+(0Jgclt=r-S>rWzP#RG+hcf7_wgVe<+}{h ziuwmPI|l<1 zBy~J)hF(F;6QXC~1Q(AFpVQ8^NwoBgx3}g&M|95@CL$Vi;j1}5gE?7K259UX#jqAh zM4qMFq)$kMi#Vj{2zhum& zqpM-;x+am;+%c5TmOCfhXS#=1-Oh8F<_o35hI*T$J&DX_i(R|$%MZo=ufA{y*jSpi z%{KSsCR&WKz7bOGE(uisR@=kPgG^NU>u#o3xUR0Oto^1JabDY9XC29kO96T6;)@jX znB?C)GkR9n45egS|Fe3u?VIxoZ^~8>jnrw@JbL*gHvEayc=#a_#Y)Z}WJqq!azLjro9+PD8wEUxiUV?4vvoNb9qhm%&VTAJ9 zx%q^4UMf{)N~8&>30;LWwLbkH)Bip)Id#nJCbqqLM2rPXdn857efjqD zw?Vl~+kii<`(jr&zB{%MK6CtZ1L?WV-S_xFbOSsp`-7(Sk?ppef=D_D8)B~=<&T7)|c0(0z(^O^gI~(JMWRxvt7ebT@#QhU3f}tvN(9A9y6=CYpa=m`(=Y+cal_2 z=+Tltrn7t)at(R9q_!)2WSTf<7|Cpf>UO2ZXYg!9-%!QsA)WH5yyUM%Qq3LodphpJ zj!&ELas|O&1^L@2*o}5hCTBM;`ERhY&9bV@xrjI2Eo<}n`72>rBz#l@e4Vu9ol(ZAgGdnH8rQ`S@ovUkUY%ur`AVpVTNj5ig0}Iq3$JwZ)ql}H(1m( zGU;sxk6+oFauZHsc@{N1o!!0-ZZ~9~kaKP#l5p0$mU3YHet_QD``pKzUhiZnosSat_arUi68v&PHEAfiO{Td|#`Ru%3n{4V^>uKL`ty> zOw$UIaH!H=ptv}8ZS0#lQaujNKIY=t(4biLX4AhlKYk*9XIobD-oLd*Po8;9O(EoO zZBg2BTYe;aP=5{5l>E0=VI*ku{2hEni&Vj)#eWR-q+)QQztF31i z&ZM)-EI!#TR)||WDuZti!mNt9uO-~VFh-`QX|+1ijFqLPbCLtKFm;w|QI^+Q5Emch z5VeIaN?N;YEcf>|!^=ghIJFrbo3_>d+Ilroe@*^X2Z>p(g4dBO)Vk;qOvNp4Bs~9` zV|8##peEP~tDl3iy$0!EaQ`c}8`>Eo5&J|rp8n_T8$nhKNh>j$oYT?7)}io(Q6!y- zeasd_N#}Y}pQu}W8LlR@)(X)5a1<1xDV10LJ~eq!W+n5faG&w}-DDidHk3;+w6dJk zG;Yq6Meh|m;afS;fj$)1od1P~=KpOxX~gNj;^go?rw390Z|#Ybwz}s@y0w1xB>m;F zsweRo|GGxsXvXBPdzRc4N_XEDqEMHFht=43TH`3#1HGnM`P7z6^bSmOVft=s39 zn*)@%lyv(kZWd|HE^=s{=D+a7WT7R6{1KH5a|GF+nDmxL@C1hfmRxTr3!_+b3q$up zv7Y_X>R^JvS25C17nN}~Yfv?kA45_)1j47+cX+6x)C%^r zU5|tn>^OJ#k7S%kwvUe})#5oP=6od!b4BQKpcaF7);7>w=A__sp5Y}jSG|EG+=Kz- z1&8i1wO4kSA_tOC0Rizykp4~!)pe^i-9%1!Mpc*hc8prs@5D6I5ve;5n>^!?hP?ZJ zavr@Ij|#nPK?_iT57ndUc9iDhs}6i6r{BnIn&e&Ud^qsaSr=-|t!^!=W=IGW1pFJH zJp?z5xsH5Mk!tZ}gQLsDHlrVY7QJ)1u^d85a$Or$+j0I?V>~iScHSC#;5bX9)oagU)dUc$)so9&b5uF82lRa_@YQ2cE#E|`Pvxi@FgVr@ zg*zd$HC<#*oouYLZXdg*3&C_0FNGHhikheBFLQ5~R}MF-`fuqJT`M}>ov>n2itMCHDT$@Ou|f+Q8JPS z624zLu|||6U0#+7c0*lRa9y%vagH+jZUFd~q$Zkp7PIRfimi3|AlF3t5{rg6;XJHG z&lPn6*>GsnUwvxRx29Cu?NEiKqRgUXXsD7Q$NUFp7x*L6#M*z;tgg6D!`xPgGi|@& zY4EJVy_t6k#Yz{FIwlyg=c9Dlg(Tsa8T<57(tewSKxvRHfpDxEQUr-ouhUU~Hm#XP zPG0tdbN#)tVrwi!$)ZTF(LJM_-z{{ERi7p2{8=>}6nAVN8DKm13w0Yl8c$5eN<2a@ zp4lu|U+GtjPPgCiaC6fTLfw|UxMD3-Z+-b_RvS-v>65+HMssN$#3fGTVNaCta-DD3-!noUFO_t?Bm6V;ZuH z$c;5NkTQjA9p$3*@Nn81vj@u+vMlD|g1qYoVTRJP9kY8+l~;LrW+*3l;@c0&7xE4e zQ)zkzDkTh-qWD3wW{)RAV*=w=&+t9GbZiN83uKaD=>igk>Qd+VX~{98`+C$F1aSuG`1`3q#?92z}{~CZccrn%TxKu z(FF_cmpzaa$!<+t*KkCwji$7SIJzToag82$ltAe5#Lc3nzW!xR{`A{MCwOtUFaqKsQBgfRkwx69p`M{2Fgjk~UNkING*MAGmHl(@Edk>G+G@rYX@X T^f#Lx0sP2HDoT`#8TtJ$xVRCq literal 0 HcmV?d00001 diff --git a/src/spetlr/power_bi/PowerBi.py b/src/spetlr/power_bi/PowerBi.py index 15250a29..0da669d4 100644 --- a/src/spetlr/power_bi/PowerBi.py +++ b/src/spetlr/power_bi/PowerBi.py @@ -1,16 +1,16 @@ import time from datetime import datetime, timedelta -from typing import Dict, List, Union +from typing import Callable, Dict, List, Union import msal import pandas as pd import requests from pyspark.sql import DataFrame -from pytz import timezone, utc +from pytz import utc from spetlr.exceptions import SpetlrException from spetlr.power_bi.PowerBiClient import PowerBiClient -from spetlr.spark import Spark +from spetlr.power_bi.SparkPandasDataFrame import SparkPandasDataFrame class PowerBi: @@ -26,7 +26,10 @@ def __init__( max_minutes_after_last_refresh: int = 12 * 60, timeout_in_seconds: int = 15 * 60, number_of_retries: int = 0, - local_timezone_name: str = "UTC", + mail_on_failure: bool = False, + mail_on_completion: bool = False, + exclude_creators: List[str] = None, + local_timezone_name: str = None, ignore_errors: bool = False, ): """ @@ -35,6 +38,8 @@ def __init__( If no workspace is specified, a list of available workspaces will be displayed. If no dataset is specified, a list of available datasets in the given workspace will be displayed. + if the table names are an empty empty list, a list of available tables will + be displayed. :param PowerBiClient client: PowerBI client credentials. @@ -44,20 +49,29 @@ def __init__( :param str dataset_id: The GUID of the dataset. :param str dataset_name: The name of the dataset (specified instead of the dataset_id). - :param List[str] table_names: Optional list of table names to be - refreshed. By default all tables are refreshed. + :param list[str] table_names: Optional list of table names to be + refreshed. By default all tables are refreshed and checked. :param int max_minutes_after_last_refresh: The number of minutes for which the last succeeded refresh is considered valid, or 0 to disable time checking. Default is 12 hours. :param bool timeout_in_seconds: The number of seconds after which the refresh() method times out. Default is 15 minutes. :param int number_of_retries: The number of retries on transient - errors when calling refresh(). Default is 0 (no retries). - (e.g. 1 means two attempts in total.) + errors when calling refresh() and start_refresh(). + Default is 0 (no retries). (E.g. 1 means two attempts in total.) Used only when the timeout_in_seconds parameter allows it! - :param str local_timezone_name: The timezone to use when showing - refresh timestamps. Only used for printing timestamps. - Default timezone is UTC. + :param str local_timezone_name: The timezone to use when parsing + timestamp columns. The default timezone is UTC. + If the timezone is UTC, all timestamp columns will have a suffix "Utc". + Otherwise they will have a suffix "Local". + :param bool mail_on_failure: True to send an e-mail to the dataset + owner when the refresh fails. Does not work with service principals! + :param bool mail_on_completion: True to send an e-mail to the dataset + owner when the refresh completes. Does not work with service principals! + :param list[str] exclude_creators: When getting refresh histories/tables + from several datasets simultaneously, exclude datasets configured + by the specified creators (list of e-mails). + This is to prevent the "Skipping unauthorized" message. :param bool ignore_errors: True to print errors in the output or False (default) to cast a SpetlrException. """ @@ -81,29 +95,33 @@ def __init__( "must be greater than or equal zero!" ) + if (mail_on_failure or mail_on_completion) and table_names is not None: + raise ValueError( + "The options for refreshing selected tables " + "and for sending notification emails cannot be combined!" + ) + + self.client = client self.workspace_id = workspace_id self.workspace_name = workspace_name self.dataset_id = dataset_id self.dataset_name = dataset_name self.table_names = table_names - - # Set access parameters - self.client = client - self.max_minutes_after_last_refresh = max_minutes_after_last_refresh self.timeout_in_seconds = timeout_in_seconds self.number_of_retries = number_of_retries - self.local_timezone_name = ( - local_timezone_name if local_timezone_name is not None else "UTC" + self.mail_on_failure = mail_on_failure + self.mail_on_completion = mail_on_completion + self.exclude_creators = ( + [(creator.lower() if creator else None) for creator in exclude_creators] + if exclude_creators + else [] ) - self.is_utc = self.local_timezone_name.upper() == "UTC" - self.time_column_suffix = "Utc" if self.is_utc else "Local" - self.time_description_suffix = " (UTC)" if self.is_utc else " (local time)" - + self.local_timezone_name = local_timezone_name self.ignore_errors = ignore_errors + self.api_header = None self.expire_time = 0 - self.last_status = None self.last_exception = None self.last_refresh_utc = None @@ -115,12 +133,57 @@ def _raise_error(self, message: str) -> None: else: raise SpetlrException(message) - def _raise_api_error(self, message: str, api_call: requests.Response) -> None: + def _raise_base_api_error( + self, + message: str, + api_call: requests.Response, + *, + post_body: str = None, + additional_message: str = None, + ) -> None: print(api_call.text) + print(api_call.url) + if post_body is not None: + print(post_body) + if additional_message is not None: + print(additional_message) self._raise_error( message + f" Response: {api_call.status_code} {api_call.reason}" ) + def _raise_api_error( + self, + message: str, + api_call: requests.Response, + *, + post_body: str = None, + workspace_id: str = None, + dataset_id: str = None, + request_id: str = None, + ignore_unauthorized: bool = False, + ) -> None: + additional_message = "" + if workspace_id is not None or dataset_id is not None or request_id is not None: + additional_message = ( + f'workspace_id="{workspace_id}", dataset_id="{dataset_id}"' + ) + if request_id is not None: + additional_message += f', request_id="{request_id}"' + if api_call.status_code == 404 and request_id is None: + message = ( + "The specified dataset or workspace cannot be found, " + "or the dataset doesn't have a user with the required permissions!" + ) + if api_call.status_code == 401 and ignore_unauthorized: + print("Skipping unauthorized: " + additional_message) + else: + self._raise_base_api_error( + message, + api_call, + post_body=post_body, + additional_message=additional_message, + ) + def _get_access_token(self) -> bool: """ Acquires an access token to connect to PowerBI. @@ -167,48 +230,294 @@ def _get_access_token(self) -> bool: } return True - @staticmethod - def _show_workspaces(df: pd.DataFrame) -> None: - print("Available workspaces:") - df.rename( - columns={"id": "workspace_id", "name": "workspace_name"}, - inplace=True, + def _get_workspaces(self) -> Union[SparkPandasDataFrame, None]: + """ + Returns the list of available PowerBI workspaces. + + :return: data frame with workspaces if succeeded, + or None if failed (when ignore_errors==True) + :rtype: SparkPandas data frame + :raises SpetlrException: if failed and ignore_errors==False + """ + + api_call = requests.get( + url=f"{self.powerbi_url}groups", headers=self.api_header ) - df.display() - - @staticmethod - def _show_datasets(df: pd.DataFrame) -> None: - print("Available datasets:") - df.rename( - columns={"id": "dataset_id", "name": "dataset_name"}, - inplace=True, + if api_call.status_code == 200: + return SparkPandasDataFrame( + api_call.json()["value"], + [ + ("name", "WorkspaceName", "string"), + ("isReadOnly", "IsReadOnly", "boolean"), + ("isOnDedicatedCapacity", "IsOnDedicatedCapacity", "boolean"), + ("capacityId", "CapacityId", "string"), + ( + "defaultDatasetStorageFormat", + "DefaultDatasetStorageFormat", + "string", + ), + ("dataflowStorageId", "DataflowStorageId", "string"), + ("id", "WorkspaceId", "string"), + ], + indexing_columns="id", + sorting_columns="name", + ) + self._raise_base_api_error("Failed to fetch workspaces!", api_call) + return None + + def _get_datasets(self, workspace_id: str) -> Union[SparkPandasDataFrame, None]: + """ + Returns the list of available PowerBI datasets. + + :param str workspace_id: the id of the workspace to fetch + :return: data frame with datasets if succeeded, + or None if failed (when ignore_errors==True) + :rtype: SparkPandas data frame + :raises SpetlrException: if failed and ignore_errors==False + """ + + api_call = requests.get( + url=f"{self.powerbi_url}groups/{workspace_id}/datasets", + headers=self.api_header, ) - df.display() + if api_call.status_code == 200: + return SparkPandasDataFrame( + api_call.json()["value"], + [ + ("name", "DatasetName", "string"), + ("configuredBy", "ConfiguredBy", "string"), + ("isRefreshable", "IsRefreshable", "boolean"), + ("addRowsAPIEnabled", "AddRowsApiEnabled", "boolean"), + ( + "isEffectiveIdentityRequired", + "IsEffectiveIdentityRequired", + "boolean", + ), + ( + "isEffectiveIdentityRolesRequired", + "IsEffectiveIdentityRolesRequired", + "boolean", + ), + ("isOnPremGatewayRequired", "IsOnPremGatewayRequired", "boolean"), + ("id", "DatasetId", "string"), + ], + indexing_columns="id", + sorting_columns=["name", "configuredBy"], + ) - def _get_workspace(self) -> bool: + self._raise_base_api_error("Failed to fetch datasets!", api_call) + return None + + def _get_refresh_history( + self, + workspace_id: str, + dataset_id: str, + ignore_unauthorized: bool = False, + ) -> Union[SparkPandasDataFrame, None]: + """ + Returns the PowerBI dataset refresh history for the specified dataset. + + :param str workspace_id: the id of the workspace + :param str dataset_id: the id of the dataset + :param bool ignore_unauthorized: return None on HTTP 401 Unauthorized + :return: data frame with the refresh history if succeeded, + or None if failed (when ignore_errors==True) + :rtype: SparkPandas data frame + :raises SpetlrException: if failed and ignore_errors==False + """ + + api_url = ( + f"{self.powerbi_url}groups/{workspace_id}/datasets/{dataset_id}/refreshes" + ) + schema = [ + ("refreshType", "RefreshType", "string"), + ("status", "Status", "string"), + ( + lambda df: (df["endTime"] - df["startTime"]) / pd.Timedelta(seconds=1), + "Seconds", + "long", + ), + ("startTime", "StartTime", "timestamp"), + ("endTime", "EndTime", "timestamp"), + ("serviceExceptionJson", "Error", "string"), + ("requestId", "RequestId", "string"), + ("id", "RefreshId", "long"), + ("refreshAttempts", "RefreshAttempts", "string"), + ] + api_call = requests.get(url=api_url, headers=self.api_header) + if api_call.status_code == 200: + return SparkPandasDataFrame( + api_call.json()["value"], + schema, + indexing_columns="id", + local_timezone_name=self.local_timezone_name, + ) + self._raise_api_error( + "Failed to fetch refresh history!", + api_call, + workspace_id=workspace_id, + dataset_id=dataset_id, + ignore_unauthorized=ignore_unauthorized, + ) + return None + + def _get_refresh_history_details( + self, + workspace_id: str, + dataset_id: str, + request_id: str, + ignore_unauthorized: bool = False, + ) -> Union[SparkPandasDataFrame, None]: + """ + Returns the PowerBI dataset refresh history details for the specified refresh. + + :param str workspace_id: the id of the workspace + :param str dataset_id: the id of the dataset + :param str request_id: the id of the refresh request + :param bool ignore_unauthorized: return None on HTTP 401 Unauthorized + :return: data frame with the refresh history details if succeeded, + or None if failed (when ignore_errors==True) + :rtype: SparkPandas data frame + :raises SpetlrException: if failed and ignore_errors==False + """ + + api_url = f"{self.powerbi_url}groups/{workspace_id}/datasets/{dataset_id}/refreshes/{request_id}" + schema = [ + ("table", "TableName", "string"), + ("partition", "PartitionName", "string"), + ("status", "Status", "string"), + ] + api_call = requests.get(url=api_url, headers=self.api_header) + if api_call.status_code == 200: + return SparkPandasDataFrame( + api_call.json()["objects"], + schema, + indexing_columns=["table", "partition"], + sorting_columns=["table", "partition"], + ) + self._raise_api_error( + "Failed to fetch refresh history details!", + api_call, + workspace_id=workspace_id, + dataset_id=dataset_id, + request_id=request_id, + ignore_unauthorized=ignore_unauthorized, + ) + return None + + def _get_partition_tables( + self, + workspace_id: str, + dataset_id: str, + ignore_unauthorized: bool = False, + ) -> Union[SparkPandasDataFrame, None]: + """ + Returns the PowerBI dataset partition info with table names + and their refresh times. + + :param str workspace_id: the id of the workspace + :param str dataset_id: the id of the dataset + :param bool ignore_unauthorized: return None on HTTP 401 Unauthorized + :return: data frame with the partition info if succeeded or None if failed + (when ignore_errors==True) + :rtype: Pandas data frame + :raises SpetlrException: if failed and ignore_errors==False + """ + + api_url = ( + f"{self.powerbi_url}groups/{workspace_id}" + f"/datasets/{dataset_id}/executeQueries" + ) + dax_query = { + "queries": [{"query": "EVALUATE INFO.PARTITIONS()"}], + "serializerSettings": {"includeNulls": True}, + } + schema = [ + ("[Name]", "TableName", "string"), + ( + "[RefreshedTime]", + "RefreshTime", + "timestamp", + ), + ("[ModifiedTime]", "ModifiedTime", "timestamp"), + ("[Description]", "Description", "string"), + ("[ErrorMessage]", "ErrorMessage", "string"), + ("[ID]", "Id", "long"), + ("[TableID]", "TableId", "long"), + ("[DataSourceID]", "DataSourceId", "int"), + ("[QueryDefinition]", "QueryDefinition", "string"), + ("[State]", "State", "int"), + ("[Type]", "Type", "int"), + ("[PartitionStorageID]", "PartitionStorageId", "int"), + ("[Mode]", "Mode", "int"), + ("[DataView]", "DataView", "int"), + ("[SystemFlags]", "SystemFlags", "int"), + ( + "[RetainDataTillForceCalculate]", + "RetainDataTillForceCalculate", + "boolean", + ), + ("[RangeStart]", "RangeStart", "timestamp"), + ("[RangeEnd]", "RangeEnd", "timestamp"), + ("[RangeGranularity]", "RangeGranularity", "int"), + ("[RefreshBookmark]", "RefreshBookmark", "string"), + ("[QueryGroupID]", "QueryGroupId", "int"), + ("[ExpressionSourceID]", "ExpressionSourceId", "int"), + ("[MAttributes]", "MAttributes", "int"), + ] + api_call = requests.post(url=api_url, headers=self.api_header, json=dax_query) + if api_call.status_code == 200: + return SparkPandasDataFrame( + api_call.json()["results"][0]["tables"][0]["rows"], + schema, + indexing_columns="TableId", + sorting_columns="TableName", + local_timezone_name=self.local_timezone_name, + ) + self._raise_api_error( + "Failed to fetch partition info!", + api_call, + post_body=str(dax_query), + workspace_id=workspace_id, + dataset_id=dataset_id, + ignore_unauthorized=ignore_unauthorized, + ) + return None + + def _verify_workspace(self, *, force_verify: bool = False) -> bool: """ Gets workspace ID based on the workspace name, or shows all workspaces if no parameter was specified. + :param bool force_verify: True if the workspace_id should be verified as well :return: True if succeeded or False if failed (when ignore_errors==True) :rtype: bool :raises SpetlrException: if failed and ignore_errors==False """ - if self.workspace_id is None: - api_call = requests.get( - url=f"{self.powerbi_url}groups", headers=self.api_header - ) - if api_call.status_code != 200: - self._raise_api_error("Failed to fetch workspaces!", api_call) + if self.workspace_id is None or force_verify: + workspaces = self._get_workspaces() + if workspaces is None: return False - df = pd.DataFrame(api_call.json()["value"], columns=["id", "name"]) - + df = workspaces.get_pandas_df() + if self.workspace_id is not None: + if self.workspace_id not in df["WorkspaceId"].values: + self._raise_error( + f"Workspace id '{self.workspace_id}' cannot be found!" + ) + return False + return True if self.workspace_name is None: - self._show_workspaces(df) + workspaces.show( + "Available workspaces:", + "No workspaces found.", + filter_columns=[ + ("WorkspaceId", "workspace_id"), + ("WorkspaceName", "workspace_name"), + ], + ) return False - - rows = df.loc[df["name"] == self.workspace_name, "id"] + rows = df.loc[df["WorkspaceName"] == self.workspace_name, "WorkspaceId"] if rows.empty: self._raise_error( f"Workspace name '{self.workspace_name}' cannot be found!" @@ -218,31 +527,42 @@ def _get_workspace(self) -> bool: return True - def _get_dataset(self) -> bool: + def _verify_dataset(self, *, force_verify: bool = False) -> bool: """ Gets dataset ID based on the dataset name, or shows all datasets if no parameter was specified. + :param bool force_verify: True if the dataset_id should be verified as well :return: True if succeeded or False if failed (when ignore_errors==True) :rtype: bool :raises SpetlrException: if failed and ignore_errors==False """ - if self.dataset_id is None: - api_call = requests.get( - url=f"{self.powerbi_url}groups/{self.workspace_id}/datasets", - headers=self.api_header, - ) - if api_call.status_code != 200: - self._raise_api_error("Failed to fetch datasets!", api_call) + if self.dataset_id is None or force_verify: + datasets = self._get_datasets(self.workspace_id) + if datasets is None: return False - df = pd.DataFrame(api_call.json()["value"], columns=["id", "name"]) - + df = datasets.get_pandas_df() + if df.empty and not self._verify_workspace(force_verify=True): + return False + if self.dataset_id is not None: + if self.dataset_id not in df["DatasetId"].values: + self._raise_error( + f"Dataset id '{self.dataset_id}' cannot be found!" + ) + return False + return True if self.dataset_name is None: - self._show_datasets(df) + datasets.show( + "Available datasets:", + "No datasets found.", + filter_columns=[ + ("DatasetId", "dataset_id"), + ("DatasetName", "dataset_name"), + ], + ) return False - - rows = df.loc[df["name"] == self.dataset_name, "id"] + rows = df.loc[df["DatasetName"] == self.dataset_name, "DatasetId"] if rows.empty: self._raise_error( f"Dataset name '{self.dataset_name}' cannot be found, " @@ -250,12 +570,23 @@ def _get_dataset(self) -> bool: ) return False self.dataset_id = rows.values[0] + if self.table_names is not None and ( + not self.table_names or not isinstance(self.table_names, list) + ): + tables = self._get_partition_tables(self.workspace_id, self.dataset_id) + if tables is not None: + tables.show( + "Available tables:", + "The dataset has not tables.", + filter_columns=[("TableName", "table_name")], + ) + return False return True def _connect(self) -> bool: """ - Connects or reconnects to PowerBI to fetch refresh history or trigger a refresh. + Connects or reconnects to the PowerBI API. :return: True if succeeded or False if failed (when ignore_errors==True) :rtype: bool @@ -264,140 +595,228 @@ def _connect(self) -> bool: if not self._get_access_token(): return False - if not self._get_workspace(): + if not self._verify_workspace(): return False - if not self._get_dataset(): + if not self._verify_dataset(): return False return True - def _get_refresh_history( - self, newest_only: bool = False - ) -> Union[pd.DataFrame, None]: + def _connect_and_get_workspaces( + self, + *, + skip_read_only: bool = False, + ) -> Union[List[tuple[str, str]], None]: """ - Return the PowerBI dataset refresh history. + Connects or reconnects to the PowerBI API and returns a list of + workspace id's and names. - :param bool newest_only : Limit the result to only the latest history row. - :return: data frame with the refresh history if succeeded or None if failed - (when ignore_errors==True) - :rtype: Pandas data frame + :param bool skip_read_only: True to exclude read-only workspaces. + :rtype: SparkPandas data frame :raises SpetlrException: if failed and ignore_errors==False """ - if not self._connect(): + if not self._get_access_token(): return None + if ( + self.workspace_id is not None + or self.workspace_name is not None + or self.dataset_id is not None + or self.dataset_name is not None + ): + if not self._verify_workspace(): + return None + if self.dataset_id is not None or self.dataset_name is not None: + if not self._verify_dataset(): + return None + workspace_list = [(self.workspace_id, self.workspace_name)] + if self.workspace_id is None: + workspaces = self._get_workspaces() + if workspaces is None: + return None + workspace_list = [ + (df["WorkspaceId"], df["WorkspaceName"]) + for _, df in workspaces.get_pandas_df().iterrows() + if not (skip_read_only and df["IsReadOnly"]) + ] + return workspace_list + + def _combine_datasets(self) -> Union[SparkPandasDataFrame, None]: + """ + Returns a list of PowerBI datasets for a single workspace or + combined lists for datasets from all workspaces. - self.last_status = None - self.last_exception = None - self.last_refresh_utc = None - - api_url = ( - f"{self.powerbi_url}groups/{self.workspace_id}" - f"/datasets/{self.dataset_id}/refreshes" - ) - if newest_only: - # Note: we fetch only the latest refresh record, i.e. top=1 - api_url = api_url + "?$top=1" - - api_call = requests.get(url=api_url, headers=self.api_header) - if api_call.status_code == 200: - json = api_call.json() - df = pd.DataFrame( - json["value"], - columns=[ - "id", - "refreshType", - "status", - "startTime", - "endTime", - "serviceExceptionJson", - "requestId", - "refreshAttempts", - ], - ) + :return: data frame with PowerBI datasets if succeeded, + or None if failed (when ignore_errors==True) + :rtype: SparkPandas data frame + :raises SpetlrException: if failed and ignore_errors==False + """ - self.schema = ( - "Id long, RefreshType string, Status string, Seconds long, " - f"StartTime{self.time_column_suffix} timestamp, " - f"EndTime{self.time_column_suffix} timestamp, " - "Error string, RequestId string, RefreshAttempts string" - ) + workspace_list = self._connect_and_get_workspaces() + if workspace_list is None: + return None + if self.workspace_id is not None: + return self._get_datasets(self.workspace_id) + result = None + for workspace_id, workspace_name in workspace_list: + datasets = self._get_datasets(workspace_id) + if datasets is None: + return None + prefix_columns = [("WorkspaceName", workspace_name, "string")] + suffix_columns = [("WorkspaceId", workspace_id, "string")] + result = datasets.append(result, prefix_columns, suffix_columns) + + return result + + def _combine_dataframes( + self, + function: Callable[[str, str, bool], Union[SparkPandasDataFrame, None]], + *, + skip_read_only: bool = False, + skip_not_refreshable: bool = False, + skip_effective_identity: bool = False, + ) -> Union[SparkPandasDataFrame, None]: + """ + Returns a single data frame loaded from PowerBI, or the same + combined data frame fetched from all workspaces and datasets. + + :param callable function: The function that returns the data frame + :param bool skip_read_only: True to exclude read-only workspaces. + :param bool skip_not_refreshable: True to exclude PowerBI datasets + that are not refreshable. + :param bool skip_effective_identity: True to exclude PowerBI datasets + requiring an effective identity + (effective identity is not supported in this version of spetlr). + :return: requested data frame if succeeded, + or None if failed (when ignore_errors==True) + :rtype: SparkPandas data frame + :raises SpetlrException: if failed and ignore_errors==False + """ - if df.empty: - return pd.DataFrame( - { - "Id": pd.Series(dtype="int64"), - "RefreshType": pd.Series(dtype="object"), - "Status": pd.Series(dtype="object"), - "Seconds": pd.Series(dtype="int64"), - ("StartTime" + self.time_column_suffix): pd.Series( - dtype="datetime64[ns]" - ), - ("EndTime" + self.time_column_suffix): pd.Series( - dtype="datetime64[ns]" - ), - "Error": pd.Series(dtype="object"), - "RequestId": pd.Series(dtype="object"), - "RefreshAttempts": pd.Series(dtype="object"), - } + workspace_list = self._connect_and_get_workspaces(skip_read_only=skip_read_only) + if workspace_list is None: + return None + if self.workspace_id is not None and self.dataset_id is not None: + return function(self.workspace_id, self.dataset_id) + result = None + for workspace_id, workspace_name in workspace_list: + datasets = self._get_datasets(workspace_id) + if datasets is None: + return None + for dataset_id, dataset_name in ( + (df["DatasetId"], df["DatasetName"]) + for _, df in datasets.get_pandas_df().iterrows() + if not (skip_not_refreshable and not df["IsRefreshable"]) + and not ( + skip_effective_identity + and ( + df["IsEffectiveIdentityRequired"] + or df["IsEffectiveIdentityRolesRequired"] + ) ) + and (df["ConfiguredBy"].lower() if df["ConfiguredBy"] else None) + not in self.exclude_creators + ): + data = function(workspace_id, dataset_id, True) + if data is None: + continue + prefix_columns = [ + ("WorkspaceName", workspace_name, "string"), + ("DatasetName", dataset_name, "string"), + ] + suffix_columns = [ + ("WorkspaceId", workspace_id, "string"), + ("DatasetId", dataset_id, "string"), + ] + if self.workspace_id is not None: + prefix_columns = prefix_columns[1:] + suffix_columns = suffix_columns[1:] + result = data.append(result, prefix_columns, suffix_columns) + + return result + + def _combine_refresh_history_details(self): + """ + Returns a single data frame with refresh history details, or the same + combined data frame fetched from all workspaces and datasets. + Only "ViaEnhancedApi" and "Completed" requests are included + (i.e. when the refresh was executed with the "table_names" + parameter specified). + + :return: data frame with refresh history details if succeeded, + or None if failed (when ignore_errors==True) + :rtype: SparkPandas data frame + :raises SpetlrException: if failed and ignore_errors==False + """ - df.set_index("id") - df["startTime"] = pd.to_datetime(df["startTime"]) - df["endTime"] = pd.to_datetime(df["endTime"]) - df.insert( - 3, - "seconds", - (df["endTime"] - df["startTime"]) - .astype("timedelta64[s]") - .astype("int64"), - ) - zone = timezone(self.local_timezone_name) - df["startTime"] = df["startTime"].dt.tz_convert(zone).dt.tz_localize(None) - df["endTime"] = df["endTime"].dt.tz_convert(zone).dt.tz_localize(None) - df.rename( - columns={ - "id": "Id", - "refreshType": "RefreshType", - "status": "Status", - "seconds": "Seconds", - "startTime": ("StartTime" + self.time_column_suffix), - "endTime": ("EndTime" + self.time_column_suffix), - "serviceExceptionJson": "Error", - "requestId": "RequestId", - "refreshAttempts": "RefreshAttempts", - }, - inplace=True, + history = self._combine_dataframes( + self._get_refresh_history, skip_read_only=True, skip_not_refreshable=True + ) + if history is None: + return None + result = None + for workspace_id, workspace_name, dataset_id, dataset_name, request_id in ( + ( + self.workspace_id if self.workspace_id else df["WorkspaceId"], + self.workspace_name if self.workspace_id else df["WorkspaceName"], + self.dataset_id if self.dataset_id else df["DatasetId"], + self.dataset_name if self.dataset_id else df["DatasetName"], + df["RequestId"], ) - return df - - elif api_call.status_code == 404: - self._raise_error( - "The specified dataset or workspace cannot be found, " - "or the dataset doesn't have a user with the required permissions!" + for _, df in history.get_pandas_df().iterrows() + if df["RefreshType"] == "ViaEnhancedApi" and df["Status"] == "Completed" + ): + data = self._get_refresh_history_details( + workspace_id, dataset_id, request_id, True ) - else: - self._raise_api_error("Failed to fetch refresh history!", api_call) - return None - - def _get_last_refresh(self) -> bool: + if data is None: + continue + prefix_columns = [ + ("WorkspaceName", workspace_name, "string"), + ("DatasetName", dataset_name, "string"), + ] + suffix_columns = [ + ("WorkspaceId", workspace_id, "string"), + ("DatasetId", dataset_id, "string"), + ("RequestId", request_id, "string"), + ] + if self.dataset_id is not None: + prefix_columns = prefix_columns[2:] + suffix_columns = suffix_columns[2:] + elif self.workspace_id is not None: + prefix_columns = prefix_columns[1:] + suffix_columns = suffix_columns[1:] + result = data.append(result, prefix_columns, suffix_columns) + + return result + + def _get_last_refresh(self, *, finished_only: bool = False) -> bool: """ Gets the latest record in the PowerBI dataset refresh history. + :param bool finished_only: Ignore history row where refresh is in progress :return: True if succeeded or False if failed (when ignore_errors==True) :rtype: bool :raises SpetlrException: if failed and ignore_errors==False """ - df = self._get_refresh_history() - if df is None: + self.last_status = None + self.last_exception = None + self.last_refresh_utc = None + self.table_name = None + if not self._connect(): return False - + history = self._get_refresh_history(self.workspace_id, self.dataset_id) + if history is None: + return False + df = history.get_pandas_df() if not df.empty: - self.last_status = df.Status.iloc[0] - self.last_exception = df.Error.iloc[0] - # claculate the average duration of all previous API refresh calls - # without any tables specified + skip = finished_only and df.Status.iloc[0] == "Unknown" and df.shape[0] > 1 + first = 1 if skip else 0 + self.last_status = df.Status.iloc[first] + self.last_exception = df.Error.iloc[first] + # calculate the average duration of all previous API refresh calls + # when there were no table names specified mean = df.loc[ (df["RefreshType"] == "ViaApi") & (df["Status"] == "Completed"), df.Seconds.name, @@ -406,14 +825,33 @@ def _get_last_refresh(self) -> bool: self.last_duration_in_seconds = 0 else: self.last_duration_in_seconds = int(mean) - zone = timezone(self.local_timezone_name) if self.last_status == "Completed": - if self.is_utc: - self.last_refresh_utc = utc.localize(df.EndTimeUtc.iloc[0]) - else: - self.last_refresh_utc = zone.localize( - df.EndTimeLocal.iloc[0] - ).astimezone(utc) + refresh_time = df["EndTime" + history.time_column_suffix].iloc[first] + if self.table_names is not None: + tables = self._get_partition_tables( + self.workspace_id, self.dataset_id, ignore_unauthorized=True + ) + if tables is not None: + df = tables.get_pandas_df() + df = df[ + df.TableName.isin(self.table_names) + & df.ErrorMessage.notnull() + ] + if not df.empty: + self.last_status = "Failed" + self.last_exception = df.ErrorMessage.iloc[0] + self.table_name = df.TableName.iloc[0] + return True + else: + df = tables.get_pandas_df() + df = df[df.TableName.isin(self.table_names)] + column = "RefreshTime" + tables.time_column_suffix + idx = df[column].idxmin() + self.table_name = df.TableName.loc[idx] + refresh_time = df[column].loc[idx] + + self.last_refresh_utc = history.get_utc_time(refresh_time) + self.last_refresh_str = history.get_local_time_str(refresh_time) return True @@ -433,9 +871,6 @@ def _verify_last_refresh(self) -> bool: if self.last_refresh_utc is None: self._raise_error("Completed at unknown refresh time!") else: - last_refresh_str = self.last_refresh_utc.astimezone( - timezone(self.local_timezone_name) - ).strftime("%Y-%m-%d %H:%M") + (self.time_description_suffix) min_refresh_time_utc = datetime.now(utc) - timedelta( minutes=self.max_minutes_after_last_refresh ) @@ -444,45 +879,84 @@ def _verify_last_refresh(self) -> bool: and self.last_refresh_utc < min_refresh_time_utc ): self._raise_error( - "Last refresh finished more than " - f"{self.max_minutes_after_last_refresh} " - f"minutes ago at {last_refresh_str} !" + ( + "Last refresh finished more than " + f"{self.max_minutes_after_last_refresh} " + f"minutes ago at {self.last_refresh_str} !" + ) + if self.table_name is None + else ( + f"Last refresh of the table '{self.table_name}' finished " + f"more than {self.max_minutes_after_last_refresh} " + f"minutes ago at {self.last_refresh_str} !" + ) ) else: - print(f"Refresh completed successfully at {last_refresh_str}.") + print( + f"Refresh completed successfully at {self.last_refresh_str}." + if self.table_name is None + else ( + f"Last refresh of '{self.table_name}' (the oldest in " + "the specified tables) completed successfully " + f"at {self.last_refresh_str}." + ) + ) return True elif self.last_status == "Unknown": self._raise_error("Refresh is still in progress!") elif self.last_status == "Disabled": self._raise_error("Refresh is disabled!") elif self.last_status == "Failed": - self._raise_error(f"Last refresh failed! {self.last_exception}") + self._raise_error( + f"Last refresh failed! {self.last_exception}" + if self.table_name is None + else f"Last refresh for the table '{self.table_name}' " + f"failed! {self.last_exception}" + ) else: self._raise_error( f"Unknown refresh status: {self.last_status}! {self.last_exception}" ) return False - def _get_table_names_json(self) -> Union[Dict[str, object], None]: + def _get_refresh_argument_json( + self, with_wait: bool + ) -> Union[Dict[str, object], None]: """ Returns the HTTP body of the PowerBI refresh API call - containing table names to refresh. + containing table names to refresh or other parameters if necessary. + :param bool with_wait: True if we need to wait for the refresh to finish. :rtype: JSON object or None """ + + result = {} if self.table_names: - return { + result = { "type": "full", "commitMode": "transactional", "objects": [{"table": table} for table in self.table_names], "applyRefreshPolicy": "false", } - return None + elif self.mail_on_failure or self.mail_on_completion: + result["notifyOption"] = ( + "MailOnCompletion" if self.mail_on_completion else "MailOnFailure" + ) + if self.number_of_retries > 0 and not with_wait: + if self.table_names: + result["retryCount"] = self.number_of_retries + else: + print( + "The 'number_of_retries' parameter is ignored in " + "start_refresh() if 'table_names' is not specified!" + ) + return result if result else None - def _trigger_new_refresh(self) -> bool: + def _trigger_new_refresh(self, with_wait: bool) -> bool: """ Starts a refresh of the PowerBI dataset. + :param bool with_wait: True if we need to wait for the refresh to finish. :return: True if succeeded or False if failed (when ignore_errors==True) :rtype: bool :raises SpetlrException: if failed and ignore_errors==False @@ -493,18 +967,25 @@ def _trigger_new_refresh(self) -> bool: print(f"Warning: Last refresh failed! {self.last_exception}") print() + post_body = self._get_refresh_argument_json(with_wait) api_url = ( f"{self.powerbi_url}groups/{self.workspace_id}" f"/datasets/{self.dataset_id}/refreshes" ) api_call = requests.post( - url=api_url, headers=self.api_header, json=self._get_table_names_json() + url=api_url, headers=self.api_header, json=post_body ) if api_call.status_code == 202: print("A new refresh has been successfully triggered.") return True else: - self._raise_api_error("Failed to trigger a refresh!", api_call) + self._raise_api_error( + "Failed to trigger a refresh!", + api_call, + post_body=str(post_body), + workspace_id=self.workspace_id, + dataset_id=self.dataset_id, + ) elif self.last_status == "Unknown": print("Refresh is already in progress!") return True @@ -522,6 +1003,7 @@ def _get_seconds_to_wait(self, elapsed_seconds: int) -> int: if the refresh has completed. The method makes sure as few requests to the PowerBI API as possible would be made. + :param int elapsed_seconds: The number of seconds elapsed so far. :return: number of seconds to wait :rtype: int """ @@ -549,15 +1031,17 @@ def _get_seconds_to_wait(self, elapsed_seconds: int) -> int: def check(self) -> bool: """ - Checks if the last refresh of the PowerBI dataset completed successfully, - and verifies if it happened recently enough. + Checks if the last refresh of the PowerBI dataset or its selected tables + completed successfully, and verifies if it happened recently enough. :return: True if succeeded or False if failed (when ignore_errors==True) :rtype: bool :raises SpetlrException: if failed and ignore_errors==False """ - return self._get_last_refresh() and self._verify_last_refresh() + return ( + self._get_last_refresh(finished_only=True) and self._verify_last_refresh() + ) def start_refresh(self) -> bool: """ @@ -568,7 +1052,7 @@ def start_refresh(self) -> bool: :raises SpetlrException: if failed and ignore_errors==False """ - return self._get_last_refresh() and self._trigger_new_refresh() + return self._get_last_refresh() and self._trigger_new_refresh(with_wait=False) def refresh(self) -> bool: """ @@ -581,7 +1065,7 @@ def refresh(self) -> bool: retries = self.number_of_retries start_time = time.time() - if not self.start_refresh(): + if not (self._get_last_refresh() and self._trigger_new_refresh(with_wait=True)): return False while True: @@ -598,7 +1082,7 @@ def refresh(self) -> bool: if self.last_status == "Failed" and retries > 0: retries = retries - 1 - if not self._trigger_new_refresh(): + if not self._trigger_new_refresh(with_wait=True): return False continue @@ -611,33 +1095,134 @@ def show_history(self) -> None: """ Displays the refresh history of a PowerBI dataset. - :return: data frame with the refresh history if succeeded or None if failed - (when ignore_errors==True) - :rtype: Pandas data frame + :rtype: None :raises SpetlrException: if failed and ignore_errors==False """ - df = self._get_refresh_history() - if df is None: - return None - if df.empty: - print("The refresh history list is empty.") - df.display() + history = self._combine_dataframes( + self._get_refresh_history, skip_read_only=True, skip_not_refreshable=True + ) + if history is not None: + history.show("Refresh history:", "The refresh history is empty.") def get_history(self) -> Union[DataFrame, None]: """ Returns the refresh history of a PowerBI dataset in a Spark data frame. - :return: data frame with the refresh history if succeeded or None if failed - (when ignore_errors==True) + :return: the data frame with the refresh history if succeeded, + or None if failed (when ignore_errors==True) :rtype: Spark data frame :raises SpetlrException: if failed and ignore_errors==False """ - df = self._get_refresh_history() - if df is None: - return None - if df.empty: - return Spark.get().createDataFrame(df, self.schema) - else: - return Spark.get().createDataFrame(df) + history = self._combine_dataframes( + self._get_refresh_history, skip_read_only=True, skip_not_refreshable=True + ) + return history.get_spark_df() if history is not None else None + + def show_history_details(self) -> None: + """ + Displays refresh history details of a PowerBI dataset. + + :rtype: None + :raises SpetlrException: if failed and ignore_errors==False + """ + + details = self._combine_refresh_history_details() + if details is not None: + details.show("Refresh history details:", "The refresh history is empty.") + + def get_history_details(self) -> Union[DataFrame, None]: + """ + Returns refresh history details of a PowerBI dataset in a Spark data frame. + + :return: the data frame with the refresh history details if succeeded, + or None if failed (when ignore_errors==True) + :rtype: Spark data frame + :raises SpetlrException: if failed and ignore_errors==False + """ + + history = self._combine_refresh_history_details() + return history.get_spark_df() if history is not None else None + + def show_tables(self) -> None: + """ + Displays the tables of a PowerBI dataset. + + :rtype: None + :raises SpetlrException: if failed and ignore_errors==False + """ + + tables = self._combine_dataframes( + self._get_partition_tables, skip_effective_identity=True + ) + if tables is not None: + tables.show("Dataset tables:", "No dataset tables found.") + + def get_tables(self) -> Union[DataFrame, None]: + """ + Returns the tables of a PowerBI dataset as a Spark data frame. + + :return: the data frame with the partition tables if succeeded, + or None if failed (when ignore_errors==True) + :rtype: Spark data frame + :raises SpetlrException: if failed and ignore_errors==False + """ + + tables = self._combine_dataframes( + self._get_partition_tables, skip_effective_identity=True + ) + return tables.get_spark_df() if tables is not None else None + + def show_workspaces(self) -> None: + """ + Displays the available workspaces. + + :rtype: None + :raises SpetlrException: if failed and ignore_errors==False + """ + + if self._get_access_token(): + workspaces = self._get_workspaces() + if workspaces is not None: + workspaces.show("Workspaces:", "No workspaces found.") + + def get_workspaces(self) -> Union[DataFrame, None]: + """ + Returns the available workspaces as a Spark data frame. + + :return: the data frame with the workspaces if succeeded, + or None if failed (when ignore_errors==True) + :rtype: Spark data frame + :raises SpetlrException: if failed and ignore_errors==False + """ + + if self._get_access_token(): + workspaces = self._get_workspaces() + return workspaces.get_spark_df() if workspaces is not None else None + return None + + def show_datasets(self) -> None: + """ + Displays the available datasets. + + :rtype: None + :raises SpetlrException: if failed and ignore_errors==False + """ + + datasets = self._combine_datasets() + if datasets is not None: + datasets.show("Datasets:", "No datasets found.") + + def get_datasets(self) -> Union[DataFrame, None]: + """ + Returns the available datasets as a Spark data frame. + + :return: the data frame with the datasets if succeeded, + or None if failed (when ignore_errors==True) + :rtype: Spark data frame + :raises SpetlrException: if failed and ignore_errors==False + """ + + datasets = self._combine_datasets() + return datasets.get_spark_df() if datasets is not None else None diff --git a/src/spetlr/power_bi/SparkPandasDataFrame.py b/src/spetlr/power_bi/SparkPandasDataFrame.py new file mode 100644 index 00000000..61b82588 --- /dev/null +++ b/src/spetlr/power_bi/SparkPandasDataFrame.py @@ -0,0 +1,314 @@ +from datetime import datetime +from typing import Callable, Dict, List, Union + +import numpy as np +import pandas as pd +from dateutil.parser import parse +from pandas.core.generic import NDFrameT +from pyspark.sql import DataFrame +from pytz import timezone, utc + +from spetlr.exceptions import SpetlrException +from spetlr.spark import Spark + + +class SparkPandasDataFrame: + # converts data types from Spark to Pandas and vice versa + SPARK_TO_PANDAS_TYPES = { + "long": "Int64", + "int": "Int64", + "timestamp": "datetime64[ns]", + "boolean": "bool", + "string": "object", + "default": "object", + } + + def __init__( + self, + json: Union[Dict, List], + schema: List[tuple[Union[str, Callable[[pd.DataFrame], NDFrameT]], str, str]], + *, + indexing_columns: Union[int, List[int], str, List[str], None] = None, + sorting_columns: Union[int, List[int], str, List[str], None] = None, + ascending: bool = True, + local_timezone_name: str = None, + ): + """ + Provides a class for loading a JSON into Pandas, and for converting + between Pandas and Spark. + + :param dict json: Data source, which is a dictionary representing a JSON. + :param dict[tuple[str/callable, str, str]: schema definition where the first + value in the tuple is the original column name in the JSON, the second + is the desired column name in the output, and the third is the Spark type. + The first value can also be a lambda expression taking a Pandas + data frame as input and calculating the column value from + other available columns, e.g. lambda df: df["endTime"] - df["startTime"] + :param int/str indexing_columns: indexes (within the schema) or names + of the indexing columns. + :param int/str sorting_columns: indexes (within the schema) or names + of the sorting columns. + :param str local_timezone_name: The timezone to use when parsing + timestamp columns. The default timezone is UTC. + If the timezone is UTC, all timestamp columns will have a suffix "Utc". + Otherwise they will have a suffix "Local". + """ + + self.local_timezone_name = ( + local_timezone_name if local_timezone_name is not None else "UTC" + ) + self.is_utc = self.local_timezone_name.upper() == "UTC" + self.time_column_suffix = "Utc" if self.is_utc else "Local" + self.schema = schema + + for i, column in enumerate(schema): + if column[2] == "timestamp": + schema[i] = (column[0], column[1] + self.time_column_suffix, column[2]) + df = pd.DataFrame( + json, + columns=[column[0] for column in schema if isinstance(column[0], str)], + ) + if df.empty: + self.df = pd.DataFrame() + else: + df = df.replace(np.nan, None) + if indexing_columns is not None: + df.set_index(self._get_columns(indexing_columns, False)) + for column in schema: + if isinstance(column[0], str): + if column[2] == "timestamp": + df[column[0]] = df[column[0]].apply( + lambda text: self._parse_time(text) + ) + else: + df[column[0]] = df[column[0]].astype( + self.SPARK_TO_PANDAS_TYPES.get( + column[2], self.SPARK_TO_PANDAS_TYPES["default"] + ) + ) + for i, column in enumerate(schema): + if callable(column[0]): + df.insert( + i, + column[1], + column[0](df).astype( + self.SPARK_TO_PANDAS_TYPES.get( + column[2], self.SPARK_TO_PANDAS_TYPES["default"] + ) + ), + ) + for column in schema: + if column[2] == "timestamp" and isinstance(column[0], str): + df[column[0]] = df[column[0]].apply( + lambda text: self._localize_time(text, self.local_timezone_name) + ) + df = df.rename( + columns=dict( + (column[0], column[1]) + for column in schema + if isinstance(column[0], str) + ), + ) + if sorting_columns is not None: + df = df.sort_values( + self._get_columns(sorting_columns, True), ascending=ascending + ) + self.df = df + + def _get_columns( + self, identifiers: Union[int, List[int], str, List[str]], target: bool + ) -> Union[str, List[str]]: + """ + Returns the name of the specified column in the schema. + + :param int/str identifiers: the column(s) to look for, which is either + the column index(es), or column source name(s), or column target name(s). + :param bool target: return the target column name if target==True, + or source column name otherwise. + :return: the requested column + :rtype: str + :raises SpetlrException: if the column cannot be found + """ + + result = [] + if isinstance(identifiers, int) or isinstance(identifiers, str): + identifiers = [identifiers] + if isinstance(identifiers, list): + for identifier in identifiers: + if isinstance(identifier, int) and identifier < len(self.schema): + result.append(self.schema[identifier][int(target)]) + elif isinstance(identifier, str): + result.append( + next( + ( + column[int(target)] + for column in self.schema + if identifier == column[0] or identifier == column[1] + ), + None, + ) + ) + if not result or ( + isinstance(identifiers, list) and len(result) != len(identifiers) + ): + raise SpetlrException(f"{identifiers} column(s) not found in the schema") + return result if len(result) > 1 else result[0] + + @staticmethod + def _parse_time(text: str) -> Union[datetime, None]: + """ + Parses an ISO 8601 time text and converts it to datetime. + The result is a timezone-unaware UTC datetime regardless of input. + + :param str text: an ISO 8601 time text or None + :return: timezone-unaware UTC datetime or None + :rtype: datetime + """ + + if text is None or text != text: + return None + time = parse(text) + if time.tzinfo is None or time.tzinfo.utcoffset(time) is None: + time = utc.localize(time) + else: + time = time.astimezone(utc) + return time.replace(microsecond=0, tzinfo=None) + + @staticmethod + def _localize_time( + time: Union[datetime, None], local_timezone_name: str + ) -> Union[datetime, None]: + """ + Converts a timezone-unaware UTC datetime to a timezone-unaware local datetime. + + :param datetime time: timezone-unaware UTC datetime or None + :return: timezone-unaware local datetime or None + :rtype: datetime + """ + + if time is None or time != time: + return None + if time.tzinfo is None or time.tzinfo.utcoffset(time) is None: + time = utc.localize(time) + time = time.astimezone(timezone(local_timezone_name)) + return time.replace(tzinfo=None) + + def get_pandas_df(self) -> Union[pd.DataFrame, None]: + """ + Returns the data frame as a Pandas data frame. + + :return: Pandas data frame + :rtype: Pandas data frame + """ + + return self.df + + def get_spark_df(self) -> Union[DataFrame, None]: + """ + Returns the data frame as a Spark data frame. + + :return: Spark data frame + :rtype: Spark data frame + """ + + df = self.df + if df is None: + return None + spark_schema = ", ".join(f"{column[1]} {column[2]}" for column in self.schema) + if df.empty: + return Spark.get().createDataFrame(df, spark_schema) + else: + return Spark.get().createDataFrame(df) + + def show( + self, + message: str, + when_empty: str, + *, + filter_columns: Union[List[tuple[str, str]], None] = None, + ) -> None: + """ + Displays the Pandas data frame. + + :param str message: The message to show when the data frame is not empty. + :param str when_empty: The message to show when the data frame is empty. + Note: Pandas fails when showing empty data frames. + :param list[tuple[str, str]] filter_columns: a subset of columns to show + and their new names + :rtype: None + """ + + if self.df is None or self.df.empty: + print(when_empty) + else: + print(message) + df = self.df + if filter_columns: + df = df.rename(columns=dict(filter_columns)) + df = df[[column[1] for column in filter_columns]] + df.display() + + def get_local_time_str(self, time: datetime) -> Union[str, None]: + """ + Returns the specified time as a local timestamp converted to text. + + :param datetime time: the time to convert from + :return: text of a timezone-aware local datetime or None + :rtype: str + """ + + time_description_suffix = " (UTC)" if self.is_utc else " (local time)" + if time is not None: + return time.strftime("%Y-%m-%d %H:%M") + time_description_suffix + return None + + def get_utc_time(self, time: datetime) -> Union[datetime, None]: + """ + Returns the specified time as an UTC timestamp. + + :param datetime time: the time to convert from + :return: a timezone-aware UTC datetime or None + :rtype: datetime + """ + + zone = timezone(self.local_timezone_name) + if time is not None: + if self.is_utc: + time = utc.localize(time) + else: + time = zone.localize(time).astimezone(utc) + return time + + def append( + self, + source, + prefix_columns: List[tuple[str, str, str]], + suffix_columns: List[tuple[str, str, str]], + ): + """ + Appends another instance of this class to this data frame, which will result + in a union of two Pandas data frames. Additionally, new columns should + be added to this instance, which are already present in the other instance. + This will allow to make a distinction between both data frames. + + :param SparkPandasDataFrame source: another object of this class + that needs to be appended to this data frame + :param list[tuple[str, str]] prefix_columns: new columns with values + to add at the beginning + :param list[tuple[str, str]] suffix_columns: new columns with values + to add at the end + :rtype: SparkPandasDataFrame + """ + for column in reversed(prefix_columns): + self.schema.insert(0, (column[0], column[0], column[2])) + if not self.df.empty: + self.df.insert(0, column[0], column[1]) + for column in suffix_columns: + self.schema.append((column[0], column[0], column[2])) + if not self.df.empty: + self.df[column[0]] = column[1] + if source is not None and not source.get_pandas_df().empty: + self.df = pd.concat([source.get_pandas_df(), self.df]) + if not self.df.empty: + self.df = self.df.reset_index(drop=True) + return self diff --git a/tests/local/power_bi/test_power_bi.py b/tests/local/power_bi/test_power_bi.py index 0f5894a2..e8385992 100644 --- a/tests/local/power_bi/test_power_bi.py +++ b/tests/local/power_bi/test_power_bi.py @@ -13,7 +13,7 @@ class TestPowerBi(unittest.TestCase): @patch("requests.get") - def test_get_workspace_success(self, mock_get): + def test_verify_workspace_success(self, mock_get): # Arrange mock_response = Mock() mock_response.status_code = 200 @@ -23,103 +23,963 @@ def test_get_workspace_success(self, mock_get): mock_get.return_value = mock_response sut = PowerBi(PowerBiClient(), workspace_name="Finance") - sut.powerbi_url = "test" + sut.powerbi_url = "test/" # Act - result = sut._get_workspace() + result = sut._verify_workspace() # Assert self.assertTrue(result) self.assertEqual("614850c2-3a5c-4d2d-bcaa-d3f20f32a2e0", sut.workspace_id) @patch("requests.get") - def test_get_workspace_failure(self, mock_get): + def test_verify_workspace_failure(self, mock_get): + # Arrange + mock_response = Mock() + mock_response.status_code = 401 + mock_response.text = "error" + mock_get.return_value = mock_response + + sut = PowerBi(PowerBiClient(), workspace_name="Finance") + sut.powerbi_url = "test/" + + # Act + with self.assertRaises(SpetlrException) as context: + sut._verify_workspace() + + # Assert + self.assertIn("Failed to fetch workspaces", str(context.exception)) + + @patch("requests.get") + def test_verify_dataset_success(self, mock_get): + # Arrange + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "value": [ + {"id": "b1f0a07e-e348-402c-a2b2-11f3e31181ce", "name": "Invoicing"} + ] + } + mock_get.return_value = mock_response + + sut = PowerBi( + PowerBiClient(), + workspace_name="Finance", + dataset_name="Invoicing", + ) + sut.powerbi_url = "test/" + + # Act + result = sut._verify_dataset() + + # Assert + self.assertTrue(result) + self.assertEqual("b1f0a07e-e348-402c-a2b2-11f3e31181ce", sut.dataset_id) + + @patch("requests.get") + def test_verify_dataset_failure(self, mock_get): # Arrange mock_response = Mock() mock_response.status_code = 404 mock_response.text = "error" mock_get.return_value = mock_response - sut = PowerBi(PowerBiClient(), workspace_name="Finance") - sut.powerbi_url = "test" + sut = PowerBi( + PowerBiClient(), + workspace_name="Finance", + dataset_name="Invoicing", + ) + sut.powerbi_url = "test/" + + # Act + with self.assertRaises(SpetlrException) as context: + sut._verify_dataset() + + # Assert + self.assertIn("Failed to fetch datasets", str(context.exception)) + + @patch("requests.get") + def test_get_refresh_history_success(self, mock_get): + # Arrange + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "value": [ + { + "id": 1, + "refreshType": "OnDemand", + "status": "Completed", + "startTime": "2024-02-26T10:00:00Z", + "endTime": "2024-02-26T10:05:00Z", # winter time: 1 hour + "serviceExceptionJson": None, # difference from UTC + "requestId": "74d25c0b-0473-4dd9-96ff-3ca737b072a7", + "refreshAttempts": None, + }, + { + "id": 2, + "refreshType": "ViaEnhancedApi", + "status": "Completed", + "startTime": "2024-03-31T00:00:00Z", # summer time: change from 1 + "endTime": "2024-03-31T02:00:00Z", # hour to 2 hours difference + "serviceExceptionJson": None, + "requestId": "aec28227-f7af-4c2d-a4e6-fcb01cd570ec", + "refreshAttempts": None, + }, + ] + } + + def requests_get(url, headers): + nonlocal mock_response + if url.endswith("refreshes") and headers == "api_header": + return mock_response + raise ValueError("Unknown URL!") + + mock_get.side_effect = requests_get + + sut = PowerBi( + PowerBiClient(), + workspace_id="test", + dataset_id="test", + local_timezone_name="Europe/Copenhagen", + ) + sut.api_header = "api_header" + sut.powerbi_url = "test/" + sut._get_access_token = lambda: True + expected = pd.DataFrame( + { + "RefreshType": ["OnDemand", "ViaEnhancedApi"], + "Status": ["Completed", "Completed"], + "Seconds": [ + 300, + 7200, + ], # despite the daylight saving change it + # correctly shows 7200 seconds ! + "StartTimeLocal": [ + datetime(2024, 2, 26, 11, 0), + datetime(2024, 3, 31, 1, 0), + ], + "EndTimeLocal": [ + datetime(2024, 2, 26, 11, 5), + datetime(2024, 3, 31, 4, 0), + ], + "Error": [None, None], + "RequestId": [ + "74d25c0b-0473-4dd9-96ff-3ca737b072a7", + "aec28227-f7af-4c2d-a4e6-fcb01cd570ec", + ], + "RefreshId": [1, 2], + "RefreshAttempts": None, + } + ) + expected["RefreshId"] = expected["RefreshId"].astype("Int64") + expected["Seconds"] = expected["Seconds"].astype("Int64") + + # Act + result = sut._combine_dataframes(sut._get_refresh_history) + + # Assert + self.assertIsNotNone(result) + assert_frame_equal(expected, result.get_pandas_df()) + + @patch("requests.get") + def test_get_refresh_history_failure(self, mock_get): + # Arrange + mock_response = Mock() + mock_response.status_code = 404 # dataset or workspace not found + mock_get.return_value = mock_response + + sut = PowerBi(PowerBiClient(), workspace_id="test", dataset_id="test") + sut.powerbi_url = "test/" + sut._get_access_token = lambda: True + + # Act + with self.assertRaises(SpetlrException) as context: + sut._combine_dataframes(sut._get_refresh_history) + + # Assert + self.assertIn( + "The specified dataset or workspace cannot be found", str(context.exception) + ) + + @patch("requests.post") + def test_get_partition_tables_success(self, mock_post): + # Arrange + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "results": [ + { + "tables": [ + { + "rows": [ + { + "[ID]": 1, + "[TableID]": 1001, + "[Name]": "Invoices", + "[RefreshedTime]": "2024-02-26T10:00:00Z", + "[ModifiedTime]": "2024-02-26T10:05:00Z", + "[Description]": "contains invoices", + "[ErrorMessage]": None, + }, + { + "[ID]": 2, + "[TableID]": 1002, + "[Name]": "Customers", + "[RefreshedTime]": "2024-02-26T15:00:00Z", + "[ModifiedTime]": "2024-02-26T15:05:00Z", + "[Description]": "contains customers", + "[ErrorMessage]": "Refresh error", + }, + ] + } + ] + } + ] + } + mock_post.return_value = mock_response + + sut = PowerBi( + PowerBiClient(), + workspace_id="test", + dataset_id="test", + local_timezone_name="Europe/Copenhagen", + ) + sut.powerbi_url = "test/" + sut._get_access_token = lambda: True + expected = pd.DataFrame( + { + "TableName": ["Invoices", "Customers"], + "RefreshTimeLocal": [ + datetime(2024, 2, 26, 11, 0), + datetime(2024, 2, 26, 16, 0), + ], + "ModifiedTimeLocal": [ + datetime(2024, 2, 26, 11, 5), + datetime(2024, 2, 26, 16, 5), + ], + "Description": ["contains invoices", "contains customers"], + "ErrorMessage": [None, "Refresh error"], + "Id": [1, 2], + "TableId": [1001, 1002], + } + ).sort_values("TableName") + expected["Id"] = expected["Id"].astype("Int64") + expected["TableId"] = expected["TableId"].astype("Int64") + + # Act + result = sut._combine_dataframes(sut._get_partition_tables) + + # Assert + self.assertIsNotNone(result) + assert_frame_equal( + expected, result.get_pandas_df()[list(expected.columns.values)] + ) + + @patch("requests.post") + def test_get_partition_tables_failure(self, mock_post): + # Arrange + mock_response = Mock() + mock_response.status_code = 401 + mock_post.return_value = mock_response + + sut = PowerBi(PowerBiClient(), workspace_id="test", dataset_id="test") + sut.powerbi_url = "test/" + sut._get_access_token = lambda: True + + # Act + with self.assertRaises(SpetlrException) as context: + sut._combine_dataframes(sut._get_partition_tables) + + # Assert + self.assertIn("Failed to fetch partition info", str(context.exception)) + + @patch("requests.get") + def test_combine_datasets_on_workspace_level_with_success(self, mock_get): + # Arrange + get_workspaces = { + "value": [ + { + "id": "614850c2-3a5c-4d2d-bcaa-d3f20f32a2e0", + "name": "Finance", + "isReadOnly": False, + }, + { + "id": "5da990e9-089e-472c-a7fa-4fc3dd096d01", + "name": "CRM", + "isReadOnly": True, + }, + ] + } + get_finance_datasets = { + "value": [ + { + "id": "b1f0a07e-e348-402c-a2b2-11f3e31181ce", + "name": "Invoicing", + "configuredBy": "james@contoso.com", + "isRefreshable": True, + "isEffectiveIdentityRequired": False, + "isEffectiveIdentityRolesRequired": False, + }, + { + "id": "2e848e9a-47a3-4b0e-a22a-af35507ec8c4", + "name": "Reimbursement", + "configuredBy": "olivia@contoso.com", + "isRefreshable": True, + "isEffectiveIdentityRequired": False, + "isEffectiveIdentityRolesRequired": False, + }, + { + "id": "4de28a6f-f7d4-4186-a529-bf6c65e67b31", + "name": "Fees", + "configuredBy": "evelyn@contoso.com", + "isRefreshable": False, + "isEffectiveIdentityRequired": False, + "isEffectiveIdentityRolesRequired": False, + }, + { + "id": "c3c7591a-35bf-4f25-b03c-9c5ed6cdae14", + "name": "Fraud", + "configuredBy": "evelyn@contoso.com", + "isRefreshable": True, + "isEffectiveIdentityRequired": True, + "isEffectiveIdentityRolesRequired": False, + }, + ] + } + get_crm_datasets = { + "value": [ + { + "id": "4869cc38-646c-45b6-89fd-62615beb9853", + "name": "Strategy", + "configuredBy": "mark@contoso.com", + "isRefreshable": True, + "isEffectiveIdentityRequired": False, + "isEffectiveIdentityRolesRequired": True, + }, + { + "id": "68a2565d-4959-4421-b91c-bafe792796e1", + "name": "Strategy", + "configuredBy": "amelia@contoso.com", + "isRefreshable": False, + "isEffectiveIdentityRequired": False, + "isEffectiveIdentityRolesRequired": True, + }, + ] + } + + def requests_get(url, headers): + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = None + if headers == "api_header": + if url.endswith("groups"): + mock_response.json.return_value = get_workspaces + elif ( + url.endswith("datasets") + and "614850c2-3a5c-4d2d-bcaa-d3f20f32a2e0" in url + ): + mock_response.json.return_value = get_finance_datasets + elif ( + url.endswith("datasets") + and "5da990e9-089e-472c-a7fa-4fc3dd096d01" in url + ): + mock_response.json.return_value = get_crm_datasets + if mock_response.json.return_value or mock_response.status_code != 200: + return mock_response + raise ValueError("Unknown URL! " + url) + + mock_get.side_effect = requests_get + + sut = PowerBi( + PowerBiClient(), + local_timezone_name="Europe/Copenhagen", + ) + sut.api_header = "api_header" + sut.powerbi_url = "test/" + sut._get_access_token = lambda: True + expected = pd.DataFrame( + { + "WorkspaceName": [ + "CRM", + "CRM", + "Finance", + "Finance", + "Finance", + "Finance", + ], + "DatasetName": [ + "Strategy", + "Strategy", + "Fees", + "Fraud", + "Invoicing", + "Reimbursement", + ], + "ConfiguredBy": [ + "amelia@contoso.com", + "mark@contoso.com", + "evelyn@contoso.com", + "evelyn@contoso.com", + "james@contoso.com", + "olivia@contoso.com", + ], + "IsRefreshable": [False, True, False, True, True, True], + "AddRowsApiEnabled": [False, False, False, False, False, False], + "IsEffectiveIdentityRequired": [ + False, + False, + False, + True, + False, + False, + ], + "IsEffectiveIdentityRolesRequired": [ + True, + True, + False, + False, + False, + False, + ], + "IsOnPremGatewayRequired": [False, False, False, False, False, False], + "DatasetId": [ + "68a2565d-4959-4421-b91c-bafe792796e1", + "4869cc38-646c-45b6-89fd-62615beb9853", + "4de28a6f-f7d4-4186-a529-bf6c65e67b31", + "c3c7591a-35bf-4f25-b03c-9c5ed6cdae14", + "b1f0a07e-e348-402c-a2b2-11f3e31181ce", + "2e848e9a-47a3-4b0e-a22a-af35507ec8c4", + ], + "WorkspaceId": [ + "5da990e9-089e-472c-a7fa-4fc3dd096d01", + "5da990e9-089e-472c-a7fa-4fc3dd096d01", + "614850c2-3a5c-4d2d-bcaa-d3f20f32a2e0", + "614850c2-3a5c-4d2d-bcaa-d3f20f32a2e0", + "614850c2-3a5c-4d2d-bcaa-d3f20f32a2e0", + "614850c2-3a5c-4d2d-bcaa-d3f20f32a2e0", + ], + } + ) + + # Act + result = sut._combine_datasets() + + # Assert + self.assertIsNotNone(result) + assert_frame_equal(expected, result.get_pandas_df()) + + @patch("requests.get") + def test_combine_dataframes_on_workspace_level_with_success(self, mock_get): + # Arrange + get_workspaces = { + "value": [ + { + "id": "614850c2-3a5c-4d2d-bcaa-d3f20f32a2e0", + "name": "Finance", + "isReadOnly": False, + }, + { + "id": "5da990e9-089e-472c-a7fa-4fc3dd096d01", + "name": "CRM", + "isReadOnly": True, # excluded + }, + ] + } + get_finance_datasets = { + "value": [ + { + "id": "b1f0a07e-e348-402c-a2b2-11f3e31181ce", + "name": "Invoicing", + "configuredBy": "james@contoso.com", + "isRefreshable": True, + "isEffectiveIdentityRequired": False, + "isEffectiveIdentityRolesRequired": False, + }, + { + "id": "2e848e9a-47a3-4b0e-a22a-af35507ec8c4", + "name": "Reimbursement", + "configuredBy": "olivia@contoso.com", + "isRefreshable": True, + "isEffectiveIdentityRequired": False, + "isEffectiveIdentityRolesRequired": False, + }, + { + "id": "4869cc38-646c-45b6-89fd-62615beb9853", # excluded HTTP 401 + "name": "Strategy", + "configuredBy": "mark@contoso.com", + "isRefreshable": True, + "isEffectiveIdentityRequired": False, + "isEffectiveIdentityRolesRequired": False, + }, + { + "id": "68a2565d-4959-4421-b91c-bafe792796e1", + "name": "Strategy", + "configuredBy": "amelia@contoso.com", # exclude_creators + "isRefreshable": True, + "isEffectiveIdentityRequired": False, + "isEffectiveIdentityRolesRequired": False, + }, + { + "id": "8e4413eb-ba48-4a29-a0c2-fc80272d858e", + "name": "Test", + "configuredBy": None, # exclude_creators + "isRefreshable": True, + "isEffectiveIdentityRequired": False, + "isEffectiveIdentityRolesRequired": False, + }, + { + "id": "4de28a6f-f7d4-4186-a529-bf6c65e67b31", + "name": "Fees", + "configuredBy": "evelyn@contoso.com", + "isRefreshable": False, # excluded + "isEffectiveIdentityRequired": False, + "isEffectiveIdentityRolesRequired": False, + }, + { + "id": "c3c7591a-35bf-4f25-b03c-9c5ed6cdae14", + "name": "Fraud", + "configuredBy": "evelyn@contoso.com", + "isRefreshable": True, + "isEffectiveIdentityRequired": True, # excluded, only in unit test! + "isEffectiveIdentityRolesRequired": False, + }, + ] + } + get_invoicing_refresh_history = { + "value": [ + { + "id": 2, + "refreshType": "ViaEnhancedApi", + "status": "Completed", + "startTime": "2024-03-31T00:00:00Z", # summer time: change from 1 + "endTime": "2024-03-31T02:00:00Z", # hour to 2 hours difference + "serviceExceptionJson": None, + "requestId": "aec28227-f7af-4c2d-a4e6-fcb01cd570ec", + "refreshAttempts": None, + }, + { + "id": 1, + "refreshType": "OnDemand", + "status": "Completed", + "startTime": "2024-02-26T10:00:00Z", + "endTime": "2024-02-26T10:05:00Z", # winter time: 1 hour + "serviceExceptionJson": None, # difference from UTC + "requestId": "74d25c0b-0473-4dd9-96ff-3ca737b072a7", + "refreshAttempts": None, + }, + ] + } + get_reimbursement_refresh_history = { + "value": [ + { + "id": 2, + "refreshType": "ViaEnhancedApi", + "status": "Completed", + "startTime": "2024-02-24T10:00:00Z", + "endTime": "2024-02-24T10:09:00Z", + "serviceExceptionJson": None, + "requestId": "8d315d90-6105-450b-af2b-8654b962db98", + "refreshAttempts": None, + }, + { + "id": 1, + "refreshType": "ViaApi", + "status": "Completed", + "startTime": "2024-02-23T10:00:00Z", + "endTime": "2024-02-23T10:10:00Z", + "serviceExceptionJson": None, + "requestId": "11abf327-6e6b-4cdf-a9a0-f443d83680ec", + "refreshAttempts": None, + }, + ] + } + + def requests_get(url, headers): + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = None + if headers == "api_header": + if url.endswith("groups"): + mock_response.json.return_value = get_workspaces + elif ( + url.endswith("datasets") + and "614850c2-3a5c-4d2d-bcaa-d3f20f32a2e0" in url + ): + mock_response.json.return_value = get_finance_datasets + elif ( + url.endswith("refreshes") + and "b1f0a07e-e348-402c-a2b2-11f3e31181ce" in url + ): + mock_response.json.return_value = get_invoicing_refresh_history + elif ( + url.endswith("refreshes") + and "2e848e9a-47a3-4b0e-a22a-af35507ec8c4" in url + ): + mock_response.json.return_value = get_reimbursement_refresh_history + elif ( + url.endswith("refreshes") + and "4869cc38-646c-45b6-89fd-62615beb9853" in url + ): + mock_response.status_code = 401 + if mock_response.json.return_value or mock_response.status_code != 200: + return mock_response + raise ValueError("Unknown URL! " + url) + + mock_get.side_effect = requests_get + + sut = PowerBi( + PowerBiClient(), + local_timezone_name="Europe/Copenhagen", + exclude_creators=["Amelia@contoso.com", None], + ) + sut.api_header = "api_header" + sut.powerbi_url = "test/" + sut._get_access_token = lambda: True + expected = pd.DataFrame( + { + "WorkspaceName": ["Finance", "Finance", "Finance", "Finance"], + "DatasetName": [ + "Invoicing", + "Invoicing", + "Reimbursement", + "Reimbursement", + ], + "RefreshType": [ + "ViaEnhancedApi", + "OnDemand", + "ViaEnhancedApi", + "ViaApi", + ], + "Status": ["Completed", "Completed", "Completed", "Completed"], + "Seconds": [7200, 300, 540, 600], + "StartTimeLocal": [ + datetime(2024, 3, 31, 1, 0), + datetime(2024, 2, 26, 11, 0), + datetime(2024, 2, 24, 11, 0), + datetime(2024, 2, 23, 11, 0), + ], + "EndTimeLocal": [ + datetime(2024, 3, 31, 4, 0), + datetime(2024, 2, 26, 11, 5), + datetime(2024, 2, 24, 11, 9), + datetime(2024, 2, 23, 11, 10), + ], + "Error": [None, None, None, None], + "RequestId": [ + "aec28227-f7af-4c2d-a4e6-fcb01cd570ec", + "74d25c0b-0473-4dd9-96ff-3ca737b072a7", + "8d315d90-6105-450b-af2b-8654b962db98", + "11abf327-6e6b-4cdf-a9a0-f443d83680ec", + ], + "RefreshId": [2, 1, 2, 1], + "RefreshAttempts": None, + "WorkspaceId": [ + "614850c2-3a5c-4d2d-bcaa-d3f20f32a2e0", + "614850c2-3a5c-4d2d-bcaa-d3f20f32a2e0", + "614850c2-3a5c-4d2d-bcaa-d3f20f32a2e0", + "614850c2-3a5c-4d2d-bcaa-d3f20f32a2e0", + ], + "DatasetId": [ + "b1f0a07e-e348-402c-a2b2-11f3e31181ce", + "b1f0a07e-e348-402c-a2b2-11f3e31181ce", + "2e848e9a-47a3-4b0e-a22a-af35507ec8c4", + "2e848e9a-47a3-4b0e-a22a-af35507ec8c4", + ], + } + ) + expected["RefreshId"] = expected["RefreshId"].astype("Int64") + expected["Seconds"] = expected["Seconds"].astype("Int64") + + # Act + result = sut._combine_dataframes( + sut._get_refresh_history, + skip_read_only=True, + skip_not_refreshable=True, + skip_effective_identity=True, + ) + + # Assert + self.assertIsNotNone(result) + assert_frame_equal(expected, result.get_pandas_df()) + + @patch("requests.get") + def test_get_last_refresh_success(self, mock_get): + # Arrange + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "value": [ + { + "requestId": "5", + "id": "4", + "refreshType": "ViaApi", + "startTime": "2024-02-27T17:00:00Z", + "endTime": "2024-02-27T17:05:00Z", + "status": "Unknown", # skip because refresh is in progress + "serviceExceptionJson": None, + }, + { + "requestId": "4", + "id": "4", + "refreshType": "ViaApi", + "startTime": "2024-02-26T10:00:00Z", + "endTime": "2024-02-26T10:05:00Z", + "status": "Completed", + "serviceExceptionJson": None, + }, + { + "requestId": "3", + "id": "3", + "refreshType": "OnDemand", + "startTime": "2024-02-25T10:00:00Z", + "endTime": "2024-02-25T10:05:00Z", + "status": "Completed", + "serviceExceptionJson": None, + }, + { + "requestId": "2", + "id": "2", + "refreshType": "ViaEnhancedApi", + "startTime": "2024-02-24T10:00:00Z", + "endTime": "2024-02-24T10:09:00Z", + "status": "Completed", + "serviceExceptionJson": None, + }, + { + "requestId": "1", + "id": "1", + "refreshType": "ViaApi", + "startTime": "2024-02-23T10:00:00Z", + "endTime": "2024-02-23T10:10:00Z", + "status": "Completed", + "serviceExceptionJson": None, + }, + ] + } + mock_get.return_value = mock_response + + sut = PowerBi(PowerBiClient(), workspace_id="test", dataset_id="test") + sut.powerbi_url = "test/" + sut._connect = lambda: True # Act - with self.assertRaises(SpetlrException) as context: - sut._get_workspace() + result = sut._get_last_refresh(finished_only=True) # Assert - self.assertIn("Failed to fetch workspaces", str(context.exception)) + self.assertTrue(result) + self.assertEqual("Completed", sut.last_status) + self.assertIsNone(sut.last_exception) + self.assertEqual("2024-02-26 10:05:00+00:00", str(sut.last_refresh_utc)) + # average of ViaApi only + self.assertEqual(int(7.5 * 60), sut.last_duration_in_seconds) @patch("requests.get") - def test_get_dataset_success(self, mock_get): + @patch("requests.post") + def test_get_last_refresh_with_tables_success(self, mock_post, mock_get): # Arrange - mock_response = Mock() - mock_response.status_code = 200 - mock_response.json.return_value = { + mock_get_response = Mock() + mock_get_response.status_code = 200 + mock_get_response.json.return_value = { "value": [ - {"id": "b1f0a07e-e348-402c-a2b2-11f3e31181ce", "name": "Invoicing"} + { + "requestId": "2", + "id": "2", + "refreshType": "ViaApi", + "startTime": "2024-02-26T10:00:00Z", + "endTime": "2024-02-26T10:05:00Z", + "status": "Completed", + "serviceExceptionJson": None, + }, + { + "requestId": "1", + "id": "1", + "refreshType": "OnDemand", + "startTime": "2024-02-25T10:00:00Z", + "endTime": "2024-02-25T10:05:00Z", + "status": "Completed", + "serviceExceptionJson": None, + }, ] } - mock_get.return_value = mock_response + mock_post_response = Mock() + mock_post_response.status_code = 200 + mock_post_response.json.return_value = { + "results": [ + { + "tables": [ + { + "rows": [ + { + "[ID]": 1, + "[TableID]": 1001, + "[Name]": "Invoices", + "[RefreshedTime]": "2024-02-26T10:50:00+01:00", + "[ModifiedTime]": "2024-02-26T10:55:00+01:00", + "[Description]": "contains invoices", + "[ErrorMessage]": None, + }, + { + "[ID]": 2, + "[TableID]": 1002, + "[Name]": "Customers", + "[RefreshedTime]": "2024-02-26T15:50:00+01:00", + "[ModifiedTime]": "2024-02-26T15:55:00+01:00", + "[Description]": "contains customers", + "[ErrorMessage]": None, + }, + ] + } + ] + } + ] + } + mock_get.return_value = mock_get_response + mock_post.return_value = mock_post_response sut = PowerBi( PowerBiClient(), - workspace_name="Finance", - dataset_name="Invoicing", + workspace_id="test", + dataset_id="test", + table_names=["Invoices", "Customers"], ) - sut.powerbi_url = "test" + sut.powerbi_url = "test/" + sut._connect = lambda: True # Act - result = sut._get_dataset() + result = sut._get_last_refresh() # Assert self.assertTrue(result) - self.assertEqual("b1f0a07e-e348-402c-a2b2-11f3e31181ce", sut.dataset_id) + self.assertEqual("Completed", sut.last_status) + self.assertEqual("Invoices", sut.table_name) + self.assertIsNone(sut.last_exception) + self.assertEqual("2024-02-26 09:50:00+00:00", str(sut.last_refresh_utc)) + self.assertEqual(0, sut.last_duration_in_seconds) # tables were specified! @patch("requests.get") - def test_get_dataset_failure(self, mock_get): + @patch("requests.post") + def test_get_last_refresh_with_failed_tables_success(self, mock_post, mock_get): # Arrange - mock_response = Mock() - mock_response.status_code = 404 - mock_response.text = "error" - mock_get.return_value = mock_response + mock_get_response = Mock() + mock_get_response.status_code = 200 + mock_get_response.json.return_value = { + "value": [ + { + "requestId": "2", + "id": "2", + "refreshType": "ViaApi", + "startTime": "2024-02-26T10:00:00Z", + "endTime": "2024-02-26T10:05:00Z", + "status": "Completed", + "serviceExceptionJson": None, + }, + { + "requestId": "1", + "id": "1", + "refreshType": "OnDemand", + "startTime": "2024-02-25T10:00:00Z", + "endTime": "2024-02-25T10:05:00Z", + "status": "Completed", + "serviceExceptionJson": None, + }, + ] + } + mock_post_response = Mock() + mock_post_response.status_code = 200 + mock_post_response.json.return_value = { + "results": [ + { + "tables": [ + { + "rows": [ + { + "[ID]": 1, + "[TableID]": 1001, + "[Name]": "Invoices", + "[RefreshedTime]": "2024-02-26T10:50:00+01:00", + "[ModifiedTime]": "2024-02-26T10:55:00+01:00", + "[Description]": "contains invoices", + "[ErrorMessage]": None, + }, + { + "[ID]": 2, + "[TableID]": 1002, + "[Name]": "Customers", + "[RefreshedTime]": "2024-02-26T15:50:00+01:00", + "[ModifiedTime]": "2024-02-26T15:55:00+01:00", + "[Description]": "contains customers", + "[ErrorMessage]": "Refresh error", + }, + ] + } + ] + } + ] + } + mock_get.return_value = mock_get_response + mock_post.return_value = mock_post_response sut = PowerBi( PowerBiClient(), - workspace_name="Finance", - dataset_name="Invoicing", + workspace_id="test", + dataset_id="test", + table_names=["Invoices", "Customers"], ) - sut.powerbi_url = "test" + sut.powerbi_url = "test/" + sut._connect = lambda: True # Act - with self.assertRaises(SpetlrException) as context: - sut._get_dataset() + result = sut._get_last_refresh() # Assert - self.assertIn("Failed to fetch datasets", str(context.exception)) + self.assertTrue(result) + self.assertEqual("Failed", sut.last_status) + self.assertEqual("Customers", sut.table_name) + self.assertEqual("Refresh error", sut.last_exception) + self.assertIsNone(sut.last_refresh_utc) @patch("requests.get") - def test_get_last_refresh_success(self, mock_get): + @patch("requests.post") + def test_get_last_refresh_with_unauthorized_tables_success( + self, mock_post, mock_get + ): # Arrange - mock_response = Mock() - mock_response.status_code = 200 - mock_response.json.return_value = { + mock_get_response = Mock() + mock_get_response.status_code = 200 + mock_get_response.json.return_value = { "value": [ { - "requestId": "1", - "id": "1", + "requestId": "2", + "id": "2", "refreshType": "ViaApi", "startTime": "2024-02-26T10:00:00Z", "endTime": "2024-02-26T10:05:00Z", "status": "Completed", "serviceExceptionJson": None, - } + }, + { + "requestId": "1", + "id": "1", + "refreshType": "OnDemand", + "startTime": "2024-02-25T10:00:00Z", + "endTime": "2024-02-25T10:05:00Z", + "status": "Completed", + "serviceExceptionJson": None, + }, ] } - mock_get.return_value = mock_response + mock_post_response = Mock() + mock_post_response.status_code = 401 + mock_get.return_value = mock_get_response + mock_post.return_value = mock_post_response - sut = PowerBi(PowerBiClient(), workspace_id="test", dataset_id="test") - sut.powerbi_url = "test" + sut = PowerBi( + PowerBiClient(), + workspace_id="test", + dataset_id="test", + table_names=["Invoices", "Customers"], + ) + sut.powerbi_url = "test/" sut._connect = lambda: True # Act @@ -128,34 +988,53 @@ def test_get_last_refresh_success(self, mock_get): # Assert self.assertTrue(result) self.assertEqual("Completed", sut.last_status) + self.assertIsNone(sut.table_name) self.assertIsNone(sut.last_exception) self.assertEqual("2024-02-26 10:05:00+00:00", str(sut.last_refresh_utc)) - self.assertEqual(5 * 60, sut.last_duration_in_seconds) + self.assertEqual(0, sut.last_duration_in_seconds) # tables were specified! @patch("requests.get") - def test_get_last_refresh_failure(self, mock_get): + def test_get_last_refresh_empty(self, mock_get): # Arrange mock_response = Mock() - mock_response.status_code = 404 # dataset or workspace not found + mock_response.status_code = 200 + mock_response.json.return_value = {"value": []} mock_get.return_value = mock_response sut = PowerBi(PowerBiClient(), workspace_id="test", dataset_id="test") - sut.powerbi_url = "test" + sut.powerbi_url = "test/" sut.last_status = "test" sut.last_duration_in_seconds = 5 sut._connect = lambda: True + # Act + result = sut._get_last_refresh() + + # Assert + self.assertTrue(result) + self.assertIsNone(sut.last_exception) + self.assertIsNone(sut.last_status) # must be cleared! + self.assertEqual(5, sut.last_duration_in_seconds) # must be kept unchanged! + + @patch("requests.get") + def test_get_last_refresh_failure(self, mock_get): + # Arrange + mock_response = Mock() + mock_response.status_code = 404 # dataset or workspace not found + mock_get.return_value = mock_response + + sut = PowerBi(PowerBiClient(), workspace_id="test", dataset_id="test") + sut.powerbi_url = "test/" + sut._connect = lambda: True + # Act with self.assertRaises(SpetlrException) as context: sut._get_last_refresh() # Assert self.assertIn( - "The specified dataset or workspace cannot be found", - str(context.exception), + "The specified dataset or workspace cannot be found", str(context.exception) ) - self.assertIsNone(sut.last_status) # must be cleared! - self.assertEqual(5, sut.last_duration_in_seconds) # must be kept unchanged! def test_verify_last_refresh_success(self): # Arrange @@ -167,7 +1046,9 @@ def test_verify_last_refresh_success(self): local_timezone_name="Europe/Copenhagen", ) sut.last_status = "Completed" + sut.table_name = "TestTable" sut.last_refresh_utc = datetime.now(utc) - timedelta(minutes=1) + sut.last_refresh_str = str(sut.last_refresh_utc) # Act result = sut._verify_last_refresh() @@ -185,6 +1066,8 @@ def test_verify_last_refresh_failure(self): local_timezone_name="Europe/Copenhagen", ) sut.last_status = "Completed" + sut.table_name = "TestTable" + sut.last_refresh_str = "2024-05-01" sut.last_refresh_utc = datetime.now(utc) - timedelta(minutes=10) # Act @@ -192,9 +1075,10 @@ def test_verify_last_refresh_failure(self): sut._verify_last_refresh() # Assert - self.assertIn("Last refresh finished more than", str(context.exception)) + self.assertIn("finished more than", str(context.exception)) + self.assertIn("TestTable", str(context.exception)) - def test_get_table_names_json_default(self): + def test_get_refresh_argument_json_default(self): # Arrange sut = PowerBi( PowerBiClient(), @@ -203,35 +1087,94 @@ def test_get_table_names_json_default(self): ) # Act - result = sut._get_table_names_json() + result = sut._get_refresh_argument_json(True) # Assert self.assertIsNone(result) - def test_get_table_names_json_explicit(self): + def test_get_refresh_argument_json_combination_failure(self): + # Act + with self.assertRaises(ValueError) as context: + PowerBi( + PowerBiClient(), + workspace_id="614850c2-3a5c-4d2d-bcaa-d3f20f32a2e0", + dataset_id="b1f0a07e-e348-402c-a2b2-11f3e31181ce", + table_names=["Invoices", "Customers"], + mail_on_failure=True, + ) + + # Assert + self.assertIn("cannot be combined", str(context.exception)) + + def test_get_refresh_argument_json_without_tables_and_without_wait(self): + # Arrange + sut = PowerBi( + PowerBiClient(), + workspace_id="614850c2-3a5c-4d2d-bcaa-d3f20f32a2e0", + dataset_id="b1f0a07e-e348-402c-a2b2-11f3e31181ce", + number_of_retries=2, # should be ignored without tables + mail_on_failure=True, + ) + expected_result = "MailOnFailure" + + # Act + result = sut._get_refresh_argument_json(with_wait=False) + + # Assert + self.assertIsNotNone(result) + self.assertFalse("objects" in result) + self.assertFalse("retryCount" in result) + self.assertTrue("notifyOption" in result) + self.assertEqual(expected_result, result["notifyOption"]) + + def test_get_refresh_argument_json_with_tables_and_without_wait(self): + # Arrange + sut = PowerBi( + PowerBiClient(), + workspace_id="614850c2-3a5c-4d2d-bcaa-d3f20f32a2e0", + dataset_id="b1f0a07e-e348-402c-a2b2-11f3e31181ce", + table_names=["Invoices", "Customers"], + number_of_retries=2, # should not be ignored + ) + expected_result1 = 2 + expected_result2 = [{"table": "Invoices"}, {"table": "Customers"}] + + # Act + result = sut._get_refresh_argument_json(with_wait=False) + + # Assert + self.assertIsNotNone(result) + self.assertTrue("objects" in result) + self.assertTrue("retryCount" in result) + self.assertEqual(expected_result1, result["retryCount"]) + self.assertEqual(expected_result2, result["objects"]) + + def test_get_refresh_argument_json_with_tables_and_with_wait(self): # Arrange sut = PowerBi( PowerBiClient(), workspace_id="614850c2-3a5c-4d2d-bcaa-d3f20f32a2e0", dataset_id="b1f0a07e-e348-402c-a2b2-11f3e31181ce", table_names=["Invoices", "Customers"], + number_of_retries=2, # should be ignored if with_wait=True ) expected_result = [{"table": "Invoices"}, {"table": "Customers"}] # Act - result = sut._get_table_names_json() + result = sut._get_refresh_argument_json(with_wait=True) # Assert self.assertIsNotNone(result) self.assertTrue("objects" in result) + self.assertFalse("retryCount" in result) self.assertEqual(expected_result, result["objects"]) @patch("requests.post") - def test_trigger_new_refresh_success(self, mock_get): + def test_trigger_new_refresh_success(self, mock_post): # Arrange mock_response = Mock() mock_response.status_code = 202 - mock_get.return_value = mock_response + mock_post.return_value = mock_response sut = PowerBi( PowerBiClient(), @@ -239,21 +1182,21 @@ def test_trigger_new_refresh_success(self, mock_get): dataset_id="b1f0a07e-e348-402c-a2b2-11f3e31181ce", ) sut.last_status = "Completed" - sut.powerbi_url = "test" + sut.powerbi_url = "test/" # Act - result = sut._trigger_new_refresh() + result = sut._trigger_new_refresh(True) # Assert self.assertTrue(result) @patch("requests.post") - def test_trigger_new_refresh_failure(self, mock_get): + def test_trigger_new_refresh_failure(self, mock_post): # Arrange mock_response = Mock() - mock_response.status_code = 404 + mock_response.status_code = 401 mock_response.text = "error" - mock_get.return_value = mock_response + mock_post.return_value = mock_response sut = PowerBi( PowerBiClient(), @@ -261,11 +1204,11 @@ def test_trigger_new_refresh_failure(self, mock_get): dataset_id="b1f0a07e-e348-402c-a2b2-11f3e31181ce", ) sut.last_status = "Completed" - sut.powerbi_url = "test" + sut.powerbi_url = "test/" # Act with self.assertRaises(SpetlrException) as context: - sut._trigger_new_refresh() + sut._trigger_new_refresh(True) # Assert self.assertIn("Failed to trigger a refresh", str(context.exception)) @@ -308,99 +1251,3 @@ def test_get_seconds_to_wait_exceeding_timeout(self): # Assert self.assertEqual(expected_result, result) - - @patch("requests.get") - def test_get_refresh_history_success(self, mock_get): - # Arrange - mock_response = Mock() - mock_response.status_code = 200 - mock_response.json.return_value = { - "value": [ - { - "id": "1", - "refreshType": "OnDemand", - "status": "Completed", - "startTime": "2024-02-26T10:00:00Z", - "endTime": "2024-02-26T10:05:00Z", # winter time: 1 hour - "serviceExceptionJson": None, # difference from UTC - "requestId": "74d25c0b-0473-4dd9-96ff-3ca737b072a7", - "refreshAttempts": None, - }, - { - "id": "2", - "refreshType": "ViaEnhancedApi", - "status": "Completed", - "startTime": "2024-03-31T00:00:00Z", # summer time: change from 1 - "endTime": "2024-03-31T02:00:00Z", # hour to 2 hours difference - "serviceExceptionJson": None, - "requestId": "aec28227-f7af-4c2d-a4e6-fcb01cd570ec", - "refreshAttempts": None, - }, - ] - } - mock_get.return_value = mock_response - - sut = PowerBi( - PowerBiClient(), - workspace_id="test", - dataset_id="test", - local_timezone_name="Europe/Copenhagen", - ) - sut.powerbi_url = "test" - sut._connect = lambda: True - expected = pd.DataFrame( - { - "Id": ["1", "2"], - "RefreshType": ["OnDemand", "ViaEnhancedApi"], - "Status": ["Completed", "Completed"], - "Seconds": [ - 300, - 7200, - ], # despite the daylight saving change it - # correctly shows 7200 seconds ! - "StartTimeLocal": [ - datetime(2024, 2, 26, 11, 0), - datetime(2024, 3, 31, 1, 0), - ], - "EndTimeLocal": [ - datetime(2024, 2, 26, 11, 5), - datetime(2024, 3, 31, 4, 0), - ], - "Error": [None, None], - "RequestId": [ - "74d25c0b-0473-4dd9-96ff-3ca737b072a7", - "aec28227-f7af-4c2d-a4e6-fcb01cd570ec", - ], - "RefreshAttempts": None, - } - ) - - # Act - result = sut._get_refresh_history() - - # Assert - self.assertIsNotNone(result) - assert_frame_equal(expected, result) - - @patch("requests.get") - def test_get_refresh_history_failure(self, mock_get): - # Arrange - mock_response = Mock() - mock_response.status_code = 404 # dataset or workspace not found - mock_get.return_value = mock_response - - sut = PowerBi(PowerBiClient(), workspace_id="test", dataset_id="test") - sut.powerbi_url = "test" - sut.last_status = "test" - sut._connect = lambda: True - - # Act - with self.assertRaises(SpetlrException) as context: - sut._get_refresh_history() - - # Assert - self.assertIn( - "The specified dataset or workspace cannot be found", - str(context.exception), - ) - self.assertIsNone(sut.last_status) # must be cleared! diff --git a/tests/local/power_bi/test_spark_pandas_dataframe.py b/tests/local/power_bi/test_spark_pandas_dataframe.py new file mode 100644 index 00000000..f5a15057 --- /dev/null +++ b/tests/local/power_bi/test_spark_pandas_dataframe.py @@ -0,0 +1,142 @@ +import unittest +from datetime import datetime + +import pandas as pd +from pandas.testing import assert_frame_equal + +from spetlr.power_bi.SparkPandasDataFrame import SparkPandasDataFrame + + +class TestSparkPandasDataFrame(unittest.TestCase): + def test_get_workspace_success(self): + # Arrange + json = [ + { + "id": 3, + "refreshType": "ViaEnhancedApi", + "status": "Completed", + "startTime": "2024-03-31T00:00:00Z", # summer time: change from 1 + "endTime": "2024-03-31T02:00:00Z", # hour to 2 hours difference + "serviceExceptionJson": None, + "requestId": "aec28227-f7af-4c2d-a4e6-fcb01cd570ec", + "refreshAttempts": None, + }, + { + "id": 2, + "refreshType": "ViaApi", + "status": "Completed", + "startTime": "2024-02-26T10:00:00Z", + "endTime": "2024-02-26T10:05:00Z", # winter time: 1 hour + "serviceExceptionJson": None, # difference from UTC + "requestId": "74d25c0b-0473-4dd9-96ff-3ca737b072a7", + "refreshAttempts": None, + }, + { + "id": 1, + "refreshType": "OnDemand", + "status": "Completed", + "startTime": "2024-01-16T10:00:00Z", + "endTime": None, # will calculate None seconds + "serviceExceptionJson": None, + "requestId": "191cc73a-fc75-4eff-9ffe-67aa5290d7f4", + "refreshAttempts": None, + }, + ] + schema = [ + ("id", "Id", "long"), + ("refreshType", "RefreshType", "string"), + ("status", "Status", "string"), + ( + lambda d: (d["endTime"] - d["startTime"]) / pd.Timedelta(seconds=1), + "Seconds", + "long", + ), + ("startTime", "StartTime", "timestamp"), + ("endTime", "EndTime", "timestamp"), + ("serviceExceptionJson", "Error", "string"), + ("requestId", "RequestId", "string"), + ("refreshAttempts", "RefreshAttempts", "string"), + ] + expected = pd.DataFrame( + { + "Id": [1, 2, 3], + "RefreshType": ["OnDemand", "ViaApi", "ViaEnhancedApi"], + "Status": ["Completed", "Completed", "Completed"], + "Seconds": [ + None, + 300, + 7200, + ], # despite the daylight saving change it + # correctly shows 7200 seconds ! + "StartTimeLocal": [ + datetime(2024, 1, 16, 11, 0), + datetime(2024, 2, 26, 11, 0), + datetime(2024, 3, 31, 1, 0), + ], + "EndTimeLocal": [ + None, + datetime(2024, 2, 26, 11, 5), + datetime(2024, 3, 31, 4, 0), + ], + "Error": [None, None, None], + "RequestId": [ + "191cc73a-fc75-4eff-9ffe-67aa5290d7f4", + "74d25c0b-0473-4dd9-96ff-3ca737b072a7", + "aec28227-f7af-4c2d-a4e6-fcb01cd570ec", + ], + "RefreshAttempts": None, + } + ) + expected["Id"] = expected["Id"].astype("Int64") + expected["Seconds"] = expected["Seconds"].astype("Int64") + + # Act + sut = SparkPandasDataFrame( + json, + schema, + indexing_columns="Id", + sorting_columns=["StartTimeLocal", "Id"], + local_timezone_name="Europe/Copenhagen", + ) + + pd.options.display.max_columns = None + pd.options.display.max_rows = None + + # Assert + assert_frame_equal( + expected.reset_index(drop=True), sut.get_pandas_df().reset_index(drop=True) + ) + + def test_parse_time_success(self): + # Arrange + data = [ + (None, None), + (float("NaN"), None), + ("2024-05-02", datetime(2024, 5, 2, 0, 0, 0, 0, tzinfo=None)), + ("2024-05-03T13:35", datetime(2024, 5, 3, 13, 35, 0, 0, tzinfo=None)), + ( + "2024-05-04T15:35:28+00:00", + datetime(2024, 5, 4, 15, 35, 28, 0, tzinfo=None), + ), + ( + "2024-05-05T17:15:28.232223", + datetime(2024, 5, 5, 17, 15, 28, 0, tzinfo=None), + ), + ( + "2024-05-05T21:33:04.232223+02:00", + datetime(2024, 5, 5, 19, 33, 4, 0, tzinfo=None), + ), + ( + "2024-05-07T08:46:17.230735322332Z", + datetime(2024, 5, 7, 8, 46, 17, 0, tzinfo=None), + ), + ] + + # Act + for value, result in data: + with self.subTest(value=value, result=result): + # Act + expected_result = SparkPandasDataFrame._parse_time(value) + + # Assert + self.assertEqual(expected_result, result)