Merge pull request #16 from TogetherCrew/feat/14-mediawiki-etl

feat: added retry policy for mediwiki activities!

Author: amindadgar

hivemind_etl/mediawiki/workflows.py

@@ -2,6 +2,7 @@
 from datetime import timedelta
 
 from temporalio import workflow
+from temporalio.common import RetryPolicy
 
 with workflow.unsafe.imports_passed_through():
     from hivemind_etl.mediawiki.activities import (
@@ -31,6 +32,10 @@ async def run(self, platform_id: str | None = None) -> None:
                 get_hivemind_mediawiki_platforms,
                 platform_id,
                 start_to_close_timeout=timedelta(minutes=1),
+                retry_policy=RetryPolicy(
+                    initial_interval=timedelta(minutes=1),
+                    maximum_attempts=3,
+                ),
             )
 
             for platform in platforms:
@@ -45,13 +50,21 @@ async def run(self, platform_id: str | None = None) -> None:
                         extract_mediawiki,
                         mediawiki_platform,
                         start_to_close_timeout=timedelta(days=5),
+                        retry_policy=RetryPolicy(
+                            initial_interval=timedelta(minutes=1),
+                            maximum_attempts=3,
+                        ),
                     )
 
                     # Transform the extracted data
                     documents = await workflow.execute_activity(
                         transform_mediawiki_data,
                         platform["community_id"],
                         start_to_close_timeout=timedelta(minutes=30),
+                        retry_policy=RetryPolicy(
+                            initial_interval=timedelta(minutes=1),
+                            maximum_attempts=3,
+                        ),
                     )
 
                     # Load the transformed data
@@ -60,6 +73,10 @@ async def run(self, platform_id: str | None = None) -> None:
                         documents,
                         platform["community_id"],
                         start_to_close_timeout=timedelta(minutes=30),
+                        retry_policy=RetryPolicy(
+                            initial_interval=timedelta(minutes=1),
+                            maximum_attempts=3,
+                        ),
                     )
 
                     logging.info(