fix(source-github): fix state handling for stream WorkflowRuns (#44592)

airbytehq · Sep 4, 2024 · 84f3b03 · 84f3b03
1 parent 1f2fd6e
commit 84f3b03
Show file tree

Hide file tree

Showing 4 changed files with 65 additions and 60 deletions.
diff --git a/airbyte-integrations/connectors/source-github/metadata.yaml b/airbyte-integrations/connectors/source-github/metadata.yaml
@@ -10,7 +10,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e
-  dockerImageTag: 1.8.7
+  dockerImageTag: 1.8.8
   dockerRepository: airbyte/source-github
   documentationUrl: https://docs.airbyte.com/integrations/sources/github
   erdUrl: https://dbdocs.io/airbyteio/source-github?view=relationships

diff --git a/airbyte-integrations/connectors/source-github/pyproject.toml b/airbyte-integrations/connectors/source-github/pyproject.toml
@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "1.8.7"
+version = "1.8.8"
 name = "source-github"
 description = "Source implementation for GitHub."
 authors = [ "Airbyte <contact@airbyte.io>",]

diff --git a/airbyte-integrations/connectors/source-github/source_github/streams.py b/airbyte-integrations/connectors/source-github/source_github/streams.py
@@ -1459,6 +1459,8 @@ def read_records(
         # only to look behind on 30 days to find all records which were updated.
         start_point = self.get_starting_point(stream_state=stream_state, stream_slice=stream_slice)
         break_point = None
+        # the state is updated only in the end of the sync as records are sorted in reverse order
+        new_state = self.state
         if start_point:
             break_point = (pendulum.parse(start_point) - pendulum.duration(days=self.re_run_period)).to_iso8601_string()
         for record in super(SemiIncrementalMixin, self).read_records(
@@ -1468,8 +1470,10 @@ def read_records(
             created_at = record["created_at"]
             if not start_point or cursor_value > start_point:
                 yield record
+                new_state = self._get_updated_state(new_state, record)
             if break_point and created_at < break_point:
                 break
+        self.state = new_state
 
 
 class WorkflowJobs(SemiIncrementalMixin, GithubStream):

diff --git a/docs/integrations/sources/github.md b/docs/integrations/sources/github.md
@@ -222,64 +222,65 @@ Your token should have at least the `repo` scope. Depending on which streams you
 
 | Version | Date       | Pull Request                                                                                                      | Subject                                                                                                                                                             |
 |:--------|:-----------|:------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| 1.8.7 | 2024-08-31 | [45061](https://github.com/airbytehq/airbyte/pull/45061) | Update dependencies |
-| 1.8.6 | 2024-08-24 | [44703](https://github.com/airbytehq/airbyte/pull/44703) | Update dependencies |
-| 1.8.5 | 2024-08-17 | [44227](https://github.com/airbytehq/airbyte/pull/44227) | Update dependencies |
-| 1.8.4 | 2024-08-12 | [43749](https://github.com/airbytehq/airbyte/pull/43749) | Update dependencies |
-| 1.8.3 | 2024-08-10 | [42671](https://github.com/airbytehq/airbyte/pull/42671) | Update dependencies |
-| 1.8.2 | 2024-08-20 | [42966](https://github.com/airbytehq/airbyte/pull/42966) | Bump cdk version and enable RFR for all non-incremental streams |
-| 1.8.1 | 2024-07-20 | [42342](https://github.com/airbytehq/airbyte/pull/42342) | Update dependencies |
-| 1.8.0 | 2024-07-16 | [41677](https://github.com/airbytehq/airbyte/pull/41677) | Update to 3.4.0 CDK |
-| 1.7.13 | 2024-07-13 | [41746](https://github.com/airbytehq/airbyte/pull/41746) | Update dependencies |
-| 1.7.12 | 2024-07-10 | [41354](https://github.com/airbytehq/airbyte/pull/41354) | Update dependencies |
-| 1.7.11 | 2024-07-09 | [41221](https://github.com/airbytehq/airbyte/pull/41221) | Update dependencies |
-| 1.7.10 | 2024-07-06 | [41000](https://github.com/airbytehq/airbyte/pull/41000) | Update dependencies |
-| 1.7.9 | 2024-06-25 | [40289](https://github.com/airbytehq/airbyte/pull/40289) | Update dependencies |
-| 1.7.8 | 2024-06-22 | [40128](https://github.com/airbytehq/airbyte/pull/40128) | Update dependencies |
-| 1.7.7 | 2024-06-17 | [39513](https://github.com/airbytehq/airbyte/pull/39513) | Update deprecated state handling method |
-| 1.7.6 | 2024-06-04 | [39078](https://github.com/airbytehq/airbyte/pull/39078) | [autopull] Upgrade base image to v1.2.1 |
-| 1.7.5 | 2024-05-29 | [38341](https://github.com/airbytehq/airbyte/pull/38341) | Add `max_waiting_time` to configuration |
-| 1.7.4 | 2024-05-21 | [38341](https://github.com/airbytehq/airbyte/pull/38341) | Update CDK authenticator package |
-| 1.7.3 | 2024-05-20 | [38299](https://github.com/airbytehq/airbyte/pull/38299) | Fixed spec typo |
-| 1.7.2 | 2024-04-19 | [36636](https://github.com/airbytehq/airbyte/pull/36636) | Updating to 0.80.0 CDK |
-| 1.7.1 | 2024-04-12 | [36636](https://github.com/airbytehq/airbyte/pull/36636) | schema descriptions |
-| 1.7.0 | 2024-03-19 | [36267](https://github.com/airbytehq/airbyte/pull/36267) | Pin airbyte-cdk version to `^0` |
-| 1.6.5 | 2024-03-12 | [35986](https://github.com/airbytehq/airbyte/pull/35986) | Handle rate limit exception as config error |
-| 1.6.4 | 2024-03-08 | [35915](https://github.com/airbytehq/airbyte/pull/35915) | Fix per stream error handler; Make use the latest CDK version |
-| 1.6.3 | 2024-02-15 | [35271](https://github.com/airbytehq/airbyte/pull/35271) | Update branches schema |
-| 1.6.2 | 2024-02-12 | [34933](https://github.com/airbytehq/airbyte/pull/34933) | Update Airbyte CDK for integration tests |
-| 1.6.1 | 2024-02-09 | [35087](https://github.com/airbytehq/airbyte/pull/35087) | Manage dependencies with Poetry. |
-| 1.6.0 | 2024-02-02 | [34700](https://github.com/airbytehq/airbyte/pull/34700) | Continue Sync on Stream failure |
-| 1.5.7 | 2024-01-29 | [34598](https://github.com/airbytehq/airbyte/pull/34598) | Fix MultipleToken sleep time |
-| 1.5.6 | 2024-01-26 | [34503](https://github.com/airbytehq/airbyte/pull/34503) | Fix MultipleToken rotation logic |
-| 1.5.5 | 2023-12-26 | [33783](https://github.com/airbytehq/airbyte/pull/33783) | Fix retry for 504 error in GraphQL based streams |
-| 1.5.4 | 2023-11-20 | [32679](https://github.com/airbytehq/airbyte/pull/32679) | Return AirbyteMessage if max retry exeeded for 202 status code |
-| 1.5.3 | 2023-10-23 | [31702](https://github.com/airbytehq/airbyte/pull/31702) | Base image migration: remove Dockerfile and use the python-connector-base image |
-| 1.5.2 | 2023-10-13 | [31386](https://github.com/airbytehq/airbyte/pull/31386) | Handle `ContributorActivity` continuous `ACCEPTED` response |
-| 1.5.1 | 2023-10-12 | [31307](https://github.com/airbytehq/airbyte/pull/31307) | Increase backoff_time for stream `ContributorActivity` |
-| 1.5.0 | 2023-10-11 | [31300](https://github.com/airbytehq/airbyte/pull/31300) | Update Schemas: Add date-time format to fields |
-| 1.4.6 | 2023-10-04 | [31056](https://github.com/airbytehq/airbyte/pull/31056) | Migrate spec properties' `repository` and `branch` type to \<array\> |
-| 1.4.5 | 2023-10-02 | [31023](https://github.com/airbytehq/airbyte/pull/31023) | Increase backoff for stream `Contributor Activity` |
-| 1.4.4 | 2023-10-02 | [30971](https://github.com/airbytehq/airbyte/pull/30971) | Mark `start_date` as optional. |
-| 1.4.3 | 2023-10-02 | [30979](https://github.com/airbytehq/airbyte/pull/30979) | Fetch archived records in `Project Cards` |
-| 1.4.2 | 2023-09-30 | [30927](https://github.com/airbytehq/airbyte/pull/30927) | Provide actionable user error messages |
-| 1.4.1 | 2023-09-30 | [30839](https://github.com/airbytehq/airbyte/pull/30839) | Update CDK to Latest version |
-| 1.4.0 | 2023-09-29 | [30823](https://github.com/airbytehq/airbyte/pull/30823) | Add new stream `issue Timeline Events` |
-| 1.3.1 | 2023-09-28 | [30824](https://github.com/airbytehq/airbyte/pull/30824) | Handle empty response in stream `ContributorActivity` |
-| 1.3.0 | 2023-09-25 | [30731](https://github.com/airbytehq/airbyte/pull/30731) | Add new stream `ProjectsV2` |
-| 1.2.1 | 2023-09-22 | [30693](https://github.com/airbytehq/airbyte/pull/30693) | Handle 404 error in `TeamMemberShips` |
-| 1.2.0 | 2023-09-22 | [30647](https://github.com/airbytehq/airbyte/pull/30647) | Add support for self-hosted GitHub instances |
-| 1.1.1 | 2023-09-21 | [30654](https://github.com/airbytehq/airbyte/pull/30654) | Rewrite source connection error messages |
-| 1.1.0 | 2023-08-03 | [30615](https://github.com/airbytehq/airbyte/pull/30615) | Add new stream `Contributor Activity` |
-| 1.0.4 | 2023-08-03 | [29031](https://github.com/airbytehq/airbyte/pull/29031) | Reverted `advancedAuth` spec changes |
-| 1.0.3 | 2023-08-01 | [28910](https://github.com/airbytehq/airbyte/pull/28910) | Updated `advancedAuth` broken references |
-| 1.0.2 | 2023-07-11 | [28144](https://github.com/airbytehq/airbyte/pull/28144) | Add `archived_at` property to `Organizations` schema parameter |
-| 1.0.1 | 2023-05-22 | [25838](https://github.com/airbytehq/airbyte/pull/25838) | Deprecate "page size" input parameter |
-| 1.0.0 | 2023-05-19 | [25778](https://github.com/airbytehq/airbyte/pull/25778) | Improve repo(s) name validation on UI |
-| 0.5.0 | 2023-05-16 | [25793](https://github.com/airbytehq/airbyte/pull/25793) | Implement client-side throttling of requests |
-| 0.4.11 | 2023-05-12 | [26025](https://github.com/airbytehq/airbyte/pull/26025) | Added more transparent depiction of the personal access token expired |
-| 0.4.10 | 2023-05-15 | [26075](https://github.com/airbytehq/airbyte/pull/26075) | Add more specific error message description for no repos case. |
-| 0.4.9 | 2023-05-01 | [24523](https://github.com/airbytehq/airbyte/pull/24523) | Add undeclared columns to spec |
+| 1.8.8   | 2024-08-23 | [44592](https://github.com/airbytehq/airbyte/pull/44592) | Fix state handling for stream WorkflowRuns                                                                                                                          |
+| 1.8.7   | 2024-08-31 | [45061](https://github.com/airbytehq/airbyte/pull/45061) | Update dependencies |
+| 1.8.6   | 2024-08-24 | [44703](https://github.com/airbytehq/airbyte/pull/44703) | Update dependencies |
+| 1.8.5   | 2024-08-17 | [44227](https://github.com/airbytehq/airbyte/pull/44227) | Update dependencies |
+| 1.8.4   | 2024-08-12 | [43749](https://github.com/airbytehq/airbyte/pull/43749) | Update dependencies |
+| 1.8.3   | 2024-08-10 | [42671](https://github.com/airbytehq/airbyte/pull/42671) | Update dependencies |
+| 1.8.2   | 2024-08-20 | [42966](https://github.com/airbytehq/airbyte/pull/42966) | Bump cdk version and enable RFR for all non-incremental streams |
+| 1.8.1   | 2024-07-20 | [42342](https://github.com/airbytehq/airbyte/pull/42342) | Update dependencies |
+| 1.8.0   | 2024-07-16 | [41677](https://github.com/airbytehq/airbyte/pull/41677) | Update to 3.4.0 CDK |
+| 1.7.13  | 2024-07-13 | [41746](https://github.com/airbytehq/airbyte/pull/41746) | Update dependencies |
+| 1.7.12  | 2024-07-10 | [41354](https://github.com/airbytehq/airbyte/pull/41354) | Update dependencies |
+| 1.7.11  | 2024-07-09 | [41221](https://github.com/airbytehq/airbyte/pull/41221) | Update dependencies |
+| 1.7.10  | 2024-07-06 | [41000](https://github.com/airbytehq/airbyte/pull/41000) | Update dependencies |
+| 1.7.9   | 2024-06-25 | [40289](https://github.com/airbytehq/airbyte/pull/40289) | Update dependencies |
+| 1.7.8   | 2024-06-22 | [40128](https://github.com/airbytehq/airbyte/pull/40128) | Update dependencies |
+| 1.7.7   | 2024-06-17 | [39513](https://github.com/airbytehq/airbyte/pull/39513) | Update deprecated state handling method |
+| 1.7.6   | 2024-06-04 | [39078](https://github.com/airbytehq/airbyte/pull/39078) | [autopull] Upgrade base image to v1.2.1 |
+| 1.7.5   | 2024-05-29 | [38341](https://github.com/airbytehq/airbyte/pull/38341) | Add `max_waiting_time` to configuration |
+| 1.7.4   | 2024-05-21 | [38341](https://github.com/airbytehq/airbyte/pull/38341) | Update CDK authenticator package |
+| 1.7.3   | 2024-05-20 | [38299](https://github.com/airbytehq/airbyte/pull/38299) | Fixed spec typo |
+| 1.7.2   | 2024-04-19 | [36636](https://github.com/airbytehq/airbyte/pull/36636) | Updating to 0.80.0 CDK |
+| 1.7.1   | 2024-04-12 | [36636](https://github.com/airbytehq/airbyte/pull/36636) | schema descriptions |
+| 1.7.0   | 2024-03-19 | [36267](https://github.com/airbytehq/airbyte/pull/36267) | Pin airbyte-cdk version to `^0` |
+| 1.6.5   | 2024-03-12 | [35986](https://github.com/airbytehq/airbyte/pull/35986) | Handle rate limit exception as config error |
+| 1.6.4   | 2024-03-08 | [35915](https://github.com/airbytehq/airbyte/pull/35915) | Fix per stream error handler; Make use the latest CDK version |
+| 1.6.3   | 2024-02-15 | [35271](https://github.com/airbytehq/airbyte/pull/35271) | Update branches schema |
+| 1.6.2   | 2024-02-12 | [34933](https://github.com/airbytehq/airbyte/pull/34933) | Update Airbyte CDK for integration tests |
+| 1.6.1   | 2024-02-09 | [35087](https://github.com/airbytehq/airbyte/pull/35087) | Manage dependencies with Poetry. |
+| 1.6.0   | 2024-02-02 | [34700](https://github.com/airbytehq/airbyte/pull/34700) | Continue Sync on Stream failure |
+| 1.5.7   | 2024-01-29 | [34598](https://github.com/airbytehq/airbyte/pull/34598) | Fix MultipleToken sleep time |
+| 1.5.6   | 2024-01-26 | [34503](https://github.com/airbytehq/airbyte/pull/34503) | Fix MultipleToken rotation logic |
+| 1.5.5   | 2023-12-26 | [33783](https://github.com/airbytehq/airbyte/pull/33783) | Fix retry for 504 error in GraphQL based streams |
+| 1.5.4   | 2023-11-20 | [32679](https://github.com/airbytehq/airbyte/pull/32679) | Return AirbyteMessage if max retry exeeded for 202 status code |
+| 1.5.3   | 2023-10-23 | [31702](https://github.com/airbytehq/airbyte/pull/31702) | Base image migration: remove Dockerfile and use the python-connector-base image |
+| 1.5.2   | 2023-10-13 | [31386](https://github.com/airbytehq/airbyte/pull/31386) | Handle `ContributorActivity` continuous `ACCEPTED` response |
+| 1.5.1   | 2023-10-12 | [31307](https://github.com/airbytehq/airbyte/pull/31307) | Increase backoff_time for stream `ContributorActivity` |
+| 1.5.0   | 2023-10-11 | [31300](https://github.com/airbytehq/airbyte/pull/31300) | Update Schemas: Add date-time format to fields |
+| 1.4.6   | 2023-10-04 | [31056](https://github.com/airbytehq/airbyte/pull/31056) | Migrate spec properties' `repository` and `branch` type to \<array\> |
+| 1.4.5   | 2023-10-02 | [31023](https://github.com/airbytehq/airbyte/pull/31023) | Increase backoff for stream `Contributor Activity` |
+| 1.4.4   | 2023-10-02 | [30971](https://github.com/airbytehq/airbyte/pull/30971) | Mark `start_date` as optional. |
+| 1.4.3   | 2023-10-02 | [30979](https://github.com/airbytehq/airbyte/pull/30979) | Fetch archived records in `Project Cards` |
+| 1.4.2   | 2023-09-30 | [30927](https://github.com/airbytehq/airbyte/pull/30927) | Provide actionable user error messages |
+| 1.4.1   | 2023-09-30 | [30839](https://github.com/airbytehq/airbyte/pull/30839) | Update CDK to Latest version |
+| 1.4.0   | 2023-09-29 | [30823](https://github.com/airbytehq/airbyte/pull/30823) | Add new stream `issue Timeline Events` |
+| 1.3.1   | 2023-09-28 | [30824](https://github.com/airbytehq/airbyte/pull/30824) | Handle empty response in stream `ContributorActivity` |
+| 1.3.0   | 2023-09-25 | [30731](https://github.com/airbytehq/airbyte/pull/30731) | Add new stream `ProjectsV2` |
+| 1.2.1   | 2023-09-22 | [30693](https://github.com/airbytehq/airbyte/pull/30693) | Handle 404 error in `TeamMemberShips` |
+| 1.2.0   | 2023-09-22 | [30647](https://github.com/airbytehq/airbyte/pull/30647) | Add support for self-hosted GitHub instances |
+| 1.1.1   | 2023-09-21 | [30654](https://github.com/airbytehq/airbyte/pull/30654) | Rewrite source connection error messages |
+| 1.1.0   | 2023-08-03 | [30615](https://github.com/airbytehq/airbyte/pull/30615) | Add new stream `Contributor Activity` |
+| 1.0.4   | 2023-08-03 | [29031](https://github.com/airbytehq/airbyte/pull/29031) | Reverted `advancedAuth` spec changes |
+| 1.0.3   | 2023-08-01 | [28910](https://github.com/airbytehq/airbyte/pull/28910) | Updated `advancedAuth` broken references |
+| 1.0.2   | 2023-07-11 | [28144](https://github.com/airbytehq/airbyte/pull/28144) | Add `archived_at` property to `Organizations` schema parameter |
+| 1.0.1   | 2023-05-22 | [25838](https://github.com/airbytehq/airbyte/pull/25838) | Deprecate "page size" input parameter |
+| 1.0.0   | 2023-05-19 | [25778](https://github.com/airbytehq/airbyte/pull/25778) | Improve repo(s) name validation on UI |
+| 0.5.0   | 2023-05-16 | [25793](https://github.com/airbytehq/airbyte/pull/25793) | Implement client-side throttling of requests |
+| 0.4.11  | 2023-05-12 | [26025](https://github.com/airbytehq/airbyte/pull/26025) | Added more transparent depiction of the personal access token expired |
+| 0.4.10  | 2023-05-15 | [26075](https://github.com/airbytehq/airbyte/pull/26075) | Add more specific error message description for no repos case. |
+| 0.4.9   | 2023-05-01 | [24523](https://github.com/airbytehq/airbyte/pull/24523) | Add undeclared columns to spec |
 | 0.4.8   | 2023-04-19 | [00000](https://github.com/airbytehq/airbyte/pull/25312)                                                          | Fix repo name validation                                                                                                                                            |
 | 0.4.7   | 2023-03-24 | [24457](https://github.com/airbytehq/airbyte/pull/24457)                                                          | Add validation and transformation for repositories config                                                                                                           |
 | 0.4.6   | 2023-03-24 | [24398](https://github.com/airbytehq/airbyte/pull/24398)                                                          | Fix caching for `get_starting_point` in stream "Commits"                                                                                                            |