Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

🎉 Source Facebook: rate limit not always handled #3525 #3600

Merged
merged 13 commits into from
May 26, 2021
Merged

🎉 Source Facebook: rate limit not always handled #3525 #3600

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
d4233f0
fix backoff of batch requests
eugene-kulak May 22, 2021
9c858ab
fix backoff of batch requests
eugene-kulak May 24, 2021
dce049d
fix tests
eugene-kulak May 25, 2021
ec952d0
fix tests
eugene-kulak May 25, 2021
d1f3a13
fix requirements
eugene-kulak May 25, 2021
e21a25b
fix tests
eugene-kulak May 25, 2021
a4ff12a
fix tests
eugene-kulak May 25, 2021
62a0781
sort codes
eugene-kulak May 25, 2021
442b921
format
eugene-kulak May 25, 2021
a10e14a
fix setup
eugene-kulak May 25, 2021
b983a11
fix integration tests
eugene-kulak May 26, 2021
f46e6cf
bump version
eugene-kulak May 26, 2021
13c16ed
fix integration tests
eugene-kulak May 26, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
format
  • Loading branch information
@eugene-kulak
eugene-kulak committed May 25, 2021
commit 442b921904fb8588439c53026fb3b19a67e0aa32
1 change: 1 addition & 0 deletions ...egrations/connectors/source-facebook-marketing/source_facebook_marketing/client/common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#

import json
import sys
from time import sleep
Expand Down
74 changes: 40 additions & 34 deletions airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,12 +79,7 @@ def fb_account_response_fixture(account_id):
"id": f"act_{account_id}",
}
],
"paging": {
"cursors": {
"before": "MjM4NDYzMDYyMTcyNTAwNzEZD",
"after": "MjM4NDYzMDYyMTcyNTAwNzEZD"
}
}
"paging": {"cursors": {"before": "MjM4NDYzMDYyMTcyNTAwNzEZD", "after": "MjM4NDYzMDYyMTcyNTAwNzEZD"}},
},
"status_code": 200,
}
Expand All @@ -95,8 +90,10 @@ def test_limit_reached(self, requests_mock, client, fb_call_rate_response, accou
"""Error once, check that we retry and not fail"""
campaign_responses = [
fb_call_rate_response,
{"json": {"data": [{"id": 1, "updated_time": "2020-09-25T00:00:00Z"}, {"id": 2, "updated_time": "2020-09-25T00:00:00Z"}]},
"status_code": 200},
{
"json": {"data": [{"id": 1, "updated_time": "2020-09-25T00:00:00Z"}, {"id": 2, "updated_time": "2020-09-25T00:00:00Z"}]},
"status_code": 200,
},
]

requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/v10.0/act_{account_id}/campaigns", campaign_responses)
Expand All @@ -111,47 +108,56 @@ def test_batch_limit_reached(self, requests_mock, client, fb_call_rate_response,
"""Error once, check that we retry and not fail"""
responses = [
fb_call_rate_response,
{"json": {"data": [
{
"id": "123",
"object_type": "SHARE",
"status": "ACTIVE",
},
{
"id": "1234",
"object_type": "SHARE",
"status": "ACTIVE",
},
], "status_code": 200}}
{
"json": {
"data": [
{
"id": "123",
"object_type": "SHARE",
"status": "ACTIVE",
},
{
"id": "1234",
"object_type": "SHARE",
"status": "ACTIVE",
},
],
"status_code": 200,
}
},
]

batch_responses = [
fb_call_rate_response,
{"json": [
{
"body": json.dumps({"name": "creative 1"}),
"code": 200,
},
{
"body": json.dumps({"name": "creative 2"}),
"code": 200,
}
]}
{
"json": [
{
"body": json.dumps({"name": "creative 1"}),
"code": 200,
},
{
"body": json.dumps({"name": "creative 2"}),
"code": 200,
},
]
},
]

requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/v10.0/act_{account_id}/adcreatives", responses)
requests_mock.register_uri("POST", FacebookSession.GRAPH + f"/v10.0/", batch_responses)
requests_mock.register_uri("POST", FacebookSession.GRAPH + "/v10.0/", batch_responses)

records = list(client.read_stream(AirbyteStream(name="adcreatives", json_schema={})))

assert records == [{'name': 'creative 1'}, {'name': 'creative 2'}]
assert records == [{"name": "creative 1"}, {"name": "creative 2"}]

def test_server_error(self, requests_mock, client, account_id):
"""Error once, check that we retry and not fail"""
responses = [
{"json": {"error": {}}, "status_code": 500},
{"json": {"data": [{"id": 1, "updated_time": "2020-09-25T00:00:00Z"}, {"id": 2, "updated_time": "2020-09-25T00:00:00Z"}]},
"status_code": 200},
{
"json": {"data": [{"id": 1, "updated_time": "2020-09-25T00:00:00Z"}, {"id": 2, "updated_time": "2020-09-25T00:00:00Z"}]},
"status_code": 200,
},
]

requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/v10.0/act_{account_id}/campaigns", responses)
Expand Down