From 890e0396c94a5147715a7c08c4f354c40029b52a Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 4 Aug 2021 18:15:25 +0300 Subject: [PATCH] Remove HTTP Request source connector (#5185) --- .../9845d17a-45f1-4070-8a60-50914b1c8e2b.json | 8 -- .../resources/seed/source_definitions.yaml | 6 - airbyte-integrations/builds.md | 1 - .../connectors/source-http-request/Dockerfile | 17 --- .../connectors/source-http-request/README.md | 100 ---------------- .../source-http-request/build.gradle | 22 ---- .../integration_tests/config.json | 5 - .../integration_tests/configured_catalog.json | 15 --- .../source-http-request/main_dev.py | 33 ------ .../source-http-request/requirements.txt | 4 - .../sample_files/config.json | 5 - .../connectors/source-http-request/setup.py | 36 ------ .../source_http_request/__init__.py | 3 - .../source_http_request/source.py | 110 ------------------ .../source_http_request/spec.json | 33 ------ .../unit_tests/unit_test.py | 48 -------- .../migrate/migrations/MigrationV0_17_0.java | 1 - .../9845d17a-45f1-4070-8a60-50914b1c8e2b.json | 7 -- docs/SUMMARY.md | 1 - docs/integrations/README.md | 1 - docs/integrations/sources/http-request.md | 39 ------- 21 files changed, 495 deletions(-) delete mode 100644 airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9845d17a-45f1-4070-8a60-50914b1c8e2b.json delete mode 100644 airbyte-integrations/connectors/source-http-request/Dockerfile delete mode 100644 airbyte-integrations/connectors/source-http-request/README.md delete mode 100644 airbyte-integrations/connectors/source-http-request/build.gradle delete mode 100644 airbyte-integrations/connectors/source-http-request/integration_tests/config.json delete mode 100644 airbyte-integrations/connectors/source-http-request/integration_tests/configured_catalog.json delete mode 100644 airbyte-integrations/connectors/source-http-request/main_dev.py delete mode 100644 airbyte-integrations/connectors/source-http-request/requirements.txt delete mode 100644 airbyte-integrations/connectors/source-http-request/sample_files/config.json delete mode 100644 airbyte-integrations/connectors/source-http-request/setup.py delete mode 100644 airbyte-integrations/connectors/source-http-request/source_http_request/__init__.py delete mode 100644 airbyte-integrations/connectors/source-http-request/source_http_request/source.py delete mode 100644 airbyte-integrations/connectors/source-http-request/source_http_request/spec.json delete mode 100644 airbyte-integrations/connectors/source-http-request/unit_tests/unit_test.py delete mode 100644 airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/9845d17a-45f1-4070-8a60-50914b1c8e2b.json delete mode 100644 docs/integrations/sources/http-request.md diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9845d17a-45f1-4070-8a60-50914b1c8e2b.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9845d17a-45f1-4070-8a60-50914b1c8e2b.json deleted file mode 100644 index 9bc51f8cf26d..000000000000 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9845d17a-45f1-4070-8a60-50914b1c8e2b.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "sourceDefinitionId": "9845d17a-45f1-4070-8a60-50914b1c8e2b", - "name": "HTTP Request", - "dockerRepository": "airbyte/source-http-request", - "dockerImageTag": "0.2.4", - "documentationUrl": "https://hub.docker.com/repository/docker/airbyte/source-http-request", - "icon": "http.svg" -} diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 13f1342fd820..d04cdf0fc1fb 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -150,12 +150,6 @@ dockerRepository: airbyte/source-shopify dockerImageTag: 0.1.11 documentationUrl: https://docs.airbyte.io/integrations/sources/shopify -- sourceDefinitionId: 9845d17a-45f1-4070-8a60-50914b1c8e2b - name: HTTP Request - dockerRepository: airbyte/source-http-request - dockerImageTag: 0.2.4 - documentationUrl: https://hub.docker.com/repository/docker/airbyte/source-http-request - icon: http.svg - sourceDefinitionId: e87ffa8e-a3b5-f69c-9076-6011339de1f6 name: Redshift dockerRepository: airbyte/source-redshift diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index 8693c2e4a367..9c2e3f643c36 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -29,7 +29,6 @@ | Google Directory API | [![source-google-directory](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-google-directory%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-google-directory) | | Google Workspace Admin | [![source-google-workspace-admin-reports](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-google-workspace-admin-reports%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-google-workspace-admin-reports) | | Greenhouse | [![source-greenhouse](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-greenhouse%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-greenhouse) | -| HTTP Request | [![source-http-request](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-http-request%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-http-request) | | Hubspot | [![source-hubspot-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-hubspot%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-hubspot) | | Klaviyo | [![source-klaviyo](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-klaviyo%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-klaviyo) | | IBM Db2 | [![source-db2](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-db2%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-db2) | diff --git a/airbyte-integrations/connectors/source-http-request/Dockerfile b/airbyte-integrations/connectors/source-http-request/Dockerfile deleted file mode 100644 index 661132a0d54e..000000000000 --- a/airbyte-integrations/connectors/source-http-request/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM airbyte/integration-base-python:0.1.1 - -RUN apt-get update && apt-get install -y jq curl bash && rm -rf /var/lib/apt/lists/* - -ENV CODE_PATH="source_http_request" -ENV AIRBYTE_IMPL_MODULE="source_http_request" -ENV AIRBYTE_IMPL_PATH="SourceHttpRequest" - -WORKDIR /airbyte/integration_code -COPY $CODE_PATH ./$CODE_PATH -COPY setup.py ./ -RUN pip install . - -ENV AIRBYTE_ENTRYPOINT "/airbyte/base.sh" - -LABEL io.airbyte.version=0.2.4 -LABEL io.airbyte.name=airbyte/source-http-request diff --git a/airbyte-integrations/connectors/source-http-request/README.md b/airbyte-integrations/connectors/source-http-request/README.md deleted file mode 100644 index 40290aa5b7bc..000000000000 --- a/airbyte-integrations/connectors/source-http-request/README.md +++ /dev/null @@ -1,100 +0,0 @@ -# Http Request Source - -This is the repository for the Http Request source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/http-request). - -## Local development - -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-http-request:build -``` - -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/http-request) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_http_request/spec.json` file. -Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. -See `sample_files/sample_config.json` for a sample config file. - -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source http-request test creds` -and place them into `secrets/config.json`. - - -### Locally running the connector -``` -python main_dev.py spec -python main_dev.py check --config secrets/config.json -python main_dev.py discover --config secrets/config.json -python main_dev.py read --config secrets/config.json --catalog sample_files/configured_catalog.json -``` - -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` - -### Locally running the connector docker image - -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-http-request:dev -``` - -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-http-request:airbyteDocker -``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. - -#### Run -Then run any of the connector commands as follows: -``` -docker run --rm airbyte/source-http-request:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-http-request:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-http-request:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-http-request:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json -``` - -### Integration Tests -1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-http-request:integrationTest` to run the standard integration test suite. -1. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s integration_tests`. - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. - -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. - -### Publishing a new version of the connector -You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer). -1. Create a Pull Request -1. Pat yourself on the back for being an awesome contributor -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master diff --git a/airbyte-integrations/connectors/source-http-request/build.gradle b/airbyte-integrations/connectors/source-http-request/build.gradle deleted file mode 100644 index 27654aa3bafe..000000000000 --- a/airbyte-integrations/connectors/source-http-request/build.gradle +++ /dev/null @@ -1,22 +0,0 @@ -plugins { - id 'java' - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-standard-source-test-file' -} - -airbytePython { - moduleDirectory 'source_http_request' -} - -airbyteStandardSourceTestFile { - specPath = "source_http_request/spec.json" - configPath = "sample_files/config.json" - configuredCatalogPath = "integration_tests/configured_catalog.json" -} - - -dependencies { - implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) - implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) -} diff --git a/airbyte-integrations/connectors/source-http-request/integration_tests/config.json b/airbyte-integrations/connectors/source-http-request/integration_tests/config.json deleted file mode 100644 index af5c0301295b..000000000000 --- a/airbyte-integrations/connectors/source-http-request/integration_tests/config.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "http_method": "get", - "url": "http://api.bart.gov/api/route.aspx?cmd=routes&key=MW9S-E7SL-26DU-VV8V&json=y", - "headers": "{\"Content-Type\": \"application/json\"}" -} diff --git a/airbyte-integrations/connectors/source-http-request/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-http-request/integration_tests/configured_catalog.json deleted file mode 100644 index 1500bd4ffb03..000000000000 --- a/airbyte-integrations/connectors/source-http-request/integration_tests/configured_catalog.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "data", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object" - } - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-http-request/main_dev.py b/airbyte-integrations/connectors/source-http-request/main_dev.py deleted file mode 100644 index 741ea1c1a57b..000000000000 --- a/airbyte-integrations/connectors/source-http-request/main_dev.py +++ /dev/null @@ -1,33 +0,0 @@ -# -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# - - -import sys - -from base_python.entrypoint import launch -from source_http_request import SourceHttpRequest - -if __name__ == "__main__": - source = SourceHttpRequest() - launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-http-request/requirements.txt b/airbyte-integrations/connectors/source-http-request/requirements.txt deleted file mode 100644 index dd447512e620..000000000000 --- a/airbyte-integrations/connectors/source-http-request/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/airbyte-protocol --e ../../bases/base-python --e . diff --git a/airbyte-integrations/connectors/source-http-request/sample_files/config.json b/airbyte-integrations/connectors/source-http-request/sample_files/config.json deleted file mode 100644 index af5c0301295b..000000000000 --- a/airbyte-integrations/connectors/source-http-request/sample_files/config.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "http_method": "get", - "url": "http://api.bart.gov/api/route.aspx?cmd=routes&key=MW9S-E7SL-26DU-VV8V&json=y", - "headers": "{\"Content-Type\": \"application/json\"}" -} diff --git a/airbyte-integrations/connectors/source-http-request/setup.py b/airbyte-integrations/connectors/source-http-request/setup.py deleted file mode 100644 index b01417e89944..000000000000 --- a/airbyte-integrations/connectors/source-http-request/setup.py +++ /dev/null @@ -1,36 +0,0 @@ -# -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# - - -from setuptools import find_packages, setup - -setup( - name="source_http_request", - description="Source implementation for HTTP Request.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=["airbyte-protocol", "base-python", "requests", "pytest==6.1.2"], - package_data={"": ["*.json"]}, -) diff --git a/airbyte-integrations/connectors/source-http-request/source_http_request/__init__.py b/airbyte-integrations/connectors/source-http-request/source_http_request/__init__.py deleted file mode 100644 index ccc01f5ac60e..000000000000 --- a/airbyte-integrations/connectors/source-http-request/source_http_request/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .source import SourceHttpRequest - -__all__ = ["SourceHttpRequest"] diff --git a/airbyte-integrations/connectors/source-http-request/source_http_request/source.py b/airbyte-integrations/connectors/source-http-request/source_http_request/source.py deleted file mode 100644 index 27031d7277c7..000000000000 --- a/airbyte-integrations/connectors/source-http-request/source_http_request/source.py +++ /dev/null @@ -1,110 +0,0 @@ -# -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# - - -import json -from datetime import datetime -from typing import Dict, Generator - -import requests -from airbyte_protocol import ( - AirbyteCatalog, - AirbyteConnectionStatus, - AirbyteMessage, - AirbyteRecordMessage, - AirbyteStream, - ConfiguredAirbyteCatalog, - Status, - Type, -) -from base_python import AirbyteLogger, Source - - -class SourceHttpRequest(Source): - STREAM_NAME = "data" - - def check(self, logger: AirbyteLogger, config: json) -> AirbyteConnectionStatus: - r = self._make_request(config) - if r.status_code == 200: - return AirbyteConnectionStatus(status=Status.SUCCEEDED) - else: - return AirbyteConnectionStatus(status=Status.FAILED, message=r.text) - - def discover(self, logger: AirbyteLogger, config: json) -> AirbyteCatalog: - json_schema = { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": True, - "type": "object", - # todo (cgardens) - remove data column. added to handle UI bug where streams without fields cannot be selected. - # issue: https://github.com/airbytehq/airbyte/issues/1104 - "properties": {"data": {"type": "object"}}, - } - - # json body will be returned as the "data" stream". we can't know its schema ahead of time, so we assume it's object (i.e. valid json). - return AirbyteCatalog(streams=[AirbyteStream(name=SourceHttpRequest.STREAM_NAME, json_schema=json_schema)]) - - def read( - self, logger: AirbyteLogger, config: json, catalog: ConfiguredAirbyteCatalog, state: Dict[str, any] - ) -> Generator[AirbyteMessage, None, None]: - r = self._make_request(config) - if r.status_code != 200: - raise Exception(f"Request failed. {r.text}") - - # need to eagerly fetch the json. - data = r.json() - if not isinstance(data, list): - data = [data] - - for record in data: - yield AirbyteMessage( - type=Type.RECORD, - record=AirbyteRecordMessage( - stream=SourceHttpRequest.STREAM_NAME, data=record, emitted_at=int(datetime.now().timestamp()) * 1000 - ), - ) - - def _make_request(self, config): - parsed_config = self._parse_config(config) - http_method = parsed_config.get("http_method").lower() - url = parsed_config.get("url") - headers = parsed_config.get("headers", {}) - body = parsed_config.get("body", {}) - - if http_method == "get": - r = requests.get(url, headers=headers, json=body) - elif http_method == "post": - r = requests.post(url, headers=headers, json=body) - else: - raise Exception(f"Did not recognize http_method: {http_method}") - - return r - - # visible / separated for testing - def _parse_config(self, config): - return { - "url": config.get("url"), - "http_method": config["http_method"], - "headers": json.loads(config.get("headers", "{}")), - "body": json.loads(config.get("body", "{}")), - } diff --git a/airbyte-integrations/connectors/source-http-request/source_http_request/spec.json b/airbyte-integrations/connectors/source-http-request/source_http_request/spec.json deleted file mode 100644 index 737e5921e5de..000000000000 --- a/airbyte-integrations/connectors/source-http-request/source_http_request/spec.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.io/integrations/sources/http-request", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Rest Api Spec", - "type": "object", - "required": ["url", "http_method"], - "additionalProperties": false, - "properties": { - "url": { - "description": "url for the request, including query params. must start with http:// or https://", - "type": "string", - "pattern": "^http(s)?:\\/\\/.*$" - }, - "http_method": { - "description": "http method to use for the request", - "type": "string", - "enum": ["GET", "POST"] - }, - "headers": { - "description": "json object where keys are header names and values the values.", - "type": "string", - "examples": [ - "{ \"Accept-Charset\": \"utf-8\", \"Content-Type\": \"application/json\"}" - ] - }, - "body": { - "description": "json body to be included in the request", - "type": "string" - } - } - } -} diff --git a/airbyte-integrations/connectors/source-http-request/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-http-request/unit_tests/unit_test.py deleted file mode 100644 index 100effdc4793..000000000000 --- a/airbyte-integrations/connectors/source-http-request/unit_tests/unit_test.py +++ /dev/null @@ -1,48 +0,0 @@ -# -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# - - -import unittest - -from source_http_request import SourceHttpRequest - - -class TestSourceHttpRequest(unittest.TestCase): - def test_parse_config(self): - config = { - "http_method": "get", - "url": "http://api.bart.gov/api", - "headers": '{"Content-Type": "application/json"}', - "body": '{"something": "good"}', - } - - source = SourceHttpRequest() - actual = source._parse_config(config) - expected = { - "http_method": "get", - "url": "http://api.bart.gov/api", - "headers": {"Content-Type": "application/json"}, - "body": {"something": "good"}, - } - self.assertEqual(expected, actual) diff --git a/airbyte-migration/src/main/java/io/airbyte/migrate/migrations/MigrationV0_17_0.java b/airbyte-migration/src/main/java/io/airbyte/migrate/migrations/MigrationV0_17_0.java index 0c06e513680f..b7ad8fe1a8d5 100644 --- a/airbyte-migration/src/main/java/io/airbyte/migrate/migrations/MigrationV0_17_0.java +++ b/airbyte-migration/src/main/java/io/airbyte/migrate/migrations/MigrationV0_17_0.java @@ -88,7 +88,6 @@ public class MigrationV0_17_0 extends BaseMigration implements Migration { "airbyte/source-facebook-marketing", "airbyte/source-hubspot-singer", "airbyte/source-shopify-singer", - "airbyte/source-http-request", "airbyte/source-redshift", "airbyte/source-twilio-singer", "airbyte/source-freshdesk", diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/9845d17a-45f1-4070-8a60-50914b1c8e2b.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/9845d17a-45f1-4070-8a60-50914b1c8e2b.json deleted file mode 100644 index 43fe05829865..000000000000 --- a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/9845d17a-45f1-4070-8a60-50914b1c8e2b.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "sourceDefinitionId": "9845d17a-45f1-4070-8a60-50914b1c8e2b", - "name": "HTTP Request", - "dockerRepository": "airbyte/source-http-request", - "dockerImageTag": "0.2.1", - "documentationUrl": "https://hub.docker.com/repository/docker/airbyte/source-http-request" -} diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index ed15e76a2cb8..7a95629ab1e8 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -57,7 +57,6 @@ * [Google Sheets](integrations/sources/google-sheets.md) * [Google Workspace Admin Reports](integrations/sources/google-workspace-admin-reports.md) * [Greenhouse](integrations/sources/greenhouse.md) - * [HTTP Request](integrations/sources/http-request.md) * [Harvest](integrations/sources/harvest.md) * [Hubspot](integrations/sources/hubspot.md) * [Instagram](integrations/sources/instagram.md) diff --git a/docs/integrations/README.md b/docs/integrations/README.md index a55ebea156bd..ed606d8e9900 100644 --- a/docs/integrations/README.md +++ b/docs/integrations/README.md @@ -42,7 +42,6 @@ Airbyte uses a grading system for connectors to help users understand what to ex |[Google Sheets](./sources/google-sheets.md)| Certified | |[Google Workspace Admin Reports](./sources/google-workspace-admin-reports.md)| Certified | |[Greenhouse](./sources/greenhouse.md)| Beta | -|[HTTP Request](./sources/http-request.md)| Alpha | |[Hubspot](./sources/hubspot.md)| Certified | |[Instagram](./sources/instagram.md)| Certified | |[Intercom](./sources/intercom.md)| Beta | diff --git a/docs/integrations/sources/http-request.md b/docs/integrations/sources/http-request.md deleted file mode 100644 index 8e76cb38d4b4..000000000000 --- a/docs/integrations/sources/http-request.md +++ /dev/null @@ -1,39 +0,0 @@ -# HTTP Request - -## Overview - -This source allows you to make any http request an Airbyte source! On each sync, this source makes a single http request to the provided URL. Whatever json body is returned is written to the destination. - -#### Output schema - -It contains one stream: `data`. That stream will contain one record which is the json blob returned by the http request. - -#### Data type mapping - -`data` is a json blob. - -#### Features - -| Feature | Supported? | -| :--- | :--- | -| Full Refresh Sync | Yes | -| Incremental Sync | Coming soon | -| Namespaces | No | - -### Getting started - -#### Setup guide - -Provide a url, http\_method, \(optional\) headers, \(optional\) request body. The source will make exactly this http request. - -## Changelog - -| Version | Date | Pull Request | Subject | -| :------ | :-------- | :----- | :------ | -| 0.2.4 | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add AIRBYTE_ENTRYPOINT for Kubernetes support | -| 0.2.3 | 2021-04-20 | [3165](https://github.com/airbytehq/airbyte/pull/3165) | Version bump | -| 0.2.1 | 2021-04-03 | [2726](https://github.com/airbytehq/airbyte/pull/2726) | Fix base connector versioning | -| 0.2.0 | 2021-03-09 | [2238](https://github.com/airbytehq/airbyte/pull/2238) | Protocol allows future/unknown properties | -| 0.2.1 | 2021-04-03 | [2726](https://github.com/airbytehq/airbyte/pull/2726) | Fix base connector versioning | -| 0.1.1 | 2021-03-07 | [2342](https://github.com/airbytehq/airbyte/pull/2342) | Fix incorrect documentation | -| 0.1.0 | 2020-11-30 | [1046](https://github.com/airbytehq/airbyte/pull/1046) | Add connectors using an index YAML file |