Implement client base and storage endpoints

* Implement client base and storage endpoints

* Update everything based on PR feedback and more

* Fix a few types

* Remove forgotten comment

* Validate that resource_path does not end with "/"

* Make supplying token optional

* Update installation instructions

* Add TODOs to key-value store client

* Use imperative mood in docstrings for resource collection clients

* Improve iterating dataset items logic when no limit is set

.editorconfig

@@ -0,0 +1,9 @@
+root = true
+
+[*]
+indent_style = space
+indent_size = 4
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+end_of_line = lf

.flake8

@@ -0,0 +1,20 @@
+[flake8]
+max_line_length = 150
+# Google docstring convention + D204 & D401
+# TODO enable D100 & D401
+docstring-convention=all
+ignore=
+    D100
+    D104
+    D203
+    D213
+    D215
+    D400
+    D404
+    D406
+    D407
+    D408
+    D409
+    D413
+per-file-ignores =
+    tests/*: D

.github/workflows/lint_and_test.yaml

@@ -0,0 +1,32 @@
+name: Lint and test
+
+on: push
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python-version: [3.7, 3.8, 3.9]
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e .[dev]
+
+    - name: Lint
+      run: ./lint_and_test.sh lint
+
+    - name: Type check
+      run: ./lint_and_test.sh types
+
+    - name: Unit tests
+      run: ./lint_and_test.sh tests

.gitignore

@@ -1,4 +1,12 @@
-build
-dist
-apify_client.egg-info
+.venv
+.direnv
+.envrc
+
+.mypy_cache
+.pytest_cache
+
+*.egg-info/
+*.egg
+
+.vscode
 .idea

.isort.cfg

@@ -0,0 +1,6 @@
+[isort]
+include_trailing_comma = True
+line_length = 150
+use_parentheses = True
+multi_line_output = 3
+sections = FUTURE,STDLIB,FIRSTPARTY,THIRDPARTY,LOCALFOLDER

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2018 Apify Technologies s.r.o.
+   Copyright 2020 Apify Technologies s.r.o.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

README.md

@@ -3,4 +3,41 @@
 This will be an official client for [Apify API](https://www.apify.com/docs/api/v2).
 It's still work in progress, so please don't use it yet!
 
+## Installation
 
+Requires Python 3.7+
+
+Right now the client is not available on PyPI yet, so you can install it only from the git repo.
+To do that, run `pip install git+https://github.com/apify/apify-client-python.git`
+
+## Development
+
+### Environment
+
+For local development, it is required to have Python 3.7 installed.
+
+It is recommended to set up a virtual environment while developing this package to isolate your development environment,
+however, due to the many varied ways Python can be installed and virtual environments can be set up,
+this is left up to the developers to do themselves.
+
+One recommended way is with the builtin `venv` module:
+
+```bash
+python3 -m venv .venv
+source .venv/bin/activate
+```
+
+To improve on the experience, you can use [pyenv](https://github.com/pyenv/pyenv) to have an environment with a pinned Python version,
+and [direnv](https://github.com/direnv/direnv) to automatically activate/deactivate the environment when you enter/exit the project folder.
+
+### Dependencies
+
+To install this package and its development dependencies, run `pip install -e '.[dev]'`
+
+### Formatting
+
+We use `autopep8` and `isort` to automatically format the code to a common format. To run the formatting, just run `./format.sh`.
+
+### Linting and Testing
+
+We use `flake8` for linting, `mypy` for type checking and `pytest` for unit testing. To run these tools, just run `./lint_and_test.sh`.

format.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env /bin/bash
+
+set -o errexit
+
+echo "🔤 isort formatting..."
+python3 -m isort src tests
+
+echo "🎨 autopep8 formatting..."
+python3 -m autopep8 --in-place --recursive src tests
+
+echo "🎉 formatted 🎉"

lint_and_test.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+set -u
+
+checks_to_run=${1:-"everything"}
+shift || true
+
+
+lint() {
+    echo "⛄️ flake8 check..."
+    python3 -m flake8 src tests
+}
+
+type_check() {
+    echo "📝 checking types"
+    python3 -m mypy src
+}
+
+unit_tests() {
+    echo "👮‍♀️ running unit tests"
+    python3 -m pytest -rA tests
+}
+
+
+if [ "$checks_to_run" = "lint" ] ; then
+    lint
+elif [ "$checks_to_run" = "types" ] ; then
+    type_check
+elif [ "$checks_to_run" = "tests" ] ; then
+    unit_tests
+elif [ "$checks_to_run" = "everything" ] ; then
+    lint
+    type_check
+    unit_tests
+else
+    echo "Invalid type of test ($checks_to_run) requested. Use lint / types / tests or leave empty to run all tests"
+    exit 1
+fi
+
+echo "🎉 success 🎉"

mypy.ini

@@ -0,0 +1,11 @@
+[mypy]
+check_untyped_defs = True
+disallow_incomplete_defs = True
+disallow_untyped_calls = True
+disallow_untyped_decorators = True
+disallow_untyped_defs = True
+no_implicit_optional = True
+warn_redundant_casts = True
+warn_return_any = True
+warn_unreachable = True
+warn_unused_ignores = True

pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+addopts = --doctest-modules

setup.py

@@ -1,22 +1,52 @@
-import setuptools
+import pathlib
 
-with open("README.md", "r") as fh:
-    long_description = fh.read()
+from setuptools import find_packages, setup
+
+here = pathlib.Path(__file__).parent.resolve()
+
+long_description = (here / 'README.md').read_text(encoding='utf-8')
+
+setup(
+    name='apify_client',
+    version='0.0.1',
 
-setuptools.setup(
-    name="apify_client",
-    version="0.0.1",
     author="Apify Technologies s.r.o.",
     author_email="support@apify.com",
-    description="Work in progress: Apify API client for Python",
+    url="https://github.com/apify/apify-client-python",
+    project_urls={
+        'Apify Homepage': 'https://apify.com',
+    },
+    license='Apache Software License',
+    license_file='LICENSE',
+
+    description='Apify API client for Python',
     long_description=long_description,
-    long_description_content_type="text/markdown",
-    url="https://github.com/apifytech/apify-client-python",
-    packages=setuptools.find_packages(),
+    long_description_content_type='text/markdown',
+
     classifiers=[
         "Programming Language :: Python :: 3",
-        "License :: OSI Approved :: MIT License",
+        "License :: OSI Approved :: Apache Software License",
         "Operating System :: OS Independent",
+        'Intended Audience :: Developers',
+        'Topic :: Software Development :: Libraries',
     ],
-)
+    keywords='apify, api, client, scraping, automation',
 
+    package_dir={'': 'src'},
+    packages=find_packages(where='src'),
+    python_requires='>=3.7',
+    install_requires=['requests ~=2.25.1'],
+    extras_require={
+        'dev': [
+            'autopep8 ~= 1.5.4',
+            'flake8 ~= 3.8.4',
+            'flake8-commas ~= 2.0.0',
+            'flake8-docstrings ~= 1.5.0',
+            'flake8-isort ~= 4.0.0',
+            'isort ~= 5.7.0',
+            'mypy ~= 0.790',
+            'pep8-naming ~= 0.11.1',
+            'pytest ~= 6.2.1',
+        ],
+    },
+)

src/apify_client/__init__.py

@@ -0,0 +1,3 @@
+from .client import ApifyClient
+
+__all__ = ['ApifyClient']

src/apify_client/_errors.py

@@ -0,0 +1,69 @@
+from typing import Optional
+
+import requests
+
+
+class ApifyClientError(Exception):
+    """Base class for errors specific to the Apify API Client."""
+
+    pass
+
+
+class ApifyApiError(ApifyClientError):
+    """Error specific to requests to the Apify API.
+
+    An `ApifyApiError` is thrown for successful HTTP requests that reach the API,
+    but the API responds with an error response. Typically, those are rate limit
+    errors and internal errors, which are automatically retried, or validation
+    errors, which are thrown immediately, because a correction by the user is needed.
+    """
+
+    def __init__(self, response: requests.models.Response, attempt: int) -> None:
+        """Create the ApifyApiError instance.
+
+        Args:
+            response: The response to the failed API call
+            attempt: Which retry was the request that failed
+        """
+        self.message: Optional[str] = None
+        self.type: Optional[str] = None
+
+        response_data = response.json()
+        if 'error' in response_data:
+            self.message = response_data['error']['message']
+            self.type = response_data['error']['type']
+        else:
+            self.message = f'Unexpected error: {response.text}'
+
+        super().__init__(self.message)
+
+        self.name = 'ApifyApiError'
+        self.status_code = response.status_code
+        self.attempt = attempt
+        self.http_method = response.request.method
+
+        # TODO self.client_method
+        # TODO self.original_stack
+        # TODO self.path
+        # TODO self.stack
+
+
+class InvalidResponseBodyError(ApifyClientError):
+    """Error caused by the response body failing to be parsed.
+
+    This error exists for the quite common situation, where only a partial JSON response is received and
+    an attempt to parse the JSON throws an error. In most cases this can be resolved by retrying the
+    request. We do that by identifying this error in the _HTTPClient.
+    """
+
+    def __init__(self, response: requests.models.Response) -> None:
+        """Create the InvalidResponseBodyError instance.
+
+        Args:
+            response: The response which failed to be parsed
+        """
+        super().__init__('Response body could not be parsed')
+
+        self.name = 'InvalidResponseBodyError'
+        self.code = 'invalid-response-body'
+        self.response = response