Apify Client v1 (apify#146)

* Reimplement first actor endpoints * Progress commit * Completed tests for actors endpoint. * Task endpoint partial rewrote. * rewrote users. * rewrote logs.js * started dataset rewrote. * keyValuesStores basic methods rewrote. * rewrote request_queues.js * rewrote webhooks.js * rewrote webhook_dispatches.js. * rewrote dataset tests. Removed Crawler. * rewrote key_value_stores.js tests * xited tests that are not working, because of the client implementation changes -> up for discussion if we want to keep the support. * stuck in rewriting to async-retry. * fixed exponential backoff... I guess * make working browser POC. * make browser test work. * used brotli compressor to reduce the bundle size. * introduced latest features from master. * refactored code. split lodash to functions because of the bundle size. * removed deprecated method from task * fixed lint err * added workflows. * improved build script to build files for the test. * removed prepare script. * split build to two parts. * improved webpack browser build. * removed unnecessary dependencies and engines. * migrated to jest due to performance * fixed lint * made tests green again, * started abstract class. * Comments from PR. * renamed edpoint to resource. Deleted extra expbackoff logic from dataset. * changed webpack mode. * added defaults. * improved text decoding. * documented listActors endpoint. * updated docs for actor endpoint. * improved dataset endpoint docs. * documented dataset endpoint. * improved KV stores docs. * logs docs slightly improved. * Request queue docs improved. * improved tasks docs. * documented user endpoint. * documented webhook dispatches * documented webhooks endpoint. * added build docs action. * improved docs strategy. * fixed yml * fix * refactored actions. * refactored actions. * renamed. * on added * increased version added my parameter to list actors endpoint. * added dist * fix because of null from integration tests. * renamed tilde separated to slash separated. * added missing return. * Fixed docs. Improved README.md. Added CHANGELOG.md. Bump version. * Added possibility to remove content-type header with null and fixed JSON.strigify on buffer. * fixed getValue * Implementing schedules API * Adding tests and linting * fixed header removal and log endpoint. * improved log tests increased beta version. * Fix typo in version * Fix lint * Fixed tasks endpoint * fixed signing url. task input handling. * removed babel for nodeJs. * improved browser header to don't trigger error. * fixed docs. * bump version * Some changes * Not great, not terrible * Add missing clients * Add clients to index * Dataset tests work * Dataset tests pass * Key value stores * Key value stores finished * Logs finished * Request queue finished * Schedules finished * Tasks finished * Users finished * Webhooks finished * Webhook Dispatches finished * Remove old stuff, refactor, fix tests * Test Statistics * Bump version * Minor improvements * Add response parsing logic * Add status params to actor runs endpoint * Fix axios defaults * Bump version * Update KVS, bump version * Lint files * Some changes * Minor improvements * Add waitForFinish function to Run and Build * Add waitForFinish tests * Fix lint * Added call method to actor * Fix invalid return value in getRecord * Fix setRecord with stringified JSON, remove lodash * Added private method _call to resource client * Added call method to task client, update actor call method * Bump version * Added call method tests * fixed condition * further refactoring * small changes & made tests green again * Bump version * Change signature of start and call methods * Update comment * Change metamorph signature * Improve ApifyApiError * Fix trailing slash and add some improvements * Bump version * Fix v2 in baseUrl and tests * Improve error logging, fix 404 on redirects * Add browser test for api error, test node 14 not 10 Co-authored-by: Petr Patek <petrpatekml@gmail.com> Co-authored-by: Marek Trunkat <marek@trunkat.eu> Co-authored-by: Jan Bárta <jbarta.edu@gmail.com>
magne4000 · Oct 22, 2020 · 27f494e · 27f494e
1 parent 87322fb
commit 27f494e
Show file tree

Hide file tree

Showing 102 changed files with 5,636 additions and 7,654 deletions.
diff --git a/.babelrc b/.babelrc
diff --git a/.eslintrc.json b/.eslintrc.json
@@ -1,30 +1,3 @@
 {
-  "extends": "airbnb-base",
-  "plugins": [
-    "import",
-    "promise"
-  ],
-  "rules": {
-    "indent": ["error", 4, { "SwitchCase": 1 }],
-    "no-underscore-dangle": [2, {
-      "allow": ["_id", "_overrideMethodGroups"]
-    }],
-    "no-use-before-define": 0,
-    "no-param-reassign": 0,
-    "consistent-return": 0,
-    "no-buffer-constructor": 1,
-    "object-curly-newline": 0,
-    "function-paren-newline": 0,
-    "prefer-destructuring": 0,
-    "array-callback-return": 0,
-    "arrow-body-style": 0,
-    "no-plusplus": 0,
-    "strict": ["error", "global"],
-    "max-len": ["error", 150],
-    "no-undef": 0,
-    "func-names": 0,
-    "import/prefer-default-export": 0,
-    "import/no-absolute-path": 0,
-    "import/no-extraneous-dependencies": ["error", { "devDependencies": ["**/test/*.js"] }]
-  }
+  "extends": "@apify"
 }
diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml
@@ -0,0 +1,24 @@
+name: Build docs
+
+on: pull_request
+jobs:
+    build:
+
+        runs-on: ubuntu-latest
+
+        strategy:
+            matrix:
+                node-version: [12.x]
+
+        steps:
+            - uses: actions/checkout@v2
+            - name: Use Node.js ${{ matrix.node-version }}
+              uses: actions/setup-node@v1
+              with:
+                  node-version: ${{ matrix.node-version }}
+            - run: npm install
+            - run: npm run build --if-present
+            - run: npm run build-doc
+            - run: npm run build-toc
+              env:
+                  CI: true
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -0,0 +1,25 @@
+name: Build&Test
+
+on: pull_request
+
+jobs:
+    build:
+
+        runs-on: ubuntu-latest
+
+        strategy:
+            matrix:
+                node-version: [14.x, 12.x]
+
+        steps:
+            - uses: actions/checkout@v2
+            - name: Use Node.js ${{ matrix.node-version }}
+              uses: actions/setup-node@v1
+              with:
+                  node-version: ${{ matrix.node-version }}
+            - run: npm install
+            - run: npm run build --if-present
+            - run: npm run lint
+            - run: npm test
+              env:
+                  CI: true
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,5 @@
 build
+dist
 docs
 node_modules
 *.log

diff --git a/.npmignore b/.npmignore
diff --git a/.travis.yml b/.travis.yml
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,13 @@
+1.0.0 / NEXT
+===================
+- TODO
+- Added support for schedules API
+- Replaced `request` with `axios`.
+- Added browser compatibility.
+- Removed `setOptions` and `getOptions` client methods.
+- Removed callbacks support.
+- Removed API v1 deprecated stuff.
+
 0.6.0 / 2019/12/05
 ===================
 - Removed legacy Apify Crawler methods that are no longer supported by Apify API.

diff --git a/README.md b/README.md
@@ -45,21 +45,20 @@ const apifyClient = new ApifyClient({
 
 // Storage
 const store = await apifyClient.keyValueStores.getOrCreateStore({ storeName: 'my-store' });
-apifyClient.setOptions({ storeId: store._id });
 await apifyClient.keyValueStores.putRecord({
     key: 'foo',
     body: 'bar',
     contentType: 'text/plain',
+    storeId: store.id
 });
-const record = await apifyClient.keyValueStores.getRecord({ key: 'foo' });
-const keys = await apifyClient.keyValueStores.getRecordsKeys();
-await apifyClient.keyValueStores.deleteRecord({ key: 'foo' });
+const record = await apifyClient.keyValueStores.getRecord({ key: 'foo', storeId: store.id });
+const keys = await apifyClient.keyValueStores.listKeys({storeId: store.id});
+await apifyClient.keyValueStores.deleteRecord({ key: 'foo', storeId: store.id });
 
 // Actors
 const act = await apifyClient.acts.getAct({ actId: 'kjnjknDDNkl' });
-apifyClient.setOptions({ actId: 'kjnjknDDNkl' });
-const build = await apifyClient.acts.buildAct();
-const run = await apifyClient.acts.runAct();
+const build = await apifyClient.acts.buildAct({ actId: 'kjnjknDDNkl'});
+const run = await apifyClient.acts.runAct({ actId: 'kjnjknDDNkl'});
 
 ```
 
@@ -71,7 +70,6 @@ You can set global parameters when you are creating instance of ApifyClient:
 const apifyClient = new ApifyClient({
     userId: 'jklnDMNKLekk', // Your Apify user ID
     token: 'SNjkeiuoeD443lpod68dk', // Your API token
-    promise: Promise, // Promises dependency to use (default is native Promise)
     expBackOffMillis: 500, // Wait time in milliseconds before making a new request in a case of error
     expBackOffMaxRepeats: 8, // Maximum number of repeats in a case of error
 });
@@ -104,10 +102,6 @@ apifyClient.acts.getAct(options)
         // Do something with error ...
     });
 
-// Callback
-apifyClient.acts.getAct(options, (err, actor) => {
-    // Do something with error and actor ...
-});
 ```
 
 ## Parsing of date fields

diff --git a/jest.config.js b/jest.config.js
@@ -0,0 +1,17 @@
+const path = require('path');
+
+module.exports = {
+    verbose: true,
+    // Switch to false for debugging
+    silent: true,
+    rootDir: path.join(__dirname, './'),
+    testMatch: [path.join(__dirname, '**/test/?(*.)+(spec|test).[tj]s?(x)')],
+    testEnvironment: 'node',
+    testTimeout: 20000,
+    collectCoverage: true,
+    collectCoverageFrom: [
+        '**/src/**/*.js',
+        '!**/node_modules/**',
+    ],
+    maxWorkers: 3,
+};
diff --git a/jsdoc/apify_jsdoc_plugin.js b/jsdoc/apify_jsdoc_plugin.js
@@ -12,13 +12,6 @@ const APIFY_GENERIC_PARAMS = {
             names: ['String'],
         },
     },
-    callback: {
-        optional: true,
-        description: '<p>Callback function</p>',
-        type: {
-            names: ['function'],
-        },
-    },
 };
 
 exports.handlers = {

diff --git a/jsdoc/conf.json b/jsdoc/conf.json
@@ -3,8 +3,8 @@
         "allowUnknownTags": true
     },
     "source": {
-        "include": [ "build" ],
-        "exclude": [ "build/utils.js" ],
+        "include": [ "src" ],
+        "exclude": [ "src/utils.js" ],
         "includePattern": ".+\\.js(doc|x)?$",
         "excludePattern": "(^|\\/|\\\\)_"
     },

diff --git a/package.json b/package.json
@@ -1,79 +1,91 @@
 {
-  "name": "apify-client",
-  "version": "0.6.0",
-  "description": "Apify API client for JavaScript",
-  "main": "build/index.js",
-  "keywords": [
-    "apify",
-    "api",
-    "apifier",
-    "crawler",
-    "scraper"
-  ],
-  "engines": {
-    "node": ">=8.0.0"
-  },
-  "author": {
-    "name": "Apify",
-    "email": "support@apify.com",
-    "url": "https://apify.com"
-  },
-  "contributors": [
-    "Jan Curn <jan@apify.com>",
-    "Marek Trunkát <marek@apify.com>",
-    "Lubos Turek <lubos.turek@gmail.com>",
-    "Ondra Urban <ondra@apify.com>",
-    "Jakub Drobník <jakub.drobnik@apify.com>"
-  ],
-  "license": "Apache-2.0",
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/apifytech/apify-client-js"
-  },
-  "bugs": {
-    "url": "https://github.com/apifytech/apify-client-js/issues"
-  },
-  "homepage": "https://apify.com/docs/sdk/apify-client-js/latest",
-  "files": [
-    "build"
-  ],
-  "scripts": {
-    "build": "babel src --out-dir build",
-    "build-doc": "npm run clean && npm run build && node ./node_modules/jsdoc/jsdoc.js --package ./package.json -c ./jsdoc/conf.json -d docs",
-    "build-toc": "./node_modules/.bin/markdown-toc README.md -i",
-    "test": "npm run build && nyc --reporter=html --reporter=text mocha --timeout 5000 --compilers js:babel-core/register --recursive",
-    "prepare": "npm run build-toc && npm run build",
-    "prepublishOnly": "(test $RUNNING_FROM_SCRIPT || (echo \"You must use publish.sh instead of 'npm publish' directly!\"; exit 1)) && npm test && npm run lint",
-    "clean": "rm -rf build && rm -rf docs",
-    "lint": "npm run build && eslint src test"
-  },
-  "dependencies": {
-    "apify-shared": "^0.1.45",
-    "content-type": "^1.0.4",
-    "request": "^2.88.0",
-    "request-promise-native": "^1.0.7",
-    "type-check": "^0.3.2",
-    "underscore": "^1.9.1"
-  },
-  "devDependencies": {
-    "apify-jsdoc-template": "github:apifytech/apify-jsdoc-template",
-    "babel-cli": "^6.26.0",
-    "babel-plugin-add-module-exports": "^1.0.0",
-    "babel-preset-env": "^1.6.1",
-    "chai": "^4.1.2",
-    "deploy-web-to-s3": "^1.3.0",
-    "eslint": "^5.5.0",
-    "eslint-config-airbnb": "^17.1.0",
-    "eslint-config-airbnb-base": "^13.1.0",
-    "eslint-plugin-import": "^2.11.0",
-    "eslint-plugin-jsx-a11y": "^6.0.2",
-    "eslint-plugin-promise": "^4.0.1",
-    "eslint-plugin-react": "^7.4.0",
-    "jsdoc": "^3.4.3",
-    "jsdoc-export-default-interop": "^0.3.1",
-    "markdown-toc": "^1.2.0",
-    "mocha": "^5.2.0",
-    "nyc": "^13.0.1",
-    "sinon": "^6.3.1"
-  }
+    "name": "apify-client",
+    "version": "1.0.0-beta.12",
+    "description": "Apify API client for JavaScript",
+    "main": "src/index.js",
+    "keywords": [
+        "apify",
+        "api",
+        "apifier",
+        "crawler",
+        "scraper"
+    ],
+    "author": {
+        "name": "Apify",
+        "email": "support@apify.com",
+        "url": "https://apify.com"
+    },
+    "contributors": [
+        "Jan Curn <jan@apify.com>",
+        "Marek Trunkát <marek@apify.com>",
+        "Ondra Urban <ondra@apify.com>",
+        "Jakub Drobník <jakub.drobnik@apify.com>"
+    ],
+    "license": "Apache-2.0",
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/apifytech/apify-client-js"
+    },
+    "bugs": {
+        "url": "https://github.com/apifytech/apify-client-js/issues"
+    },
+    "homepage": "https://apify.com/docs/sdk/apify-client-js/latest",
+    "files": [
+        "src",
+        "dist"
+    ],
+    "browser": {
+        "agentkeepalive": false,
+        "fs": false,
+        "os": false,
+        "stream": false,
+        "util": false,
+        "zlib": false
+    },
+    "scripts": {
+        "build": "npm run clean && npm run build-browser",
+        "build-doc": "npm run clean && node ./node_modules/jsdoc/jsdoc.js --package ./package.json -c ./jsdoc/conf.json -d docs",
+        "build-toc": "./node_modules/.bin/markdown-toc README.md -i",
+        "test": "npm run build && jest",
+        "prepare": "npm run build",
+        "prepublishOnly": "(test $RUNNING_FROM_SCRIPT || (echo \"You must use publish.sh instead of 'npm publish' directly!\"; exit 1)) && npm test && npm run lint",
+        "clean": "rm -rf dist && rm -rf docs",
+        "lint": "eslint src test",
+        "lint:fix": "eslint --fix src test",
+        "build-browser": "webpack"
+    },
+    "dependencies": {
+        "agentkeepalive": "^4.1.3",
+        "apify-shared": "^0.5.0",
+        "async-retry": "^1.3.1",
+        "axios": "^0.20.0",
+        "content-type": "^1.0.4",
+        "ow": "^0.18.0"
+    },
+    "devDependencies": {
+        "@apify/eslint-config": "^0.1.2",
+        "@babel/cli": "^7.11.6",
+        "@babel/core": "^7.11.6",
+        "@babel/preset-env": "^7.11.5",
+        "@babel/register": "^7.11.5",
+        "@types/express": "^4.17.8",
+        "@types/jest": "^26.0.14",
+        "@types/node": "^14.11.5",
+        "apify": "^0.21.7",
+        "apify-jsdoc-template": "github:apifytech/apify-jsdoc-template",
+        "babel-loader": "^8.1.0",
+        "body-parser": "^1.19.0",
+        "brotli-webpack-plugin": "^1.1.0",
+        "compression": "^1.7.4",
+        "deploy-web-to-s3": "^1.3.1",
+        "eslint": "^7.10.0",
+        "express": "^4.17.1",
+        "jest": "^26.5.0",
+        "jsdoc": "^3.6.6",
+        "jsdoc-export-default-interop": "^0.3.1",
+        "markdown-toc": "^1.2.0",
+        "morgan": "^1.10.0",
+        "webpack": "^4.44.2",
+        "webpack-cli": "^3.3.12"
+    }
 }
-Original file line number
+Diff line change
@@ -1,4 +1,5 @@
     build
+    dist
     docs
     node_modules
     *.log
@@ Expand Down @@