refactor: split into multiple packages + TS rewrite

Packages:
- `@apify/consts`
- `@apify/datastructures`
- `@apify/git`
- `@apify/hubspot_client`
- `@apify/input_schema`
- `@apify/log`
- `@apify/markdown`
- `@apify/salesforce_client`
- `@apify/utilities`

Other notable changes:
- uses named exports except `log` where we keep the default export for logger instance
- removes `underscore` from most of the packages (except salesforce/hubspot clients)
- removed `startsWith` polyfill and `newPromise` method
- `truncate` method is duplicated so the `log` package does not depend on `utilities`
  (as those bring a lot of 3rd party deps)

TODO:
- [ ] review the split into packages from dependency point of view
- [ ] rewrite CI publishing via lerna (it will be broken now)

Closes #131

Author: B4nan

.github/workflows/check.yml

@@ -13,8 +13,9 @@ jobs:
         runs-on: ubuntu-latest
 
         strategy:
+            fail-fast: false
             matrix:
-                node-version: [10, 12, 14]
+                node-version: [12, 14, 15, 16]
 
         steps:
             -
@@ -33,6 +34,9 @@ jobs:
                         node_modules
                         build
                     key: cache-${{ github.run_id }}-v14
+            -
+                name: Upgrade NPM to have workspaces support
+                run: npm i -G npm@latest
             -
                 name: Install Dependencies
                 run: npm install

.github/workflows/release.yml

@@ -1,3 +1,4 @@
+# TODO needs lerna rewrite
 name: Check & Release
 
 on:
@@ -18,7 +19,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [10, 12, 14]
+        node-version: [12, 14, 15, 16]
 
     steps:
       -
@@ -37,6 +38,9 @@ jobs:
             node_modules
             build
           key: cache-${{ github.run_id }}-v14
+      -
+          name: Upgrade NPM to have workspaces support
+          run: npm i -G npm@latest
       -
         name: Install Dependencies
         run: npm install

.gitignore

@@ -1,4 +1,4 @@
-build
+dist
 docs
 node_modules
 *.log
@@ -12,4 +12,3 @@ pids
 .idea
 yarn.lock
 package-lock.json
-types/

lerna.json

@@ -0,0 +1,7 @@
+{
+  "packages": [
+    "packages/*"
+  ],
+  "useWorkspaces": true,
+  "version": "0.7.7"
+}

package.json

@@ -1,5 +1,6 @@
 {
     "name": "apify-shared",
+    "private": true,
     "version": "0.7.7",
     "description": "Tools and constants shared across Apify projects.",
     "main": "build/index.js",
@@ -10,71 +11,70 @@
     "author": {
         "name": "Apify",
         "email": "support@apify.com",
-        "url": "https://www.apify.com"
+        "url": "https://apify.com"
     },
     "contributors": [
         "Jan Curn <jan@apify.com>",
         "Marek Trunkát <marek@apify.com>"
     ],
     "license": "Apache-2.0",
+    "workspaces": [
+        "packages/*"
+    ],
     "repository": {
         "type": "git",
-        "url": "git+https://github.com/apifytech/apify-shared-js"
+        "url": "git+https://github.com/apify/apify-shared-js"
     },
     "bugs": {
-        "url": "https://github.com/apifytech/apify-shared-js/issues"
+        "url": "https://github.com/apify/apify-shared-js/issues"
     },
     "homepage": "https://apify.com",
     "scripts": {
-        "build": "rm -rf build types && babel src --out-dir build && cp src/*.json build && tsc ./src/*.js --allowJs --declaration --emitDeclarationOnly --declarationDir build --skipLibCheck",
-        "build-doc": "npm run clean && npm run build && node ./node_modules/jsdoc/jsdoc.js --package ./package.json -c ./jsdoc/conf.json -d docs",
-        "build-local-dev": "npm run build && cp package.json build && pushd build/ && npm i && popd",
-        "test": "npm run build && mocha --timeout 5000 --require @babel/register --recursive",
-        "test-cov": "npm run build && babel-node node_modules/isparta/bin/isparta cover --report text --report html node_modules/mocha/bin/_mocha -- --reporter dot",
+        "clean": "lerna run clean",
+        "build": "lerna run build",
+        "test": "jest --silent",
+        "test-cov": "jest --coverage",
         "prepublishOnly": "(test $CI || (echo \"Publishing is reserved to CI!\"; exit 1))",
-        "clean": "rm -rf build docs types",
         "lint": "npm run build && eslint src test",
         "lint:fix": "eslint src test --fix"
     },
-    "dependencies": {
-        "@hubspot/api-client": "^3.4.1",
-        "axios": "^0.21.1",
-        "chalk": "^4.0.0",
-        "cherow": "^1.6.9",
-        "clone": "^2.1.1",
-        "countries-list": "^2.6.1",
-        "create-hmac": "^1.1.7",
-        "git-url-parse": "^11.4.4",
-        "is-buffer": "^2.0.5",
-        "marked": "^2.0.0",
-        "match-all": "^1.2.6",
-        "moment": "^2.29.1",
-        "request": "^2.88.0",
-        "underscore": "^1.11.0",
-        "url": "^0.11.0"
+    "jest": {
+        "testTimeout": 30000,
+        "preset": "ts-jest",
+        "testEnvironment": "node",
+        "collectCoverage": false,
+        "collectCoverageFrom": [
+            "<rootDir>/packages/*/src/**/*.ts"
+        ],
+        "moduleNameMapper": {
+            "@apify/(.*)": "<rootDir>/packages/$1/src"
+        },
+        "modulePathIgnorePatterns": [
+            "dist/package.json",
+            "<rootDir>/package.json"
+        ],
+        "globals": {
+            "ts-jest": {
+                "tsconfig": "test/tsconfig.json"
+            }
+        }
     },
     "devDependencies": {
         "@apify/eslint-config": "^0.1.3",
-        "@babel/cli": "^7.12.13",
-        "@babel/core": "^7.12.13",
-        "@babel/plugin-proposal-object-rest-spread": "^7.12.13",
-        "@babel/preset-env": "^7.12.13",
-        "@babel/register": "^7.12.13",
-        "ajv": "^6.12.6",
-        "apify-jsdoc-template": "github:apifytech/apify-jsdoc-template",
-        "babel-plugin-add-module-exports": "^1.0.4",
-        "chai": "^4.2.0",
-        "chai-as-promised": "^7.1.1",
+        "@types/git-url-parse": "^9.0.0",
+        "@types/jest": "^26.0.23",
+        "@types/marked": "^2.0.2",
+        "@types/node": "^14.14.41",
+        "@types/underscore": "^1.11.2",
         "deploy-web-to-s3": "^1.3.0",
         "eslint": "^7.19.0",
-        "isparta": "^4.1.1",
-        "istanbul": "0.4.5",
-        "jsdoc": "^3.6.6",
-        "jsdoc-export-default-interop": "^0.3.1",
-        "mocha": "^7.2.0",
+        "jest": "^26.6.3",
+        "lerna": "^4.0.0",
         "nock": "^13.0.7",
-        "sinon": "^9.2.4",
         "strip-ansi": "^6.0.0",
-        "typescript": "^4.2.3"
+        "ts-jest": "^26.5.5",
+        "ts-node": "^9.1.1",
+        "typescript": "^4.2.4",
+        "underscore": "^1.13.1"
     }
 }

packages/consts/package.json

@@ -0,0 +1,49 @@
+{
+    "name": "@apify/consts",
+    "version": "1.0.0",
+    "description": "Tools and constants shared across Apify projects.",
+    "main": "dist/index.js",
+    "module": "dist/index.mjs",
+    "typings": "dist/index.d.ts",
+    "exports": {
+        ".": {
+            "import": "./dist/index.mjs",
+            "require": "./dist/index.js"
+        }
+    },
+    "keywords": [
+        "apify"
+    ],
+    "author": {
+        "name": "Apify",
+        "email": "support@apify.com",
+        "url": "https://apify.com"
+    },
+    "contributors": [
+        "Jan Curn <jan@apify.com>",
+        "Marek Trunkát <marek@apify.com>"
+    ],
+    "license": "Apache-2.0",
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/apify/apify-shared-js"
+    },
+    "bugs": {
+        "url": "https://github.com/apify/apify-shared-js/issues"
+    },
+    "homepage": "https://apify.com",
+    "scripts": {
+        "build": "npm run clean && npm run compile && npm run copy",
+        "postbuild": "yarn gen-esm-wrapper dist/index.js dist/index.mjs",
+        "clean": "rimraf ./dist",
+        "compile": "tsc -p tsconfig.build.json",
+        "copy": "ts-node -T ../../scripts/copy.ts"
+    },
+    "publishConfig": {
+        "access": "public"
+    },
+    "dependencies": {},
+    "devDependencies": {
+        "gen-esm-wrapper": "^1.1.1"
+    }
+}

src/consts.js renamed to packages/consts/src/consts.ts

@@ -376,6 +376,7 @@ export const ENV_VARS = {
     FACT: 'APIFY_FACT',
     DEDICATED_CPUS: 'APIFY_DEDICATED_CPUS',
     SDK_LATEST_VERSION: 'APIFY_SDK_LATEST_VERSION',
+    INTERNAL_PORT: 'APIFY_INTERNAL_PORT',
 
     // Deprecated, keep them for backward compatibility:
     ACT_ID: 'APIFY_ACT_ID',

packages/consts/src/index.ts

@@ -0,0 +1,2 @@
+export * from './consts';
+export * from './regexs';

src/regexs.js renamed to packages/consts/src/regexs.ts

@@ -52,7 +52,7 @@ export const TWITTER_REGEX = /^@[a-z0-9_]{1,15}$/i;
 export const GITHUB_REGEX = new RegExp(`^${GITHUB_REGEX_STR}$`, 'i');
 
 // @TODO: Discontinue usage of this regexps, in favor of HTTP_URL_REGEX
-export const URL_REGEX = /^(http:\/\/|https:\/\/)/i;
+export const URL_REGEX = /^https?:\/\//i;
 
 // Inspired by https://gist.github.com/dperini/729294, but doesn't match FTP URLs
 /* eslint-disable */

packages/consts/tsconfig.build.json

@@ -0,0 +1,7 @@
+{
+  "extends": "../../tsconfig.build.json",
+  "compilerOptions": {
+    "outDir": "./dist"
+  },
+  "include": ["src/**/*"]
+}