From 16c2a333414e34f6abf4599862cc57961969bfbd Mon Sep 17 00:00:00 2001 From: cewald Date: Wed, 8 Apr 2020 09:38:37 +0000 Subject: [PATCH 1/6] Update to `storefront-query-builder` version `1.0.0` and add helper to import custom filters automatically * Also add a sample custom filter in `example-custom-filter` module --- .eslintignore | 1 + config/default.json | 1 + package.json | 2 +- src/api/catalog.ts | 4 ++- .../filter/catalog/SampleFilter.ts | 13 ++++++++ .../extensions/example-custom-filter/index.ts | 6 ++++ src/helpers/loadCustomFilters.ts | 32 +++++++++++++++++++ src/models/order.schema.js | 4 +-- 8 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 .eslintignore create mode 100644 src/api/extensions/example-custom-filter/filter/catalog/SampleFilter.ts create mode 100644 src/api/extensions/example-custom-filter/index.ts create mode 100644 src/helpers/loadCustomFilters.ts diff --git a/.eslintignore b/.eslintignore new file mode 100644 index 00000000..b512c09d --- /dev/null +++ b/.eslintignore @@ -0,0 +1 @@ +node_modules \ No newline at end of file diff --git a/config/default.json b/config/default.json index 6397f8c0..d2693a5f 100644 --- a/config/default.json +++ b/config/default.json @@ -246,6 +246,7 @@ "cms-data", "mail-service", "example-processor", + "example-custom-filter", "elastic-stock" ], "extensions": { diff --git a/package.json b/package.json index d94248d2..4370bb34 100644 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "resource-router-middleware": "^0.6.0", "sharp": "^0.23.4", "soap": "^0.25.0", - "storefront-query-builder": "^0.0.9", + "storefront-query-builder": "^1.0.0", "syswide-cas": "latest", "winston": "^2.4.2" }, diff --git a/src/api/catalog.ts b/src/api/catalog.ts index d489c2d2..270f844e 100755 --- a/src/api/catalog.ts +++ b/src/api/catalog.ts @@ -7,6 +7,7 @@ import { sha3_224 } from 'js-sha3' import AttributeService from './attribute/service' import bodybuilder from 'bodybuilder' import { elasticsearch, SearchQuery } from 'storefront-query-builder' +import loadCustomFilters from '../helpers/loadCustomFilters' function _cacheStorageHandler (config, result, hash, tags) { if (config.server.useOutputCache && cache) { @@ -54,7 +55,8 @@ export default ({config, db}) => async function (req, res, body) { } if (req.query.request_format === 'search-query') { // search query and not Elastic DSL - we need to translate it - requestBody = await elasticsearch.buildQueryBodyFromSearchQuery({ config, queryChain: bodybuilder(), searchQuery: new SearchQuery(requestBody) }) + const customFilters = await loadCustomFilters(config) + requestBody = await elasticsearch.buildQueryBodyFromSearchQuery({ config, queryChain: bodybuilder(), searchQuery: new SearchQuery(requestBody), customFilters }) } if (req.query.response_format) responseFormat = req.query.response_format diff --git a/src/api/extensions/example-custom-filter/filter/catalog/SampleFilter.ts b/src/api/extensions/example-custom-filter/filter/catalog/SampleFilter.ts new file mode 100644 index 00000000..b63b40f7 --- /dev/null +++ b/src/api/extensions/example-custom-filter/filter/catalog/SampleFilter.ts @@ -0,0 +1,13 @@ +import { FilterInterface } from 'storefront-query-builder' + +const filter: FilterInterface = { + priority: 1, + check: ({ operator, value, attribute, queryChain }) => attribute === 'custom-filter-name', + filter ({ value, attribute, operator, queryChain }) { + // Do you custom filter logic like: queryChain.filter('terms', attribute, value) + return queryChain + }, + mutator: (value) => typeof value !== 'object' ? { 'in': [value] } : value +} + +export default filter diff --git a/src/api/extensions/example-custom-filter/index.ts b/src/api/extensions/example-custom-filter/index.ts new file mode 100644 index 00000000..d2367a25 --- /dev/null +++ b/src/api/extensions/example-custom-filter/index.ts @@ -0,0 +1,6 @@ +import { Router } from 'express' + +module.exports = () => { + let exampleFilter = Router() + return exampleFilter +} diff --git a/src/helpers/loadCustomFilters.ts b/src/helpers/loadCustomFilters.ts new file mode 100644 index 00000000..94fcf507 --- /dev/null +++ b/src/helpers/loadCustomFilters.ts @@ -0,0 +1,32 @@ +import { promises as fs } from 'fs' +import path from 'path' + +function getModuleFilterPaths (moduleName: string, type: string): Promise { + const dirPath = path.resolve(__dirname, '../../' + moduleName + '/filter/', type) + return fs.readdir(dirPath) + .then(files => files.filter(f => f.endsWith('Filter.ts')).map(file => ({ path: path.resolve(dirPath, file), file }))) + .catch(() => { + return [] + }) +} + +export default async function loadModuleCustomFilters (config: Record, type: string = 'catalog'): Promise { + let filters: any = {} + let filterPromises: Promise[] = [] + + for (const ext in config.registeredExtensions) { + filterPromises.push( + getModuleFilterPaths(config.registeredExtensions[ext], type) + .then(filePaths => filePaths.forEach(o => import(o.path) + .then(module => { + filters[o.file] = module.default + }) + .catch(e => { + console.log(e) + })) + ) + ) + } + + return Promise.all(filterPromises).then((e) => filters) +} diff --git a/src/models/order.schema.js b/src/models/order.schema.js index 65b2cb2d..aa0c26fe 100644 --- a/src/models/order.schema.js +++ b/src/models/order.schema.js @@ -142,7 +142,7 @@ exports.default = { type: 'number' }, save_address: { - type: 'number' + type: 'number' } } }, @@ -208,7 +208,7 @@ exports.default = { type: 'number' }, save_address: { - type: 'number' + type: 'number' } } } From 8ff13c0bdba4eeaafc136eee549c618a5b5a48b7 Mon Sep 17 00:00:00 2001 From: cewald Date: Wed, 8 Apr 2020 10:02:56 +0000 Subject: [PATCH 2/6] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55eea6d6..c8c00acd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Endpoint for reset password with reset token. Only for Magento 2 - @Fifciu - Varnish Cache with autoinvalidation by Cache tags as addon - @Fifciu - Add `resetPasswordUsingResetToken` to `magento1` platform - @cewald (#415) +- Update to `storefront-query-builder` version `1.0.0` - @cewald (#429) ### Fixed - add es7 support for map url module and fixed default index for es config - @gibkigonzo From 60780a23663c5364e859d5d73d9cd5f8207cab3d Mon Sep 17 00:00:00 2001 From: cewald Date: Wed, 8 Apr 2020 13:23:24 +0000 Subject: [PATCH 3/6] Make catalog-filter loader less IO-heavy --- config/default.json | 3 +++ src/helpers/loadCustomFilters.ts | 37 ++++++++++++++------------------ 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/config/default.json b/config/default.json index d2693a5f..d9fe9def 100644 --- a/config/default.json +++ b/config/default.json @@ -271,6 +271,9 @@ "resultProcessors": { "product": "my-product-processor" } + }, + "example-custom-filter": { + "catalogFilter": [ "SampleFilter" ] } }, "magento2": { diff --git a/src/helpers/loadCustomFilters.ts b/src/helpers/loadCustomFilters.ts index 94fcf507..2175098f 100644 --- a/src/helpers/loadCustomFilters.ts +++ b/src/helpers/loadCustomFilters.ts @@ -1,31 +1,26 @@ -import { promises as fs } from 'fs' import path from 'path' -function getModuleFilterPaths (moduleName: string, type: string): Promise { - const dirPath = path.resolve(__dirname, '../../' + moduleName + '/filter/', type) - return fs.readdir(dirPath) - .then(files => files.filter(f => f.endsWith('Filter.ts')).map(file => ({ path: path.resolve(dirPath, file), file }))) - .catch(() => { - return [] - }) -} - export default async function loadModuleCustomFilters (config: Record, type: string = 'catalog'): Promise { let filters: any = {} let filterPromises: Promise[] = [] - for (const ext in config.registeredExtensions) { - filterPromises.push( - getModuleFilterPaths(config.registeredExtensions[ext], type) - .then(filePaths => filePaths.forEach(o => import(o.path) - .then(module => { - filters[o.file] = module.default - }) - .catch(e => { - console.log(e) - })) + for (const mod of config.registeredExtensions) { + if (config.extensions.hasOwnProperty(mod) && config.extensions[mod].hasOwnProperty(type + 'Filter') && Array.isArray(config.extensions[mod][type + 'Filter'])) { + const moduleFilter = config.extensions[mod][type + 'Filter'] + const dirPath = [__dirname, '../api/extensions/' + mod + '/filter/', type] + for (const filterName of moduleFilter) { + const filePath = path.resolve(...dirPath, filterName + '.ts') + filterPromises.push( + import(filePath) + .then(module => { + filters[filterName] = module.default + }) + .catch(e => { + console.log(e) + }) ) - ) + } + } } return Promise.all(filterPromises).then((e) => filters) From ad158294bae94de4759f358af23d0a2b5623bff2 Mon Sep 17 00:00:00 2001 From: cewald Date: Wed, 8 Apr 2020 13:30:33 +0000 Subject: [PATCH 4/6] Update `bodybuilder` package to be able to use the `clone` method --- package.json | 2 +- yarn.lock | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 4370bb34..92104b7d 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,7 @@ "ajv": "^6.4.0", "ajv-keywords": "^3.4.0", "body-parser": "^1.18.2", - "bodybuilder": "2.2.13", + "bodybuilder": "2.2.21", "commander": "^2.19.0", "compression": "^1.7.2", "config": "^1.30.0", diff --git a/yarn.lock b/yarn.lock index d261f5ab..8b566e2f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -999,11 +999,12 @@ body-parser@1.19.0, body-parser@^1.12.2, body-parser@^1.17.1, body-parser@^1.18. raw-body "2.4.0" type-is "~1.6.17" -bodybuilder@2.2.13: - version "2.2.13" - resolved "https://registry.yarnpkg.com/bodybuilder/-/bodybuilder-2.2.13.tgz#82ced1c2144bd9ddce97c5d89fa13f7e24c40f8e" +bodybuilder@2.2.21: + version "2.2.21" + resolved "https://registry.yarnpkg.com/bodybuilder/-/bodybuilder-2.2.21.tgz#1a7a5d31189cf10a6fbd87668ac8aa386ac8cc38" + integrity sha512-Ys0Cas7ri7dcv3f62HwB2Lclk2oBCv86pakuEs4liK6hhL9ikTEairZ700k4VHD7vgvr2CW2Kogh4E5Ivmkg2A== dependencies: - lodash "^4.9.0" + lodash "^4.17.11" boxen@^1.2.1: version "1.3.0" @@ -3990,7 +3991,7 @@ lodash.unescape@4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/lodash.unescape/-/lodash.unescape-4.0.1.tgz#bf2249886ce514cda112fae9218cdc065211fc9c" -lodash@^4.0.0, lodash@^4.17.10, lodash@^4.17.11, lodash@^4.17.14, lodash@^4.17.4, lodash@^4.17.5, lodash@^4.9.0: +lodash@^4.0.0, lodash@^4.17.10, lodash@^4.17.11, lodash@^4.17.14, lodash@^4.17.4, lodash@^4.17.5: version "4.17.14" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.14.tgz#9ce487ae66c96254fe20b599f21b6816028078ba" From 23a992ccda4328fbe642e9bd8882c937694e8ca8 Mon Sep 17 00:00:00 2001 From: cewald Date: Thu, 9 Apr 2020 11:49:38 +0000 Subject: [PATCH 5/6] Update `yarn.lock` --- yarn.lock | 82 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 78 insertions(+), 4 deletions(-) diff --git a/yarn.lock b/yarn.lock index 8b566e2f..4c36389f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -505,6 +505,16 @@ "@typescript-eslint/typescript-estree" "1.13.0" eslint-scope "^4.0.0" +"@typescript-eslint/experimental-utils@2.27.0": + version "2.27.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/experimental-utils/-/experimental-utils-2.27.0.tgz#801a952c10b58e486c9a0b36cf21e2aab1e9e01a" + integrity sha512-vOsYzjwJlY6E0NJRXPTeCGqjv5OHgRU1kzxHKWJVPjDYGbPgLudBXjIlc+OD1hDBZ4l1DLbOc5VjofKahsu9Jw== + dependencies: + "@types/json-schema" "^7.0.3" + "@typescript-eslint/typescript-estree" "2.27.0" + eslint-scope "^5.0.0" + eslint-utils "^2.0.0" + "@typescript-eslint/parser@^1.7.1-alpha.17": version "1.13.0" resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-1.13.0.tgz#61ac7811ea52791c47dc9fd4dd4a184fae9ac355" @@ -514,6 +524,16 @@ "@typescript-eslint/typescript-estree" "1.13.0" eslint-visitor-keys "^1.0.0" +"@typescript-eslint/parser@^2.26.0": + version "2.27.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-2.27.0.tgz#d91664335b2c46584294e42eb4ff35838c427287" + integrity sha512-HFUXZY+EdwrJXZo31DW4IS1ujQW3krzlRjBrFRrJcMDh0zCu107/nRfhk/uBasO8m0NVDbBF5WZKcIUMRO7vPg== + dependencies: + "@types/eslint-visitor-keys" "^1.0.0" + "@typescript-eslint/experimental-utils" "2.27.0" + "@typescript-eslint/typescript-estree" "2.27.0" + eslint-visitor-keys "^1.1.0" + "@typescript-eslint/typescript-estree@1.13.0": version "1.13.0" resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-1.13.0.tgz#8140f17d0f60c03619798f1d628b8434913dc32e" @@ -521,6 +541,19 @@ lodash.unescape "4.0.1" semver "5.5.0" +"@typescript-eslint/typescript-estree@2.27.0": + version "2.27.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-2.27.0.tgz#a288e54605412da8b81f1660b56c8b2e42966ce8" + integrity sha512-t2miCCJIb/FU8yArjAvxllxbTiyNqaXJag7UOpB5DVoM3+xnjeOngtqlJkLRnMtzaRcJhe3CIR9RmL40omubhg== + dependencies: + debug "^4.1.1" + eslint-visitor-keys "^1.1.0" + glob "^7.1.6" + is-glob "^4.0.1" + lodash "^4.17.15" + semver "^6.3.0" + tsutils "^3.17.1" + JSONStream@^1.3.5: version "1.3.5" resolved "https://registry.yarnpkg.com/JSONStream/-/JSONStream-1.3.5.tgz#3208c1f08d3a4d99261ab64f92302bc15e111ca0" @@ -2021,16 +2054,36 @@ eslint-scope@^4.0.0, eslint-scope@^4.0.3: esrecurse "^4.1.0" estraverse "^4.1.1" +eslint-scope@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-5.0.0.tgz#e87c8887c73e8d1ec84f1ca591645c358bfc8fb9" + integrity sha512-oYrhJW7S0bxAFDvWqzvMPRm6pcgcnWc4QnofCAqRTRfQC0JcwenzGglTtsLyIuuWFfkqDG9vz67cnttSd53djw== + dependencies: + esrecurse "^4.1.0" + estraverse "^4.1.1" + eslint-utils@^1.3.1: version "1.4.0" resolved "https://registry.yarnpkg.com/eslint-utils/-/eslint-utils-1.4.0.tgz#e2c3c8dba768425f897cf0f9e51fe2e241485d4c" dependencies: eslint-visitor-keys "^1.0.0" +eslint-utils@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/eslint-utils/-/eslint-utils-2.0.0.tgz#7be1cc70f27a72a76cd14aa698bcabed6890e1cd" + integrity sha512-0HCPuJv+7Wv1bACm8y5/ECVfYdfsAm9xmVb7saeFlxjPYALefjhbYoCkBjPdPzGH8wWyTpAez82Fh3VKYEZ8OA== + dependencies: + eslint-visitor-keys "^1.1.0" + eslint-visitor-keys@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-1.0.0.tgz#3f3180fb2e291017716acb4c9d6d5b5c34a6a81d" +eslint-visitor-keys@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-1.1.0.tgz#e2a82cea84ff246ad6fb57f9bde5b46621459ec2" + integrity sha512-8y9YjtM1JBJU/A9Kc+SbaOV4y29sSWckBwMHa+FGtVj5gN/sbnKDf6xJUl+8g7FAij9LVaP8C24DUiH/f/2Z9A== + eslint@^5.0.0: version "5.16.0" resolved "https://registry.yarnpkg.com/eslint/-/eslint-5.16.0.tgz#a1e3ac1aae4a3fbd8296fcf8f7ab7314cbb6abea" @@ -2624,6 +2677,18 @@ glob@^7.0.0, glob@^7.0.5, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3: once "^1.3.0" path-is-absolute "^1.0.0" +glob@^7.1.6: + version "7.1.6" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.6.tgz#141f33b81a7c2492e125594307480c46679278a6" + integrity sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA== + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^3.0.4" + once "^1.3.0" + path-is-absolute "^1.0.0" + global-dirs@^0.1.0: version "0.1.1" resolved "https://registry.yarnpkg.com/global-dirs/-/global-dirs-0.1.1.tgz#b319c0dd4607f353f3be9cca4c72fc148c49f445" @@ -3133,7 +3198,7 @@ is-glob@^3.1.0: dependencies: is-extglob "^2.1.0" -is-glob@^4.0.0: +is-glob@^4.0.0, is-glob@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.1.tgz#7567dbe9f2f5e2467bc77ab83c4a29482407a5dc" dependencies: @@ -6106,10 +6171,12 @@ stealthy-require@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/stealthy-require/-/stealthy-require-1.1.1.tgz#35b09875b4ff49f26a777e509b3090a3226bf24b" -"storefront-query-builder@https://github.com/DivanteLtd/storefront-query-builder.git": - version "0.0.8" - resolved "https://github.com/DivanteLtd/storefront-query-builder.git#26d17c9ae043b54725032687f116b21e83e02a77" +storefront-query-builder@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/storefront-query-builder/-/storefront-query-builder-1.0.0.tgz#8bab0b6bd61a574dfa913cad5c2fdd17858f8b07" + integrity sha512-j0JhHOYhcfDcsBUMYWVJ0UApQDOem5zo20XikxJcxfFEcM5Et1Z16674wx3++KrWCbU5raEMgHvqEPxkDFajdw== dependencies: + "@typescript-eslint/parser" "^2.26.0" clone-deep "^4.0.1" stream-events@^1.0.1, stream-events@^1.0.4: @@ -6469,6 +6536,13 @@ tslib@^1.8.1, tslib@^1.9.0, tslib@^1.9.3: version "1.10.0" resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.10.0.tgz#c3c19f95973fb0a62973fb09d90d961ee43e5c8a" +tsutils@^3.17.1: + version "3.17.1" + resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.17.1.tgz#ed719917f11ca0dee586272b2ac49e015a2dd759" + integrity sha512-kzeQ5B8H3w60nFY2g8cJIuH7JDpsALXySGtwGJ0p2LSjLgay3NdIpqq5SoOBe46bKDW2iq25irHCr8wjomUS2g== + dependencies: + tslib "^1.8.1" + tsutils@^3.7.0: version "3.14.1" resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.14.1.tgz#f1d2b93d2a0876481f2f1f98c25ba42bbd7ee860" From f049f130e3f3e0de76b3decfa855851961043fa5 Mon Sep 17 00:00:00 2001 From: cewald Date: Thu, 9 Apr 2020 12:33:15 +0000 Subject: [PATCH 6/6] Don't import custom filter classes using file ending * This leads to an exception in production-mode because the file couldn't be found after it's parsed into JS --- src/helpers/loadCustomFilters.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/helpers/loadCustomFilters.ts b/src/helpers/loadCustomFilters.ts index 2175098f..c69d1bd5 100644 --- a/src/helpers/loadCustomFilters.ts +++ b/src/helpers/loadCustomFilters.ts @@ -9,7 +9,7 @@ export default async function loadModuleCustomFilters (config: Record {