Skip to content
This repository has been archived by the owner on Jun 5, 2024. It is now read-only.

Commit

Permalink
Merge pull request #96 from DivanteLtd/feature/es7
Browse files Browse the repository at this point in the history
Elastic 7 support
  • Loading branch information
andrzejewsky authored Feb 19, 2020
2 parents 00715de + 2b427e7 commit e2578d0
Show file tree
Hide file tree
Showing 5 changed files with 88 additions and 29 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [1.11] - UNRELEASED
### Added
- Elastic7 support - @pkarw (#96)

## [1.10] - 2019.07.10
### Added
Expand Down
22 changes: 22 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,28 @@ npm run dev

The key command is `docker-compose up -d` which runs the ElasticSearch and Redis instances - both required by `mage2vuestorefront`

### Elastic 7 Support

By default, Vue Storefront API docker files and config are based on Elastic 5.6. We plan to change the default Elastic version to 7 with the 1.11 stable release. As for now, the [Elastic 7 support](https://github.com/DivanteLtd/vue-storefront-api/pull/342) is marked as **experimental**.

In order to index data to Elastic 7 please make sure you set the proper `apiVersion` in the `config.js`:

```js
elasticsearch: {
apiVersion: process.env.ELASTICSEARCH_API_VERSION || '7.1'
},
```

or just use the env variable:

```bash
export ELASTICSEARCH_API_VERSION=7.1
```

Starting from [Elasitc 6 and 7](https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking-changes-7.0.html) we can have **just single** document type per single index. Vue Storefront used to have `product`, `category` ... types defined in the `vue_storefront_catalog`.

From now on, we're using the separate indexes per each entity type. The convention is: `${indexName}_${entityType}`. If your' **logical index name** is `vue_storefront_catalog` then it will be mapped to the **physical indexes** of: `vue_storefront_catalog_product`, `vue_storefront_catalog_category` ...

### Initial Vue Storefront import

Now, You're ready to run the importer. Please check the [config file](https://github.com/DivanteLtd/mage2vuestorefront/blob/master/src/config.js). You may setup the Magento access data and URLs by config values or ENV variables.
Expand Down
4 changes: 2 additions & 2 deletions docker-compose.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
version: '2'
services:
esm1:
image: elasticsearch:2.4.6
image: elasticsearch:7.3.2
container_name: esm1
environment:
- cluster.name=docker-cluster
Expand All @@ -19,7 +19,7 @@ services:
networks:
- esnet
esm2:
image: elasticsearch:5.5
image: elasticsearch:7.3.2
container_name: esm2
environment:
- cluster.name=docker-cluster
Expand Down
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,10 @@
"test": "echo \"Error: no test specified\" && exit 1"
},
"dependencies": {
"@elastic/elasticsearch": "^7.3.0",
"agentkeepalive": "^3.3.0",
"body-parser": "^1.17.1",
"commander": "^2.18.0",
"elasticsearch": "^15.2.0",
"elasticsearch-deletebyquery": "^1.6.0",
"express": "^4.15.2",
"jsonfile": "^4.0.0",
Expand Down
87 changes: 61 additions & 26 deletions src/adapters/nosql/elasticsearch.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
'use strict';
const AbstractNosqlAdapter = require('./abstract');
const elasticsearch = require('elasticsearch');
const elasticsearch = require('@elastic/elasticsearch');
const AgentKeepAlive = require('agentkeepalive');
const AgentKeepAliveHttps = require('agentkeepalive').HttpsAgent;

Expand Down Expand Up @@ -28,6 +28,32 @@ class ElasticsearchAdapter extends AbstractNosqlAdapter {
this.updateDocument.bind(this);
}

/**
* Get physical Elastic index name; since 7.x we're adding an entity name to get real index: vue_storefront_catalog_product, vue_storefront_catalog_category and so on
* @param {*} baseIndexName
* @param {*} config
*/
getPhysicalIndexName(collectionName, config) {
if (parseInt(config.elasticsearch.apiVersion) >= 6) {
return `${config.db.indexName}_${collectionName}`
} else {
return config.db.indexName
}
}

/**
* Get physical Elastic type name; since 7.x index can have one type _doc
* @param {*} baseIndexName
* @param {*} config
*/
getPhysicalTypeName(collectionName, config) {
if (parseInt(config.elasticsearch.apiVersion) >= 6) {
return `_doc`
} else {
return collectionName
}
}

/**
* Close the nosql database connection - abstract to the driver
*/
Expand All @@ -42,11 +68,14 @@ class ElasticsearchAdapter extends AbstractNosqlAdapter {
*/
getDocuments(collectionName, queryBody) {
return new Promise((resolve, reject) => {
this.db.search({ // requires ES 5.5
index: this.config.db.indexName,
type: collectionName,
body: queryBody
}, function (error, response) {
const searchQueryBody = {
index: this.getPhysicalIndexName(collectionName, this.config),
body: queryBody
}
if (parseInt(this.config.elasticsearch.apiVersion) < 6)
searchQueryBody.type = this.getPhysicalTypeName(collectionName, this.config)

this.db.search(searchQueryBody, function (error, { body: response }) {
if (error) reject(error);
if (response.hits && response.hits.hits) {
resolve(response.hits.hits.map(h => h._source))
Expand All @@ -62,20 +91,21 @@ class ElasticsearchAdapter extends AbstractNosqlAdapter {
* @param {object} item document to be updated in database
*/
updateDocument(collectionName, item) {

const itemtbu = item;

this.db.update({
index: this.config.db.indexName,
const updateRequestBody = {
index: this.getPhysicalIndexName(collectionName, this.config),
id: item.id,
type: collectionName,
body: {
// put the partial document under the `doc` key
upsert: itemtbu,
doc: itemtbu

}
}, function (error, response) {
}
if (parseInt(this.config.elasticsearch.apiVersion) < 6)
updateRequestBody.type = this.getPhysicalTypeName(collectionName, this.config)

this.db.update(updateRequestBody, function (error, response) {
if (error)
throw new Error(error);
});
Expand All @@ -89,11 +119,10 @@ class ElasticsearchAdapter extends AbstractNosqlAdapter {
cleanupByTransactionkey(collectionName, transactionKey) {

if (transactionKey) {
this.db.deleteByQuery({ // requires ES 5.5
index: this.config.db.indexName,
const query = {
index: this.getPhysicalIndexName(collectionName, this.config),
conflicts: 'proceed',
type: collectionName,
body: {
body: {
query: {
bool: {
must_not: {
Expand All @@ -102,9 +131,12 @@ class ElasticsearchAdapter extends AbstractNosqlAdapter {
}
}
}
}, function (error, response) {
};
if (parseInt(this.config.elasticsearch.apiVersion) < 6)
query.type = this.getPhysicalTypeName(collectionName, this.config)

this.db.deleteByQuery(query, function (error, response) {
if (error) throw new Error(error);
logger.info(response);
});
}
}
Expand All @@ -120,12 +152,15 @@ class ElasticsearchAdapter extends AbstractNosqlAdapter {
let bulkSize = 500;

for (let doc of items) {
const query = {
_index: this.getPhysicalIndexName(collectionName, this.config),
_id: doc.id,
};
if (parseInt(this.config.elasticsearch.apiVersion) < 6)
query.type = this.getPhysicalTypeName(collectionName, this.config)

requests.push({
update: {
_index: this.config.db.indexName,
_id: doc.id,
_type: collectionName,
}
update: query
});

requests.push({
Expand Down Expand Up @@ -161,15 +196,15 @@ class ElasticsearchAdapter extends AbstractNosqlAdapter {

if (!global.es) {
this.db = new elasticsearch.Client({
host: this.config.db.url,
log: 'error',
node: this.config.db.url,
log: 'debug',
apiVersion: this.config.elasticsearch.apiVersion,

maxRetries: 10,
keepAlive: true,
maxSockets: 10,
minSockets: 10,
requestTimeout: 1800000,
requestTimeout: 1800000,

createNodeAgent: function (connection, config) {
if (connection.useSsl) {
Expand Down

0 comments on commit e2578d0

Please sign in to comment.