feat(ref-imp): added support for MongoDB 4.0 based cloud storage serv…

…ices

docs/release-process.md

@@ -25,11 +25,19 @@ To create a stable release follow the following steps:
 1. Build the package `npm build`
 1. Test the package `npm test`
 1. Run `npm run version:release`, with an appropriate option such as [ `major` | `minor` | `patch` ].
-1. Observe the correctly incremented change to the `package.json` and changes to `CHANGELOG.md`
-1. Push the release branch including the newly created tags `git push origin release --tags`
-1. Open a pull request for the release, once approvals have been sought, merge the pull request using **rebase**,
+1. Observe and note down the correctly incremented version number X.Y.Z change to the `package.json` and changes to `CHANGELOG.md`
+1. Push the release branch and open a pull request for the release.
+1. Once approvals have been sought, merge the pull request using **rebase**,
     preserving the commit message as `release commit [skip ci]`
-1. Observe the triggering of the `/.github/workflows/push-release.yml` github workflow
+1. Observe the triggering of the `/.github/workflows/release.yml` github workflow
+1. Remove the tag created in the release branch: `git tag -d vX.Y.Z`
+1. Remove the local release branch:
+   1. `git checkout master`
+   1. `git branch -D release`
+1. Push a new version tag to remote master e.g. (v1.0.1):
+   1. git pull
+   1. git tag vX.Y.Z
+   1. git push origin vX.Y.Z
 
 **Note** It is important that **rebase** is used as the strategy for merging a release pull request as this preserves the created release tag.
 

lib/bitcoin/BitcoinProcessor.ts

@@ -209,7 +209,7 @@ export default class BitcoinProcessor {
     const currentServiceVersion = currentServiceVersionModel.version;
 
     const savedServiceState = await this.serviceStateStore.get();
-    const savedServiceVersion = savedServiceState?.serviceVersion;
+    const savedServiceVersion = savedServiceState.serviceVersion;
 
     if (savedServiceVersion === currentServiceVersion) {
       return;

lib/bitcoin/MongoDbBlockMetadataStore.ts

@@ -1,5 +1,5 @@
-import { Collection, Cursor } from 'mongodb';
 import BlockMetadata from './models/BlockMetadata';
+import { Cursor } from 'mongodb';
 import IBlockMetadataStore from './interfaces/IBlockMetadataStore';
 import MongoDbStore from '../common/MongoDbStore';
 
@@ -20,9 +20,9 @@ export default class MongoDbBlockMetadataStore extends MongoDbStore implements I
     super(serverUrl, MongoDbBlockMetadataStore.collectionName, databaseName);
   }
 
-  protected async createIndex (collection: Collection) {
+  public async createIndex () {
     // Create unique index, so duplicate inserts are rejected.
-    await collection.createIndex({ height: 1 }, { unique: true });
+    await this.collection.createIndex({ height: 1 }, { unique: true });
   }
 
   public async add (arrayOfBlockMetadata: BlockMetadata[]): Promise<void> {

lib/bitcoin/interfaces/IServiceStateStore.ts

@@ -11,5 +11,5 @@ export default interface IServiceStateStore<T> {
   /**
    * Gets the service state.
    */
-  get (): Promise<T | undefined>;
+  get (): Promise<T>;
 }

lib/common/MongoDbServiceStateStore.ts

@@ -19,14 +19,10 @@ export default class MongoDbServiceStateStore<T> extends MongoDbStore implements
     await this.collection!.replaceOne({ }, serviceState, { upsert: true }); // { } filter results in replacement of the first document returned.
   }
 
-  public async get (): Promise<T | undefined> {
+  public async get (): Promise<T> {
     const queryOptions = { fields: { _id: 0 } }; // Exclude `_id` field from being returned.
     const serviceState = await this.collection!.findOne({ }, queryOptions);
 
-    if (serviceState === null) {
-      return undefined;
-    } else {
-      return serviceState;
-    }
+    return serviceState;
   }
 }

lib/common/MongoDbStore.ts

@@ -45,25 +45,22 @@ export default class MongoDbStore {
     const collectionNames = collections.map(collection => collection.collectionName);
 
     // If collection exists, use it; else create it.
-    let collection;
     if (collectionNames.includes(this.collectionName)) {
       Logger.info(`Collection '${this.collectionName}' found.`);
-      collection = db.collection(this.collectionName);
+      this.collection = db.collection(this.collectionName);
     } else {
       Logger.info(`Collection '${this.collectionName}' does not exists, creating...`);
-      collection = await db.createCollection(this.collectionName);
+      this.collection = await db.createCollection(this.collectionName);
 
-      await this.createIndex(collection);
+      await this.createIndex();
       Logger.info(`Collection '${this.collectionName}' created.`);
     }
-
-    this.collection = collection;
   }
 
   /**
-   * Create the indices required by the collection passed.
+   * Create the indices required by this store.
    * To be overridden by inherited classes if a collection index is needed.
    */
-  protected async createIndex (_collection: Collection): Promise<void> {
+  public async createIndex (): Promise<void> {
   }
 }

lib/core/Core.ts

@@ -1,9 +1,11 @@
+import * as semver from 'semver';
 import * as timeSpan from 'time-span';
 import { ISidetreeCas, ISidetreeEventEmitter, ISidetreeLogger } from '..';
 import BatchScheduler from './BatchScheduler';
 import Blockchain from './Blockchain';
 import Config from './models/Config';
 import DownloadManager from './DownloadManager';
+import ErrorCode from './ErrorCode';
 import EventEmitter from '../common/EventEmitter';
 import LogColor from '../common/LogColor';
 import Logger from '../common/Logger';
@@ -18,6 +20,7 @@ import ResponseModel from '../common/models/ResponseModel';
 import ResponseStatus from '../common/enums/ResponseStatus';
 import ServiceInfo from '../common/ServiceInfoProvider';
 import ServiceStateModel from './models/ServiceStateModel';
+import SidetreeError from '../common/SidetreeError';
 import VersionManager from './VersionManager';
 import VersionModel from './models/VersionModel';
 
@@ -155,12 +158,20 @@ export default class Core {
     const currentServiceVersionModel = this.serviceInfo.getServiceVersion();
     const currentServiceVersion = currentServiceVersionModel.version;
     const savedServiceState = await this.serviceStateStore.get();
-    const savedServiceVersion = savedServiceState?.serviceVersion;
+    const savedServiceVersion = savedServiceState.serviceVersion;
 
     if (savedServiceVersion === currentServiceVersion) {
       return;
     }
 
+    // Throw if attempting to run old code on new DB.
+    if (semver.lt(currentServiceVersion, savedServiceVersion)) {
+      Logger.error(
+        LogColor.red(`Running older code ${LogColor.green(currentServiceVersion)} on newer DB ${LogColor.green(savedServiceVersion)} is not supported.`)
+      );
+      throw new SidetreeError(ErrorCode.RunningOlderCodeOnNewerDatabaseUnsupported);
+    }
+
     // Add DB upgrade code below.
 
     Logger.warn(LogColor.yellow(`Upgrading DB from version ${LogColor.green(savedServiceVersion)} to ${LogColor.green(currentServiceVersion)}...`));
@@ -171,6 +182,8 @@ export default class Core {
     await this.transactionStore.clearCollection();
     await this.unresolvableTransactionStore.clearCollection();
 
+    await this.operationStore.createIndex();
+
     await this.serviceStateStore.put({ serviceVersion: currentServiceVersion });
 
     Logger.warn(LogColor.yellow(`DB upgraded in: ${LogColor.green(timer.rounded())} ms.`));

lib/core/ErrorCode.ts

@@ -11,6 +11,7 @@ export default {
   BlockchainReadResponseBodyNotJson: 'blockchain_read_response_body_not_json',
   BlockchainReadResponseNotOk: 'blockchain_read_response_not_ok',
   BlockchainWriteUnexpectedError: 'blockchain_write_unexpected_error',
+  RunningOlderCodeOnNewerDatabaseUnsupported: 'running_older_code_on_newer_database_unsupported',
   VersionManagerVersionStringNotFound: 'version_manager_version_string_not_found',
   VersionManagerVersionMetadataIncorrectType: 'version_manager_version_metadata_incorrect_type'
 };

lib/core/MongoDbOperationStore.ts

@@ -1,4 +1,4 @@
-import { Binary, Collection, Long } from 'mongodb';
+import { Binary, Long } from 'mongodb';
 import AnchoredOperationModel from './models/AnchoredOperationModel';
 import IOperationStore from './interfaces/IOperationStore';
 import MongoDbStore from '../common/MongoDbStore';
@@ -31,9 +31,11 @@ export default class MongoDbOperationStore extends MongoDbStore implements IOper
     super(serverUrl, MongoDbOperationStore.collectionName, databaseName);
   }
 
-  protected async createIndex (collection: Collection) {
+  public async createIndex () {
     // This is an unique index, so duplicate inserts are rejected/ignored.
-    await collection.createIndex({ didSuffix: 1, txnNumber: 1, opIndex: 1, type: 1 }, { unique: true });
+    await this.collection.createIndex({ didSuffix: 1, txnNumber: 1, opIndex: 1, type: 1 }, { unique: true });
+    // The query in `get() method needs a corresponding composite index in some cloud-based services (CosmosDB 4.0) that supports MongoDB driver.
+    await this.collection.createIndex({ didSuffix: 1, txnNumber: 1, opIndex: 1 }, { unique: true });
   }
 
   public async insertOrReplace (operations: AnchoredOperationModel[]): Promise<void> {
@@ -59,7 +61,7 @@ export default class MongoDbOperationStore extends MongoDbStore implements IOper
   public async get (didUniqueSuffix: string): Promise<AnchoredOperationModel[]> {
     const mongoOperations = await this.collection!
       .find({ didSuffix: didUniqueSuffix })
-      .sort({ txnNumber: 1, opIndex: 1 })
+      .sort({ didSuffix: 1, txnNumber: 1, opIndex: 1 })
       .maxTimeMS(MongoDbStore.defaultQueryTimeoutInMilliseconds)
       .toArray();
 

lib/core/versions/latest/MongoDbOperationQueue.ts

@@ -6,7 +6,7 @@ import SidetreeError from '../../../common/SidetreeError';
 
 /**
  * Sidetree operation stored in MongoDb.
- * Note: we use the shorter property name "opIndex" instead of "operationIndex" due to a constraint imposed by CosmosDB/MongoDB:
+ * Note: we use the shorter property name "opIndex" instead of "operationIndex" due to a constraint imposed by some MongoDB service such as CosmosDB 3.2:
  * the sum of property names of a unique index keys need to be less than 40 characters.
  * Note: We represent opIndex, transactionNumber, and transactionTime as long instead of number (double) to avoid some floating
  * point comparison quirks.