diff --git a/.gitignore b/.gitignore index 4f06ea24e..3a28eb719 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,7 @@ nunitresults.xml .nyc_output/ coverage/ -style-conversion \ No newline at end of file +# Intellij +.idea/ + +style-conversion diff --git a/lib/bitcoin/BitcoinProcessor.ts b/lib/bitcoin/BitcoinProcessor.ts index fa1dade8c..a56d01e7d 100644 --- a/lib/bitcoin/BitcoinProcessor.ts +++ b/lib/bitcoin/BitcoinProcessor.ts @@ -108,7 +108,7 @@ export default class BitcoinProcessor { this.serviceStateStore = new MongoDbServiceStateStore(config.mongoDbConnectionString, config.databaseName); this.blockMetadataStore = new MongoDbBlockMetadataStore(config.mongoDbConnectionString, config.databaseName); - this.transactionStore = new MongoDbTransactionStore(); + this.transactionStore = new MongoDbTransactionStore(config.mongoDbConnectionString, config.databaseName); this.spendingMonitor = new SpendingMonitor(config.bitcoinFeeSpendingCutoffPeriodInBlocks, BitcoinClient.convertBtcToSatoshis(config.bitcoinFeeSpendingCutoff), @@ -165,7 +165,7 @@ export default class BitcoinProcessor { await this.versionManager.initialize(versionModels, this.config, this.blockMetadataStore); await this.serviceStateStore.initialize(); await this.blockMetadataStore.initialize(); - await this.transactionStore.initialize(this.config.mongoDbConnectionString, this.config.databaseName); + await this.transactionStore.initialize(); await this.bitcoinClient.initialize(); await this.mongoDbLockTransactionStore.initialize(); diff --git a/lib/bitcoin/lock/MongoDbLockTransactionStore.ts b/lib/bitcoin/lock/MongoDbLockTransactionStore.ts index a7d918778..93f62bff0 100644 --- a/lib/bitcoin/lock/MongoDbLockTransactionStore.ts +++ b/lib/bitcoin/lock/MongoDbLockTransactionStore.ts @@ -1,14 +1,11 @@ -import { Collection, Db, Long, MongoClient } from 'mongodb'; -import Logger from '../../common/Logger'; +import { Long } from 'mongodb'; +import MongoDbStore from '../../common/MongoDbStore'; import SavedLockModel from './../models/SavedLockedModel'; /** * Encapsulates functionality to store the bitcoin lock information to Db. */ -export default class MongoDbLockTransactionStore { - private db: Db | undefined; - private lockCollection: Collection | undefined; - +export default class MongoDbLockTransactionStore extends MongoDbStore { /** The collection name */ public static readonly lockCollectionName = 'locks'; @@ -18,17 +15,9 @@ export default class MongoDbLockTransactionStore { * @param databaseName The database name where the collection should be saved. */ public constructor ( - private serverUrl: string, - private databaseName: string) { - } - - /** - * Initializes this object by creating the required collection. - */ - public async initialize () { - const client = await MongoClient.connect(this.serverUrl, { useNewUrlParser: true }); // `useNewUrlParser` addresses nodejs's URL parser deprecation warning. - this.db = client.db(this.databaseName); - this.lockCollection = await MongoDbLockTransactionStore.creatLockCollectionIfNotExist(this.db); + serverUrl: string, + databaseName: string) { + super(serverUrl, MongoDbLockTransactionStore.lockCollectionName, databaseName); } /** @@ -47,22 +36,14 @@ export default class MongoDbLockTransactionStore { type: bitcoinLock.type }; - await this.lockCollection!.insertOne(lockInMongoDb); - } - - /** - * Clears the store. - */ - public async clearCollection () { - await this.lockCollection!.drop(); - this.lockCollection = await MongoDbLockTransactionStore.creatLockCollectionIfNotExist(this.db!); + await this.collection!.insertOne(lockInMongoDb); } /** * Gets the latest lock (highest create timestamp) saved in the db; or undefined if nothing saved. */ public async getLastLock (): Promise { - const lastLocks = await this.lockCollection! + const lastLocks = await this.collection! .find() .limit(1) .sort({ createTimestamp: -1 }) @@ -75,23 +56,10 @@ export default class MongoDbLockTransactionStore { return lastLocks[0] as SavedLockModel; } - private static async creatLockCollectionIfNotExist (db: Db): Promise> { - const collections = await db.collections(); - const collectionNames = collections.map(collection => collection.collectionName); - - // If 'locks' collection exists, use it; else create it. - let lockCollection; - if (collectionNames.includes(MongoDbLockTransactionStore.lockCollectionName)) { - Logger.info('Locks collection already exists.'); - lockCollection = db.collection(MongoDbLockTransactionStore.lockCollectionName); - } else { - Logger.info('Locks collection does not exists, creating...'); - lockCollection = await db.createCollection(MongoDbLockTransactionStore.lockCollectionName); - - await lockCollection.createIndex({ createTimestamp: -1 }); - Logger.info('Locks collection created.'); - } - - return lockCollection; + /** + * @inheritDoc + */ + public async createIndex (): Promise { + await this.collection.createIndex({ createTimestamp: -1 }); } } diff --git a/lib/common/ConsoleLogger.ts b/lib/common/ConsoleLogger.ts index f07316cb5..f71ea303f 100644 --- a/lib/common/ConsoleLogger.ts +++ b/lib/common/ConsoleLogger.ts @@ -4,6 +4,10 @@ import ILogger from './interfaces/ILogger'; * Console Logger. */ export default class ConsoleLogger implements ILogger { + debug (data: any): void { + console.debug(data); + } + info (data: any): void { console.info(data); } diff --git a/lib/common/Logger.ts b/lib/common/Logger.ts index c9456c29f..73ba4b722 100644 --- a/lib/common/Logger.ts +++ b/lib/common/Logger.ts @@ -17,6 +17,13 @@ export default class Logger { } } + /** + * Logs debug. + */ + public static debug (data: any): void { + Logger.singleton.debug(data); + } + /** * Logs info. */ diff --git a/lib/common/MongoDbStore.ts b/lib/common/MongoDbStore.ts index 81c41f478..52de6f82e 100644 --- a/lib/common/MongoDbStore.ts +++ b/lib/common/MongoDbStore.ts @@ -1,4 +1,4 @@ -import { Collection, Db, MongoClient } from 'mongodb'; +import { Collection, Db, LoggerState, MongoClient } from 'mongodb'; import Logger from '../common/Logger'; /** @@ -13,6 +13,44 @@ export default class MongoDbStore { /** MongoDB collection */ protected collection!: Collection; + /** + * Set the logger for mongodb command monitoring. + * @param client the mongodb client + */ + public static enableCommandResultLogging (client: MongoClient) { + client.on('commandSucceeded', (event: any) => { + Logger.info(event); + }); + client.on('commandFailed', (event: any) => { + Logger.warn(event); + }); + } + + /** + * The custom logger for general logging purpose in mongodb client + * @param _message The message is already included in the state so there is no need to log the message twice. + * @param state The complete logging event state + */ + public static customLogger (_message: string | undefined, state: LoggerState | undefined): void { + if (state === undefined) { + return; + } + + switch (state.type) { + case 'warn': + Logger.warn(state); + break; + case 'error': + Logger.error(state); + break; + case 'debug': + Logger.debug(state); + break; + default: + Logger.info(state); + } + }; + /** * Constructs a `MongoDbStore`; */ @@ -22,7 +60,14 @@ export default class MongoDbStore { * Initialize the MongoDB transaction store. */ public async initialize (): Promise { - const client = await MongoClient.connect(this.serverUrl, { useNewUrlParser: true }); // `useNewUrlParser` addresses nodejs's URL parser deprecation warning. + // `useNewUrlParser` addresses nodejs's URL parser deprecation warning. + const client = await MongoClient.connect(this.serverUrl, { + useNewUrlParser: true, + logger: MongoDbStore.customLogger, + monitorCommands: true, + loggerLevel: 'debug' + }); + MongoDbStore.enableCommandResultLogging(client); this.db = client.db(this.databaseName); await this.createCollectionIfNotExist(this.db); } diff --git a/lib/common/MongoDbTransactionStore.ts b/lib/common/MongoDbTransactionStore.ts index 2f25cfc74..1e41b9c5b 100644 --- a/lib/common/MongoDbTransactionStore.ts +++ b/lib/common/MongoDbTransactionStore.ts @@ -1,27 +1,25 @@ -import { Collection, Cursor, Db, Long, MongoClient } from 'mongodb'; +import { Cursor, Long } from 'mongodb'; import ITransactionStore from '../core/interfaces/ITransactionStore'; import Logger from '../common/Logger'; +import MongoDbStore from './MongoDbStore'; import TransactionModel from './models/TransactionModel'; /** * Implementation of ITransactionStore that stores the transaction data in a MongoDB database. */ -export default class MongoDbTransactionStore implements ITransactionStore { - /** Default database name used if not specified in constructor. */ - public static readonly defaultDatabaseName: string = 'sidetree'; +export default class MongoDbTransactionStore extends MongoDbStore implements ITransactionStore { /** Collection name for transactions. */ public static readonly transactionCollectionName: string = 'transactions'; - private db: Db | undefined; - private transactionCollection: Collection | undefined; - /** - * Initialize the MongoDB transaction store. + * Creates a new instance of this object. + * @param serverUrl The target server url. + * @param databaseName The database name where the collection should be saved. */ - public async initialize (serverUrl: string, databaseName: string): Promise { - const client = await MongoClient.connect(serverUrl, { useNewUrlParser: true }); // `useNewUrlParser` addresses nodejs's URL parser deprecation warning. - this.db = client.db(databaseName); - this.transactionCollection = await MongoDbTransactionStore.createTransactionCollectionIfNotExist(this.db); + public constructor ( + serverUrl: string, + databaseName: string) { + super(serverUrl, MongoDbTransactionStore.transactionCollectionName, databaseName); } /** @@ -29,12 +27,12 @@ export default class MongoDbTransactionStore implements ITransactionStore { * Mainly used by tests. */ public async getTransactionsCount (): Promise { - const transactionCount = await this.transactionCollection!.count(); + const transactionCount = await this.collection!.count(); return transactionCount; } public async getTransaction (transactionNumber: number): Promise { - const transactions = await this.transactionCollection!.find({ transactionNumber: Long.fromNumber(transactionNumber) }).toArray(); + const transactions = await this.collection!.find({ transactionNumber: Long.fromNumber(transactionNumber) }).toArray(); if (transactions.length === 0) { return undefined; } @@ -52,9 +50,9 @@ export default class MongoDbTransactionStore implements ITransactionStore { // If given `undefined`, return transactions from the start. if (transactionNumber === undefined) { - dbCursor = this.transactionCollection!.find(); + dbCursor = this.collection!.find(); } else { - dbCursor = this.transactionCollection!.find({ transactionNumber: { $gt: Long.fromNumber(transactionNumber) } }); + dbCursor = this.collection!.find({ transactionNumber: { $gt: Long.fromNumber(transactionNumber) } }); } // If a limit is defined then set it. @@ -75,16 +73,6 @@ export default class MongoDbTransactionStore implements ITransactionStore { return transactions; } - /** - * Clears the transaction store. - */ - public async clearCollection () { - // NOTE: We avoid implementing this by deleting and recreating the collection in rapid succession, - // because doing so against some cloud MongoDB services such as CosmosDB, - // especially in rapid repetition that can occur in tests, will lead to `MongoError: ns not found` connectivity error. - await this.transactionCollection!.deleteMany({ }); // Empty filter removes all entries in collection. - } - async addTransaction (transaction: TransactionModel): Promise { try { const transactionInMongoDb = { @@ -97,7 +85,7 @@ export default class MongoDbTransactionStore implements ITransactionStore { normalizedTransactionFee: transaction.normalizedTransactionFee, writer: transaction.writer }; - await this.transactionCollection!.insertOne(transactionInMongoDb); + await this.collection!.insertOne(transactionInMongoDb); } catch (error) { // Swallow duplicate insert errors (error code 11000) as no-op; rethrow others if (error.code !== 11000) { @@ -107,7 +95,7 @@ export default class MongoDbTransactionStore implements ITransactionStore { } async getLastTransaction (): Promise { - const lastTransactions = await this.transactionCollection!.find().limit(1).sort({ transactionNumber: -1 }).toArray(); + const lastTransactions = await this.collection!.find().limit(1).sort({ transactionNumber: -1 }).toArray(); if (lastTransactions.length === 0) { return undefined; } @@ -118,7 +106,7 @@ export default class MongoDbTransactionStore implements ITransactionStore { async getExponentiallySpacedTransactions (): Promise { const exponentiallySpacedTransactions: TransactionModel[] = []; - const allTransactions = await this.transactionCollection!.find().sort({ transactionNumber: 1 }).toArray(); + const allTransactions = await this.collection!.find().sort({ transactionNumber: 1 }).toArray(); let index = allTransactions.length - 1; let distance = 1; @@ -137,7 +125,7 @@ export default class MongoDbTransactionStore implements ITransactionStore { return; } - await this.transactionCollection!.deleteMany({ transactionNumber: { $gt: Long.fromNumber(transactionNumber) } }); + await this.collection!.deleteMany({ transactionNumber: { $gt: Long.fromNumber(transactionNumber) } }); } /** @@ -145,7 +133,7 @@ export default class MongoDbTransactionStore implements ITransactionStore { * @param transactionTimeHash the transaction time hash which the transactions should be removed for */ public async removeTransactionByTransactionTimeHash (transactionTimeHash: string) { - await this.transactionCollection!.deleteMany({ transactionTimeHash: { $eq: transactionTimeHash } }); + await this.collection!.deleteMany({ transactionTimeHash: { $eq: transactionTimeHash } }); } /** @@ -153,7 +141,7 @@ export default class MongoDbTransactionStore implements ITransactionStore { * Mainly used for test purposes. */ public async getTransactions (): Promise { - const transactions = await this.transactionCollection!.find().sort({ transactionNumber: 1 }).toArray(); + const transactions = await this.collection!.find().sort({ transactionNumber: 1 }).toArray(); return transactions; } @@ -166,9 +154,9 @@ export default class MongoDbTransactionStore implements ITransactionStore { let cursor: Cursor; if (inclusiveBeginTransactionTime === exclusiveEndTransactionTime) { // if begin === end, query for 1 transaction time - cursor = this.transactionCollection!.find({ transactionTime: { $eq: Long.fromNumber(inclusiveBeginTransactionTime) } }); + cursor = this.collection!.find({ transactionTime: { $eq: Long.fromNumber(inclusiveBeginTransactionTime) } }); } else { - cursor = this.transactionCollection!.find({ + cursor = this.collection!.find({ $and: [ { transactionTime: { $gte: Long.fromNumber(inclusiveBeginTransactionTime) } }, { transactionTime: { $lt: Long.fromNumber(exclusiveEndTransactionTime) } } @@ -181,26 +169,9 @@ export default class MongoDbTransactionStore implements ITransactionStore { } /** - * Creates the `transaction` collection with indexes if it does not exists. - * @returns The existing collection if exists, else the newly created collection. + * @inheritDoc */ - private static async createTransactionCollectionIfNotExist (db: Db): Promise> { - const collections = await db.collections(); - const collectionNames = collections.map(collection => collection.collectionName); - - // If 'transactions' collection exists, use it; else create it. - let transactionCollection; - if (collectionNames.includes(MongoDbTransactionStore.transactionCollectionName)) { - Logger.info('Transaction collection already exists.'); - transactionCollection = db.collection(MongoDbTransactionStore.transactionCollectionName); - } else { - Logger.info('Transaction collection does not exists, creating...'); - transactionCollection = await db.createCollection(MongoDbTransactionStore.transactionCollectionName); - // Note the unique index, so duplicate inserts are rejected. - await transactionCollection.createIndex({ transactionNumber: 1 }, { unique: true }); - Logger.info('Transaction collection created.'); - } - - return transactionCollection; + public async createIndex (): Promise { + await this.collection.createIndex({ transactionNumber: 1 }, { unique: true }); } } diff --git a/lib/common/interfaces/ILogger.ts b/lib/common/interfaces/ILogger.ts index 5c5351161..d7eddb52e 100644 --- a/lib/common/interfaces/ILogger.ts +++ b/lib/common/interfaces/ILogger.ts @@ -2,6 +2,11 @@ * Custom logger interface. */ export default interface ILogger { + /** + * Logs debugging data + */ + debug (data: any): void; + /** * Logs informational data. */ diff --git a/lib/core/Core.ts b/lib/core/Core.ts index 4baccd0d3..249aba8bd 100644 --- a/lib/core/Core.ts +++ b/lib/core/Core.ts @@ -57,7 +57,7 @@ export default class Core { this.blockchain = new Blockchain(config.blockchainServiceUri); this.downloadManager = new DownloadManager(config.maxConcurrentDownloads, this.cas); this.resolver = new Resolver(this.versionManager, this.operationStore); - this.transactionStore = new MongoDbTransactionStore(); + this.transactionStore = new MongoDbTransactionStore(config.mongoDbConnectionString, config.databaseName); this.unresolvableTransactionStore = new MongoDbUnresolvableTransactionStore(config.mongoDbConnectionString, config.databaseName); // Only enable real blockchain time pull if observer is enabled @@ -75,7 +75,7 @@ export default class Core { config.observingIntervalInSeconds ); - this.monitor = new Monitor(); + this.monitor = new Monitor(this.config, this.versionManager, this.blockchain); } /** @@ -88,7 +88,7 @@ export default class Core { // DB initializations. await this.serviceStateStore.initialize(); - await this.transactionStore.initialize(this.config.mongoDbConnectionString, this.config.databaseName); + await this.transactionStore.initialize(); await this.unresolvableTransactionStore.initialize(); await this.operationStore.initialize(); await this.upgradeDatabaseIfNeeded(); @@ -119,7 +119,7 @@ export default class Core { this.downloadManager.start(); - await this.monitor.initialize(this.config, this.versionManager, this.blockchain); + await this.monitor.initialize(); } /** diff --git a/lib/core/MongoDbUnresolvableTransactionStore.ts b/lib/core/MongoDbUnresolvableTransactionStore.ts index 4ba69223f..6bb5bee9a 100644 --- a/lib/core/MongoDbUnresolvableTransactionStore.ts +++ b/lib/core/MongoDbUnresolvableTransactionStore.ts @@ -1,59 +1,44 @@ -import { Collection, Db, Long, MongoClient } from 'mongodb'; import IUnresolvableTransactionStore from './interfaces/IUnresolvableTransactionStore'; import Logger from '../common/Logger'; +import { Long } from 'mongodb'; +import MongoDbStore from '../common/MongoDbStore'; import TransactionModel from '../common/models/TransactionModel'; import UnresolvableTransactionModel from './models/UnresolvableTransactionModel'; /** * Implementation of `IUnresolvableTransactionStore` that stores the transaction data in a MongoDB database. */ -export default class MongoDbUnresolvableTransactionStore implements IUnresolvableTransactionStore { +export default class MongoDbUnresolvableTransactionStore extends MongoDbStore implements IUnresolvableTransactionStore { /** Collection name for unresolvable transactions. */ public static readonly unresolvableTransactionCollectionName: string = 'unresolvable-transactions'; private exponentialDelayFactorInMilliseconds = 60000; private maximumUnresolvableTransactionReturnCount = 100; - private db: Db | undefined; - private unresolvableTransactionCollection: Collection | undefined; - /** - * Constructs a `MongoDbUnresolvableTransactionStore`; + * Creates a new instance of this object. + * @param serverUrl The target server url. + * @param databaseName The database name where the collection should be saved. * @param retryExponentialDelayFactor * The exponential delay factor in milliseconds for retries of unresolvable transactions. * e.g. if it is set to 1 seconds, then the delays for retries will be 1 second, 2 seconds, 4 seconds... until the transaction can be resolved. */ - constructor (private serverUrl: string, private databaseName: string, retryExponentialDelayFactor?: number) { + public constructor ( + serverUrl: string, + databaseName: string, + retryExponentialDelayFactor?: number) { + super(serverUrl, MongoDbUnresolvableTransactionStore.unresolvableTransactionCollectionName, databaseName); if (retryExponentialDelayFactor !== undefined) { this.exponentialDelayFactorInMilliseconds = retryExponentialDelayFactor; } } - /** - * Initialize the MongoDB unresolvable transaction store. - */ - public async initialize (): Promise { - const client = await MongoClient.connect(this.serverUrl, { useNewUrlParser: true }); // `useNewUrlParser` addresses nodejs's URL parser deprecation warning. - this.db = client.db(this.databaseName); - this.unresolvableTransactionCollection = await MongoDbUnresolvableTransactionStore.createUnresolvableTransactionCollectionIfNotExist(this.db); - } - - /** - * * Clears the unresolvable transaction store. - */ - public async clearCollection () { - // NOTE: We avoid implementing this by deleting and recreating the collection in rapid succession, - // because doing so against some cloud MongoDB services such as CosmosDB, - // especially in rapid repetition that can occur in tests, will lead to `MongoError: ns not found` connectivity error. - await this.unresolvableTransactionCollection!.deleteMany({ }); // Empty filter removes all entries in collection. - } - public async recordUnresolvableTransactionFetchAttempt (transaction: TransactionModel): Promise { // Try to get the unresolvable transaction from store. const transactionTime = transaction.transactionTime; const transactionNumber = transaction.transactionNumber; const searchFilter = { transactionTime, transactionNumber: Long.fromNumber(transactionNumber) }; - const findResults = await this.unresolvableTransactionCollection!.find(searchFilter).toArray(); + const findResults = await this.collection!.find(searchFilter).toArray(); let unresolvableTransaction: UnresolvableTransactionModel | undefined; if (findResults && findResults.length > 0) { unresolvableTransaction = findResults[0]; @@ -75,7 +60,7 @@ export default class MongoDbUnresolvableTransactionStore implements IUnresolvabl nextRetryTime: Date.now() }; - await this.unresolvableTransactionCollection!.insertOne(newUnresolvableTransaction); + await this.collection!.insertOne(newUnresolvableTransaction); } else { const retryAttempts = unresolvableTransaction.retryAttempts + 1; @@ -87,14 +72,14 @@ export default class MongoDbUnresolvableTransactionStore implements IUnresolvabl const nextRetryTime = unresolvableTransaction.firstFetchTime + requiredElapsedTimeSinceFirstFetchBeforeNextRetry; const searchFilter = { transactionTime, transactionNumber: Long.fromNumber(transactionNumber) }; - await this.unresolvableTransactionCollection!.updateOne(searchFilter, { $set: { retryAttempts, nextRetryTime } }); + await this.collection!.updateOne(searchFilter, { $set: { retryAttempts, nextRetryTime } }); } } public async removeUnresolvableTransaction (transaction: TransactionModel): Promise { const transactionTime = transaction.transactionTime; const transactionNumber = transaction.transactionNumber; - await this.unresolvableTransactionCollection!.deleteOne({ transactionTime, transactionNumber: Long.fromNumber(transactionNumber) }); + await this.collection!.deleteOne({ transactionTime, transactionNumber: Long.fromNumber(transactionNumber) }); } public async getUnresolvableTransactionsDueForRetry (maximumReturnCount?: number): Promise { @@ -106,7 +91,7 @@ export default class MongoDbUnresolvableTransactionStore implements IUnresolvabl const now = Date.now(); const unresolvableTransactionsToRetry = - await this.unresolvableTransactionCollection!.find({ nextRetryTime: { $lte: now } }).sort({ nextRetryTime: 1 }).limit(returnCount).toArray(); + await this.collection!.find({ nextRetryTime: { $lte: now } }).sort({ nextRetryTime: 1 }).limit(returnCount).toArray(); return unresolvableTransactionsToRetry; } @@ -118,7 +103,7 @@ export default class MongoDbUnresolvableTransactionStore implements IUnresolvabl return; } - await this.unresolvableTransactionCollection!.deleteMany({ transactionNumber: { $gt: Long.fromNumber(transactionNumber) } }); + await this.collection!.deleteMany({ transactionNumber: { $gt: Long.fromNumber(transactionNumber) } }); } /** @@ -126,31 +111,15 @@ export default class MongoDbUnresolvableTransactionStore implements IUnresolvabl * Mainly used for test purposes. */ public async getUnresolvableTransactions (): Promise { - const transactions = await this.unresolvableTransactionCollection!.find().sort({ transactionTime: 1, transactionNumber: 1 }).toArray(); + const transactions = await this.collection!.find().sort({ transactionTime: 1, transactionNumber: 1 }).toArray(); return transactions; } /** - * Creates the `unresolvable-transaction` collection with indexes if it does not exists. - * @returns The existing collection if exists, else the newly created collection. + * @inheritDoc */ - public static async createUnresolvableTransactionCollectionIfNotExist (db: Db): Promise> { - const collections = await db.collections(); - const collectionNames = collections.map(collection => collection.collectionName); - - // If 'unresolvable transactions' collection exists, use it; else create it. - let unresolvableTransactionCollection; - if (collectionNames.includes(MongoDbUnresolvableTransactionStore.unresolvableTransactionCollectionName)) { - Logger.info('Unresolvable transaction collection already exists.'); - unresolvableTransactionCollection = db.collection(MongoDbUnresolvableTransactionStore.unresolvableTransactionCollectionName); - } else { - Logger.info('Unresolvable transaction collection does not exists, creating...'); - unresolvableTransactionCollection = await db.createCollection(MongoDbUnresolvableTransactionStore.unresolvableTransactionCollectionName); - await unresolvableTransactionCollection.createIndex({ transactionTime: 1, transactionNumber: 1 }, { unique: true }); - await unresolvableTransactionCollection.createIndex({ nextRetryTime: 1 }); - Logger.info('Unresolvable transaction collection created.'); - } - - return unresolvableTransactionCollection; + public async createIndex (): Promise { + await this.collection.createIndex({ transactionTime: 1, transactionNumber: 1 }, { unique: true }); + await this.collection.createIndex({ nextRetryTime: 1 }); } } diff --git a/lib/core/Monitor.ts b/lib/core/Monitor.ts index 637a2b47d..2157d892b 100644 --- a/lib/core/Monitor.ts +++ b/lib/core/Monitor.ts @@ -12,21 +12,21 @@ import VersionManager from './VersionManager'; */ export default class Monitor { - private blockchain!: Blockchain; + private blockchain: Blockchain; private operationQueue: MongoDbOperationQueue; private transactionStore: MongoDbTransactionStore; - private versionManager!: VersionManager; + private readonly versionManager: VersionManager; - public constructor () { - this.operationQueue = new MongoDbOperationQueue(); - this.transactionStore = new MongoDbTransactionStore(); - } - - public async initialize (config: Config, versionManager: VersionManager, blockchain: Blockchain) { + public constructor (config: Config, versionManager: VersionManager, blockchain: Blockchain) { + this.operationQueue = new MongoDbOperationQueue(config.mongoDbConnectionString, config.databaseName); + this.transactionStore = new MongoDbTransactionStore(config.mongoDbConnectionString, config.databaseName); this.blockchain = blockchain; this.versionManager = versionManager; - await this.transactionStore.initialize(config.mongoDbConnectionString, config.databaseName); - await this.operationQueue.initialize(config.mongoDbConnectionString, config.databaseName); + } + + public async initialize () { + await this.transactionStore.initialize(); + await this.operationQueue.initialize(); } /** diff --git a/lib/core/versions/1.0/MongoDbOperationQueue.ts b/lib/core/versions/1.0/MongoDbOperationQueue.ts index 36ec37532..e8524c956 100644 --- a/lib/core/versions/1.0/MongoDbOperationQueue.ts +++ b/lib/core/versions/1.0/MongoDbOperationQueue.ts @@ -1,6 +1,7 @@ -import { Binary, Collection, Db, MongoClient } from 'mongodb'; +import { Binary } from 'mongodb'; import ErrorCode from './ErrorCode'; import IOperationQueue from './interfaces/IOperationQueue'; +import MongoDbStore from '../../../common/MongoDbStore'; import QueuedOperationModel from './models/QueuedOperationModel'; import SidetreeError from '../../../common/SidetreeError'; @@ -19,21 +20,26 @@ interface IMongoQueuedOperation { /** * Operation queue used by the Batch Writer implemented using MongoDB. */ -export default class MongoDbOperationQueue implements IOperationQueue { +export default class MongoDbOperationQueue extends MongoDbStore implements IOperationQueue { /** Collection name for queued operations. */ public static readonly collectionName: string = 'queued-operations'; - private collection: Collection | undefined; - - private db: Db | undefined; + /** + * Creates a new instance of this object. + * @param serverUrl The target server url. + * @param databaseName The database name where the collection should be saved. + */ + public constructor ( + serverUrl: string, + databaseName: string) { + super(serverUrl, MongoDbOperationQueue.collectionName, databaseName); + } /** - * Initialize the MongoDB operation store. + * @inheritDoc */ - public async initialize (serverUrl: string, databaseName: string) { - const client = await MongoClient.connect(serverUrl); - this.db = client.db(databaseName); - this.collection = await MongoDbOperationQueue.createCollectionIfNotExist(this.db); + public async createIndex (): Promise { + await this.collection.createIndex({ didUniqueSuffix: 1 }, { unique: true }); } async enqueue (didUniqueSuffix: string, operationBuffer: Buffer) { @@ -90,37 +96,6 @@ export default class MongoDbOperationQueue implements IOperationQueue { return size; } - /** - * * Clears the unresolvable transaction store. Mainly used in tests. - */ - public async clearCollection () { - await this.collection!.drop(); - this.collection = await MongoDbOperationQueue.createCollectionIfNotExist(this.db!); - } - - /** - * Creates the queued operation collection with indexes if it does not exists. - * @returns The existing collection if exists, else the newly created collection. - */ - private static async createCollectionIfNotExist (db: Db): Promise> { - // Get the names of existing collections. - const collections = await db.collections(); - const collectionNames = collections.map(collection => collection.collectionName); - - // If the queued operation collection exists, use it; else create it then use it. - let collection; - if (collectionNames.includes(this.collectionName)) { - collection = db.collection(this.collectionName); - } else { - collection = await db.createCollection(this.collectionName); - // Create an index on didUniqueSuffix make `contains()` operations more efficient. - // This is an unique index, so duplicate inserts are rejected. - await collection.createIndex({ didUniqueSuffix: 1 }, { unique: true }); - } - - return collection; - } - private static convertToQueuedOperationModel (mongoQueuedOperation: IMongoQueuedOperation): QueuedOperationModel { return { didUniqueSuffix: mongoQueuedOperation.didUniqueSuffix, diff --git a/lib/core/versions/latest/MongoDbOperationQueue.ts b/lib/core/versions/latest/MongoDbOperationQueue.ts index 36ec37532..e8524c956 100644 --- a/lib/core/versions/latest/MongoDbOperationQueue.ts +++ b/lib/core/versions/latest/MongoDbOperationQueue.ts @@ -1,6 +1,7 @@ -import { Binary, Collection, Db, MongoClient } from 'mongodb'; +import { Binary } from 'mongodb'; import ErrorCode from './ErrorCode'; import IOperationQueue from './interfaces/IOperationQueue'; +import MongoDbStore from '../../../common/MongoDbStore'; import QueuedOperationModel from './models/QueuedOperationModel'; import SidetreeError from '../../../common/SidetreeError'; @@ -19,21 +20,26 @@ interface IMongoQueuedOperation { /** * Operation queue used by the Batch Writer implemented using MongoDB. */ -export default class MongoDbOperationQueue implements IOperationQueue { +export default class MongoDbOperationQueue extends MongoDbStore implements IOperationQueue { /** Collection name for queued operations. */ public static readonly collectionName: string = 'queued-operations'; - private collection: Collection | undefined; - - private db: Db | undefined; + /** + * Creates a new instance of this object. + * @param serverUrl The target server url. + * @param databaseName The database name where the collection should be saved. + */ + public constructor ( + serverUrl: string, + databaseName: string) { + super(serverUrl, MongoDbOperationQueue.collectionName, databaseName); + } /** - * Initialize the MongoDB operation store. + * @inheritDoc */ - public async initialize (serverUrl: string, databaseName: string) { - const client = await MongoClient.connect(serverUrl); - this.db = client.db(databaseName); - this.collection = await MongoDbOperationQueue.createCollectionIfNotExist(this.db); + public async createIndex (): Promise { + await this.collection.createIndex({ didUniqueSuffix: 1 }, { unique: true }); } async enqueue (didUniqueSuffix: string, operationBuffer: Buffer) { @@ -90,37 +96,6 @@ export default class MongoDbOperationQueue implements IOperationQueue { return size; } - /** - * * Clears the unresolvable transaction store. Mainly used in tests. - */ - public async clearCollection () { - await this.collection!.drop(); - this.collection = await MongoDbOperationQueue.createCollectionIfNotExist(this.db!); - } - - /** - * Creates the queued operation collection with indexes if it does not exists. - * @returns The existing collection if exists, else the newly created collection. - */ - private static async createCollectionIfNotExist (db: Db): Promise> { - // Get the names of existing collections. - const collections = await db.collections(); - const collectionNames = collections.map(collection => collection.collectionName); - - // If the queued operation collection exists, use it; else create it then use it. - let collection; - if (collectionNames.includes(this.collectionName)) { - collection = db.collection(this.collectionName); - } else { - collection = await db.createCollection(this.collectionName); - // Create an index on didUniqueSuffix make `contains()` operations more efficient. - // This is an unique index, so duplicate inserts are rejected. - await collection.createIndex({ didUniqueSuffix: 1 }, { unique: true }); - } - - return collection; - } - private static convertToQueuedOperationModel (mongoQueuedOperation: IMongoQueuedOperation): QueuedOperationModel { return { didUniqueSuffix: mongoQueuedOperation.didUniqueSuffix, diff --git a/package.json b/package.json index d40bd98e7..f34abf51e 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,7 @@ "commit": "git-cz", "build": "tsc && copyfiles \"lib/**/*.json\" dist && copyfiles \"tests/**/*.js*\" dist && copyfiles \"package.json\" dist", "test": "npm run build && nyc jasmine --config=./tests/jasmine.json", - "test:only": "npm run build && jasmine --config=./tests/jasmine.json", + "test:only": "nyc jasmine --config=./tests/jasmine.json", "test:generate-vectors": "node dist/tests/generators/TestVectorGenerator.js", "util:generate-error-codes": "node dist/tests/generators/ErrorCodeGenerator.js", "publish:unstable": "./scripts/publish-unstable.sh", diff --git a/tests/common/MongoDbStore.spec.ts b/tests/common/MongoDbStore.spec.ts new file mode 100644 index 000000000..6863c4283 --- /dev/null +++ b/tests/common/MongoDbStore.spec.ts @@ -0,0 +1,69 @@ +import Config from '../../lib/core/models/Config'; +import Logger from '../../lib/common/Logger'; +import { MongoClient } from 'mongodb'; +import MongoDb from './MongoDb'; +import MongoDbStore from '../../lib/common/MongoDbStore'; + +describe('MongoDbStore', async () => { + const config: Config = require('../json/config-test.json'); + let mongoServiceAvailable = false; + + beforeAll(async () => { + mongoServiceAvailable = await MongoDb.isServerAvailable(config.mongoDbConnectionString); + }); + + beforeEach(async () => { + if (!mongoServiceAvailable) { + pending('MongoDB service not available'); + } + }); + + it('should invoke command monitoring logger with different log level according to command response status', async () => { + spyOn(Logger, 'info'); + spyOn(Logger, 'warn'); + spyOn(Logger, 'error'); + const client = await MongoClient.connect(config.mongoDbConnectionString, { + useNewUrlParser: true, + monitorCommands: true + }); + MongoDbStore.enableCommandResultLogging(client); + await expectAsync(client.db('sidetree-test').collection('service').findOne({ id: 1 })).toBeResolved(); + expect(Logger.info).toHaveBeenCalledWith(jasmine.objectContaining({ commandName: 'find' })); + await expectAsync(client.db('sidetree-test').collection('service').dropIndex('test')).toBeRejected(); + expect(Logger.warn).toHaveBeenCalledWith(jasmine.objectContaining({ commandName: 'dropIndexes' })); + }); + + it('should invoke logger with corresponding method according to the passed state', () => { + spyOn(Logger, 'info'); + spyOn(Logger, 'warn'); + spyOn(Logger, 'error'); + spyOn(Logger, 'debug'); + MongoDbStore.customLogger('message', undefined); + expect(Logger.info).not.toHaveBeenCalled(); + const state = { + className: 'className', + date: 0, + message: 'message', + pid: 0, + type: 'debug' + }; + MongoDbStore.customLogger('message', state); + expect(Logger.debug).toHaveBeenCalledWith(state); + + state.type = 'info'; + MongoDbStore.customLogger('message', state); + expect(Logger.info).toHaveBeenCalledWith(state); + + state.type = 'error'; + MongoDbStore.customLogger('message', state); + expect(Logger.error).toHaveBeenCalledWith(state); + + state.type = 'warn'; + MongoDbStore.customLogger('message', state); + expect(Logger.warn).toHaveBeenCalledWith(state); + + state.type = 'whatever'; + MongoDbStore.customLogger('message', state); + expect(Logger.info).toHaveBeenCalledWith(state); + }); +}); diff --git a/tests/core/Core.spec.ts b/tests/core/Core.spec.ts index 1fb54561e..cd69705b7 100644 --- a/tests/core/Core.spec.ts +++ b/tests/core/Core.spec.ts @@ -80,7 +80,8 @@ describe('Core', async () => { const customLogger = { info: () => { customLoggerInvoked = true; }, warn: () => { }, - error: () => { } + error: () => { }, + debug: () => { } }; let customEvenEmitterInvoked = false; diff --git a/tests/core/MongoDbOperationQueue.spec.ts b/tests/core/MongoDbOperationQueue.spec.ts index 638f2a025..0745547bd 100644 --- a/tests/core/MongoDbOperationQueue.spec.ts +++ b/tests/core/MongoDbOperationQueue.spec.ts @@ -9,8 +9,8 @@ import SidetreeError from '../../lib/common/SidetreeError'; * Creates a MongoDbOperationQueue and initializes it. */ async function createOperationQueue (storeUri: string, databaseName: string): Promise { - const operationQueue = new MongoDbOperationQueue(); - await operationQueue.initialize(storeUri, databaseName); + const operationQueue = new MongoDbOperationQueue(storeUri, databaseName); + await operationQueue.initialize(); return operationQueue; } diff --git a/tests/core/MongoDbTransactionStore.spec.ts b/tests/core/MongoDbTransactionStore.spec.ts index 5bb34f69d..4f63aa310 100644 --- a/tests/core/MongoDbTransactionStore.spec.ts +++ b/tests/core/MongoDbTransactionStore.spec.ts @@ -9,8 +9,8 @@ import TransactionModel from '../../lib/common/models/TransactionModel'; * Creates a MongoDbTransactionStore and initializes it. */ async function createTransactionStore (transactionStoreUri: string, databaseName: string): Promise { - const transactionStore = new MongoDbTransactionStore(); - await transactionStore.initialize(transactionStoreUri, databaseName); + const transactionStore = new MongoDbTransactionStore(transactionStoreUri, databaseName); + await transactionStore.initialize(); return transactionStore; } @@ -63,7 +63,7 @@ describe('MongoDbTransactionStore', async () => { }); it('should throw error if addTransaction throws a non 11000 error', async () => { - spyOn(transactionStore['transactionCollection'] as any, 'insertOne').and.throwError('Expected test error'); + spyOn(transactionStore['collection'] as any, 'insertOne').and.throwError('Expected test error'); try { await transactionStore.addTransaction({ transactionNumber: 1, @@ -91,7 +91,7 @@ describe('MongoDbTransactionStore', async () => { expect(collectionNames.includes(MongoDbTransactionStore.transactionCollectionName)).toBeFalsy(); console.info(`Trigger initialization.`); - await transactionStore.initialize(config.mongoDbConnectionString, databaseName); + await transactionStore.initialize(); console.info(`Verify collection exists now.`); collections = await db.collections(); @@ -137,7 +137,7 @@ describe('MongoDbTransactionStore', async () => { const transactionCount = 3; await generateAndStoreTransactions(transactionStore, transactionCount); - spyOn(transactionStore['transactionCollection'] as any, 'find').and.throwError('expected test error'); + spyOn(transactionStore['collection'] as any, 'find').and.throwError('expected test error'); const transactions = await transactionStore.getTransactionsLaterThan(1, 100); expect(transactions.length).toEqual(0); }); diff --git a/tests/core/Monitor.spec.ts b/tests/core/Monitor.spec.ts index 2b149ced4..66ea49c51 100644 --- a/tests/core/Monitor.spec.ts +++ b/tests/core/Monitor.spec.ts @@ -8,11 +8,11 @@ describe('Monitor', async () => { describe('getOperationQueueSize()', async () => { it('should get operation queue size correctly.', async () => { - const monitor = new SidetreeMonitor(); + const monitor = new SidetreeMonitor(testConfig, { } as any, { } as any); const operationQueueInitializeSpy = spyOn((monitor as any).operationQueue, 'initialize'); const transactionStoreInitializeSpy = spyOn((monitor as any).transactionStore, 'initialize'); - await monitor.initialize(testConfig, { } as any, { } as any); + await monitor.initialize(); expect(operationQueueInitializeSpy).toHaveBeenCalledTimes(1); expect(transactionStoreInitializeSpy).toHaveBeenCalledTimes(1); }); @@ -20,7 +20,7 @@ describe('Monitor', async () => { describe('getOperationQueueSize()', async () => { it('should get operation queue size correctly.', async () => { - const monitor = new SidetreeMonitor(); + const monitor = new SidetreeMonitor(testConfig, { } as any, { } as any); spyOn((monitor as any).operationQueue, 'getSize').and.returnValue(Promise.resolve(300)); const output = await monitor.getOperationQueueSize(); @@ -30,7 +30,7 @@ describe('Monitor', async () => { describe('getWriterMaxBatchSize()', async () => { it('should get writer max batch size correctly.', async () => { - const monitor = new SidetreeMonitor(); + const monitor = new SidetreeMonitor(testConfig, { } as any, { } as any); (monitor as any).blockchain = { getWriterValueTimeLock: () => { } }; spyOn((monitor as any).blockchain, 'getWriterValueTimeLock'); spyOn(BatchWriter, 'getNumberOfOperationsAllowed').and.returnValue(1000); @@ -52,7 +52,7 @@ describe('Monitor', async () => { normalizedTransactionFee: 1 }; - const monitor = new SidetreeMonitor(); + const monitor = new SidetreeMonitor(testConfig, { } as any, { } as any); spyOn((monitor as any).transactionStore as MongoDbTransactionStore, 'getLastTransaction').and.returnValue(Promise.resolve(mockTransaction)); const output = await monitor.getLastProcessedTransaction();