From b3b76de71b1d4265689d052e7837c38ec1fa4323 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 27 Feb 2023 11:55:47 +1100 Subject: [PATCH] feat: Add option `schemaCacheTtl` for schema cache pulling as alternative to `enableSchemaHooks` (#8436) --- spec/SchemaPerformance.spec.js | 54 ++++++++++++++++ .../Storage/Mongo/MongoStorageAdapter.js | 10 ++- .../Postgres/PostgresStorageAdapter.js | 9 ++- src/Adapters/Storage/StorageAdapter.js | 2 + src/Config.js | 63 +++++++++++++------ src/Controllers/SchemaController.js | 28 ++++++++- src/Options/Definitions.js | 6 ++ src/Options/docs.js | 1 + src/Options/index.js | 2 + src/ParseServer.js | 1 + 10 files changed, 150 insertions(+), 26 deletions(-) diff --git a/spec/SchemaPerformance.spec.js b/spec/SchemaPerformance.spec.js index 0471871c54..17238b0ed6 100644 --- a/spec/SchemaPerformance.spec.js +++ b/spec/SchemaPerformance.spec.js @@ -204,4 +204,58 @@ describe('Schema Performance', function () { ); expect(getAllSpy.calls.count()).toBe(2); }); + + it('does reload with schemaCacheTtl', async () => { + const databaseURI = + process.env.PARSE_SERVER_TEST_DB === 'postgres' + ? process.env.PARSE_SERVER_TEST_DATABASE_URI + : 'mongodb://localhost:27017/parseServerMongoAdapterTestDatabase'; + await reconfigureServer({ + databaseAdapter: undefined, + databaseURI, + silent: false, + databaseOptions: { schemaCacheTtl: 1000 }, + }); + const SchemaController = require('../lib/Controllers/SchemaController').SchemaController; + const spy = spyOn(SchemaController.prototype, 'reloadData').and.callThrough(); + Object.defineProperty(spy, 'reloadCalls', { + get: () => spy.calls.all().filter(call => call.args[0].clearCache).length, + }); + + const object = new TestObject(); + object.set('foo', 'bar'); + await object.save(); + + spy.calls.reset(); + + object.set('foo', 'bar'); + await object.save(); + + expect(spy.reloadCalls).toBe(0); + + await new Promise(resolve => setTimeout(resolve, 1100)); + + object.set('foo', 'bar'); + await object.save(); + + expect(spy.reloadCalls).toBe(1); + }); + + it('cannot set invalid databaseOptions', async () => { + const expectError = async (key, value, expected) => + expectAsync( + reconfigureServer({ databaseAdapter: undefined, databaseOptions: { [key]: value } }) + ).toBeRejectedWith(`databaseOptions.${key} must be a ${expected}`); + for (const databaseOptions of [[], 0, 'string']) { + await expectAsync( + reconfigureServer({ databaseAdapter: undefined, databaseOptions }) + ).toBeRejectedWith(`databaseOptions must be an object`); + } + for (const value of [null, 0, 'string', {}, []]) { + await expectError('enableSchemaHooks', value, 'boolean'); + } + for (const value of [null, false, 'string', {}, []]) { + await expectError('schemaCacheTtl', value, 'number'); + } + }); }); diff --git a/src/Adapters/Storage/Mongo/MongoStorageAdapter.js b/src/Adapters/Storage/Mongo/MongoStorageAdapter.js index c0d4c0ca9e..78833a026b 100644 --- a/src/Adapters/Storage/Mongo/MongoStorageAdapter.js +++ b/src/Adapters/Storage/Mongo/MongoStorageAdapter.js @@ -139,11 +139,12 @@ export class MongoStorageAdapter implements StorageAdapter { _maxTimeMS: ?number; canSortOnJoinTables: boolean; enableSchemaHooks: boolean; + schemaCacheTtl: ?number; constructor({ uri = defaults.DefaultMongoURI, collectionPrefix = '', mongoOptions = {} }: any) { this._uri = uri; this._collectionPrefix = collectionPrefix; - this._mongoOptions = mongoOptions; + this._mongoOptions = { ...mongoOptions }; this._mongoOptions.useNewUrlParser = true; this._mongoOptions.useUnifiedTopology = true; this._onchange = () => {}; @@ -152,8 +153,11 @@ export class MongoStorageAdapter implements StorageAdapter { this._maxTimeMS = mongoOptions.maxTimeMS; this.canSortOnJoinTables = true; this.enableSchemaHooks = !!mongoOptions.enableSchemaHooks; - delete mongoOptions.enableSchemaHooks; - delete mongoOptions.maxTimeMS; + this.schemaCacheTtl = mongoOptions.schemaCacheTtl; + for (const key of ['enableSchemaHooks', 'schemaCacheTtl', 'maxTimeMS']) { + delete mongoOptions[key]; + delete this._mongoOptions[key]; + } } watch(callback: () => void): void { diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index 444e4e8cca..82ac0c20dc 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -850,13 +850,18 @@ export class PostgresStorageAdapter implements StorageAdapter { _pgp: any; _stream: any; _uuid: any; + schemaCacheTtl: ?number; constructor({ uri, collectionPrefix = '', databaseOptions = {} }: any) { + const options = { ...databaseOptions }; this._collectionPrefix = collectionPrefix; this.enableSchemaHooks = !!databaseOptions.enableSchemaHooks; - delete databaseOptions.enableSchemaHooks; + this.schemaCacheTtl = databaseOptions.schemaCacheTtl; + for (const key of ['enableSchemaHooks', 'schemaCacheTtl']) { + delete options[key]; + } - const { client, pgp } = createClient(uri, databaseOptions); + const { client, pgp } = createClient(uri, options); this._client = client; this._onchange = () => {}; this._pgp = pgp; diff --git a/src/Adapters/Storage/StorageAdapter.js b/src/Adapters/Storage/StorageAdapter.js index 6e4573b748..7605784a43 100644 --- a/src/Adapters/Storage/StorageAdapter.js +++ b/src/Adapters/Storage/StorageAdapter.js @@ -30,6 +30,8 @@ export type FullQueryOptions = QueryOptions & UpdateQueryOptions; export interface StorageAdapter { canSortOnJoinTables: boolean; + schemaCacheTtl: ?number; + enableSchemaHooks: boolean; classExists(className: string): Promise; setClassLevelPermissions(className: string, clps: any): Promise; diff --git a/src/Config.js b/src/Config.js index c993e467fb..2e7ef389c7 100644 --- a/src/Config.js +++ b/src/Config.js @@ -9,6 +9,7 @@ import DatabaseController from './Controllers/DatabaseController'; import { logLevels as validLogLevels } from './Controllers/LoggerController'; import { AccountLockoutOptions, + DatabaseOptions, FileUploadOptions, IdempotencyOptions, LogLevels, @@ -52,23 +53,20 @@ export class Config { } static put(serverConfiguration) { - Config.validate(serverConfiguration); + Config.validateOptions(serverConfiguration); + Config.validateControllers(serverConfiguration); AppCache.put(serverConfiguration.appId, serverConfiguration); Config.setupPasswordValidator(serverConfiguration.passwordPolicy); return serverConfiguration; } - static validate({ - verifyUserEmails, - userController, - appName, + static validateOptions({ publicServerURL, revokeSessionOnPasswordReset, expireInactiveSessions, sessionLength, defaultLimit, maxLimit, - emailVerifyTokenValidityDuration, accountLockout, passwordPolicy, masterKeyIps, @@ -78,7 +76,6 @@ export class Config { readOnlyMasterKey, allowHeaders, idempotencyOptions, - emailVerifyTokenReuseIfValid, fileUpload, pages, security, @@ -88,6 +85,7 @@ export class Config { allowExpiredAuthDataToken, logLevels, rateLimit, + databaseOptions, }) { if (masterKey === readOnlyMasterKey) { throw new Error('masterKey and readOnlyMasterKey should be different'); @@ -97,17 +95,6 @@ export class Config { throw new Error('masterKey and maintenanceKey should be different'); } - const emailAdapter = userController.adapter; - if (verifyUserEmails) { - this.validateEmailConfiguration({ - emailAdapter, - appName, - publicServerURL, - emailVerifyTokenValidityDuration, - emailVerifyTokenReuseIfValid, - }); - } - this.validateAccountLockoutPolicy(accountLockout); this.validatePasswordPolicy(passwordPolicy); this.validateFileUploadOptions(fileUpload); @@ -136,6 +123,27 @@ export class Config { this.validateRequestKeywordDenylist(requestKeywordDenylist); this.validateRateLimit(rateLimit); this.validateLogLevels(logLevels); + this.validateDatabaseOptions(databaseOptions); + } + + static validateControllers({ + verifyUserEmails, + userController, + appName, + publicServerURL, + emailVerifyTokenValidityDuration, + emailVerifyTokenReuseIfValid, + }) { + const emailAdapter = userController.adapter; + if (verifyUserEmails) { + this.validateEmailConfiguration({ + emailAdapter, + appName, + publicServerURL, + emailVerifyTokenValidityDuration, + emailVerifyTokenReuseIfValid, + }); + } } static validateRequestKeywordDenylist(requestKeywordDenylist) { @@ -533,6 +541,25 @@ export class Config { } } + static validateDatabaseOptions(databaseOptions) { + if (databaseOptions == undefined) { + return; + } + if (Object.prototype.toString.call(databaseOptions) !== '[object Object]') { + throw `databaseOptions must be an object`; + } + if (databaseOptions.enableSchemaHooks === undefined) { + databaseOptions.enableSchemaHooks = DatabaseOptions.enableSchemaHooks.default; + } else if (typeof databaseOptions.enableSchemaHooks !== 'boolean') { + throw `databaseOptions.enableSchemaHooks must be a boolean`; + } + if (databaseOptions.schemaCacheTtl === undefined) { + databaseOptions.schemaCacheTtl = DatabaseOptions.schemaCacheTtl.default; + } else if (typeof databaseOptions.schemaCacheTtl !== 'number') { + throw `databaseOptions.schemaCacheTtl must be a number`; + } + } + static validateRateLimit(rateLimit) { if (!rateLimit) { return; diff --git a/src/Controllers/SchemaController.js b/src/Controllers/SchemaController.js index 62757d251d..ad3699aaa5 100644 --- a/src/Controllers/SchemaController.js +++ b/src/Controllers/SchemaController.js @@ -682,6 +682,10 @@ const typeToString = (type: SchemaField | string): string => { } return `${type.type}`; }; +const ttl = { + date: Date.now(), + duration: undefined, +}; // Stores the entire schema of the app in a weird hybrid format somewhere between // the mongo format and the Parse format. Soon, this will all be Parse format. @@ -694,10 +698,11 @@ export default class SchemaController { constructor(databaseAdapter: StorageAdapter) { this._dbAdapter = databaseAdapter; + const config = Config.get(Parse.applicationId); this.schemaData = new SchemaData(SchemaCache.all(), this.protectedFields); - this.protectedFields = Config.get(Parse.applicationId).protectedFields; + this.protectedFields = config.protectedFields; - const customIds = Config.get(Parse.applicationId).allowCustomObjectId; + const customIds = config.allowCustomObjectId; const customIdRegEx = /^.{1,}$/u; // 1+ chars const autoIdRegEx = /^[a-zA-Z0-9]{1,}$/; @@ -709,6 +714,21 @@ export default class SchemaController { }); } + async reloadDataIfNeeded() { + if (this._dbAdapter.enableSchemaHooks) { + return; + } + const { date, duration } = ttl || {}; + if (!duration) { + return; + } + const now = Date.now(); + if (now - date > duration) { + ttl.date = now; + await this.reloadData({ clearCache: true }); + } + } + reloadData(options: LoadSchemaOptions = { clearCache: false }): Promise { if (this.reloadDataPromise && !options.clearCache) { return this.reloadDataPromise; @@ -729,10 +749,11 @@ export default class SchemaController { return this.reloadDataPromise; } - getAllClasses(options: LoadSchemaOptions = { clearCache: false }): Promise> { + async getAllClasses(options: LoadSchemaOptions = { clearCache: false }): Promise> { if (options.clearCache) { return this.setAllClasses(); } + await this.reloadDataIfNeeded(); const cached = SchemaCache.all(); if (cached && cached.length) { return Promise.resolve(cached); @@ -1440,6 +1461,7 @@ export default class SchemaController { // Returns a promise for a new Schema. const load = (dbAdapter: StorageAdapter, options: any): Promise => { const schema = new SchemaController(dbAdapter); + ttl.duration = dbAdapter.schemaCacheTtl; return schema.reloadData(options).then(() => schema); }; diff --git a/src/Options/Definitions.js b/src/Options/Definitions.js index 2d53cc7c27..a25e69c70a 100644 --- a/src/Options/Definitions.js +++ b/src/Options/Definitions.js @@ -971,6 +971,12 @@ module.exports.DatabaseOptions = { action: parsers.booleanParser, default: false, }, + schemaCacheTtl: { + env: 'PARSE_SERVER_DATABASE_SCHEMA_CACHE_TTL', + help: + 'The duration in seconds after which the schema cache expires and will be refetched from the database. Use this option if using multiple Parse Servers instances connected to the same database. A low duration will cause the schema cache to be updated too often, causing unnecessary database reads. A high duration will cause the schema to be updated too rarely, increasing the time required until schema changes propagate to all server instances. This feature can be used as an alternative or in conjunction with the option `enableSchemaHooks`. Default is infinite which means the schema cache never expires.', + action: parsers.numberParser('schemaCacheTtl'), + }, }; module.exports.AuthAdapter = { enabled: { diff --git a/src/Options/docs.js b/src/Options/docs.js index 0f28270f56..0eb6488c74 100644 --- a/src/Options/docs.js +++ b/src/Options/docs.js @@ -225,6 +225,7 @@ /** * @interface DatabaseOptions * @property {Boolean} enableSchemaHooks Enables database real-time hooks to update single schema cache. Set to `true` if using multiple Parse Servers instances connected to the same database. Failing to do so will cause a schema change to not propagate to all instances and re-syncing will only happen when the instances restart. To use this feature with MongoDB, a replica set cluster with [change stream](https://docs.mongodb.com/manual/changeStreams/#availability) support is required. + * @property {Number} schemaCacheTtl The duration in seconds after which the schema cache expires and will be refetched from the database. Use this option if using multiple Parse Servers instances connected to the same database. A low duration will cause the schema cache to be updated too often, causing unnecessary database reads. A high duration will cause the schema to be updated too rarely, increasing the time required until schema changes propagate to all server instances. This feature can be used as an alternative or in conjunction with the option `enableSchemaHooks`. Default is infinite which means the schema cache never expires. */ /** diff --git a/src/Options/index.js b/src/Options/index.js index 6d0f488b88..778374e7e7 100644 --- a/src/Options/index.js +++ b/src/Options/index.js @@ -548,6 +548,8 @@ export interface DatabaseOptions { /* Enables database real-time hooks to update single schema cache. Set to `true` if using multiple Parse Servers instances connected to the same database. Failing to do so will cause a schema change to not propagate to all instances and re-syncing will only happen when the instances restart. To use this feature with MongoDB, a replica set cluster with [change stream](https://docs.mongodb.com/manual/changeStreams/#availability) support is required. :DEFAULT: false */ enableSchemaHooks: ?boolean; + /* The duration in seconds after which the schema cache expires and will be refetched from the database. Use this option if using multiple Parse Servers instances connected to the same database. A low duration will cause the schema cache to be updated too often, causing unnecessary database reads. A high duration will cause the schema to be updated too rarely, increasing the time required until schema changes propagate to all server instances. This feature can be used as an alternative or in conjunction with the option `enableSchemaHooks`. Default is infinite which means the schema cache never expires. */ + schemaCacheTtl: ?number; } export interface AuthAdapter { diff --git a/src/ParseServer.js b/src/ParseServer.js index ed21ce12e3..04379ecfd3 100644 --- a/src/ParseServer.js +++ b/src/ParseServer.js @@ -71,6 +71,7 @@ class ParseServer { Parse.initialize(appId, javascriptKey || 'unused', masterKey); Parse.serverURL = serverURL; + Config.validateOptions(options); const allControllers = controllers.getControllers(options); options.state = 'initialized'; this.config = Config.put(Object.assign({}, options, allControllers));