feat: add user-defined schema and migrations (#7418)

spec/schemas.spec.js

@@ -759,7 +759,7 @@ describe('schemas', () => {
     });
   });
 
-  it('refuses to put to existing fields, even if it would not be a change', done => {
+  it('refuses to put to existing fields with different type, even if it would not be a change', done => {
     const obj = hasAllPODobject();
     obj.save().then(() => {
       request({
@@ -769,7 +769,7 @@ describe('schemas', () => {
         json: true,
         body: {
           fields: {
-            aString: { type: 'String' },
+            aString: { type: 'Number' },
           },
         },
       }).then(fail, response => {

src/Adapters/Storage/Mongo/MongoSchemaCollection.js

@@ -212,7 +212,7 @@ class MongoSchemaCollection {
       .then(
         schema => {
           // If a field with this name already exists, it will be handled elsewhere.
-          if (schema.fields[fieldName] != undefined) {
+          if (schema.fields[fieldName] !== undefined) {
             return;
           }
           // The schema exists. Check for existing GeoPoints.
@@ -274,6 +274,22 @@ class MongoSchemaCollection {
         }
       });
   }
+
+  async updateFieldOptions(className: string, fieldName: string, fieldType: any) {
+    const { ...fieldOptions } = fieldType;
+    delete fieldOptions.type;
+    delete fieldOptions.targetClass;
+
+    await this.upsertSchema(
+      className,
+      { [fieldName]: { $exists: true } },
+      {
+        $set: {
+          [`_metadata.fields_options.${fieldName}`]: fieldOptions,
+        },
+      }
+    );
+  }
 }
 
 // Exported for testing reasons and because we haven't moved all mongo schema format

src/Adapters/Storage/Mongo/MongoStorageAdapter.js

@@ -362,6 +362,11 @@ export class MongoStorageAdapter implements StorageAdapter {
       .catch(err => this.handleError(err));
   }
 
+  async updateFieldOptions(className: string, fieldName: string, type: any) {
+    const schemaCollection = await this._schemaCollection();
+    await schemaCollection.updateFieldOptions(className, fieldName, type);
+  }
+
   addFieldIfNotExists(className: string, fieldName: string, type: any): Promise<void> {
     return this._schemaCollection()
       .then(schemaCollection => schemaCollection.addFieldIfNotExists(className, fieldName, type))

src/Adapters/Storage/Postgres/PostgresClient.js

@@ -20,7 +20,7 @@ export function createClient(uri, databaseOptions) {
 
   if (process.env.PARSE_SERVER_LOG_LEVEL === 'debug') {
     const monitor = require('pg-monitor');
-    if(monitor.isAttached()) {
+    if (monitor.isAttached()) {
       monitor.detach();
     }
     monitor.attach(initOptions);

src/Adapters/Storage/Postgres/PostgresStorageAdapter.js

@@ -1119,6 +1119,16 @@ export class PostgresStorageAdapter implements StorageAdapter {
     this._notifySchemaChange();
   }
 
+  async updateFieldOptions(className: string, fieldName: string, type: any) {
+    await this._client.tx('update-schema-field-options', async t => {
+      const path = `{fields,${fieldName}}`;
+      await t.none(
+        'UPDATE "_SCHEMA" SET "schema"=jsonb_set("schema", $<path>, $<type>)  WHERE "className"=$<className>',
+        { path, type, className }
+      );
+    });
+  }
+
   // Drops a collection. Resolves with true if it was a Parse Schema (eg. _User, Custom, etc.)
   // and resolves with false if it wasn't (eg. a join table). Rejects if deletion was impossible.
   async deleteClass(className: string) {

src/Adapters/Storage/StorageAdapter.js

@@ -35,6 +35,7 @@ export interface StorageAdapter {
   setClassLevelPermissions(className: string, clps: any): Promise<void>;
   createClass(className: string, schema: SchemaType): Promise<void>;
   addFieldIfNotExists(className: string, fieldName: string, type: any): Promise<void>;
+  updateFieldOptions(className: string, fieldName: string, type: any): Promise<void>;
   deleteClass(className: string): Promise<void>;
   deleteAllClasses(fast: boolean): Promise<void>;
   deleteFields(className: string, schema: SchemaType, fieldNames: Array<string>): Promise<void>;

src/Config.js

@@ -11,6 +11,7 @@ import {
   AccountLockoutOptions,
   PagesOptions,
   SecurityOptions,
+  SchemaOptions,
 } from './Options/Definitions';
 import { isBoolean, isString } from 'lodash';
 
@@ -76,6 +77,7 @@ export class Config {
     pages,
     security,
     enforcePrivateUsers,
+    schema,
   }) {
     if (masterKey === readOnlyMasterKey) {
       throw new Error('masterKey and readOnlyMasterKey should be different');
@@ -112,6 +114,7 @@ export class Config {
     this.validateIdempotencyOptions(idempotencyOptions);
     this.validatePagesOptions(pages);
     this.validateSecurityOptions(security);
+    this.validateSchemaOptions(schema);
     this.validateEnforcePrivateUsers(enforcePrivateUsers);
   }
 
@@ -137,6 +140,48 @@ export class Config {
     }
   }
 
+  static validateSchemaOptions(schema: SchemaOptions) {
+    if (!schema) return;
+    if (Object.prototype.toString.call(schema) !== '[object Object]') {
+      throw 'Parse Server option schema must be an object.';
+    }
+    if (schema.definitions === undefined) {
+      schema.definitions = SchemaOptions.definitions.default;
+    } else if (!Array.isArray(schema.definitions)) {
+      throw 'Parse Server option schema.definitions must be an array.';
+    }
+    if (schema.strict === undefined) {
+      schema.strict = SchemaOptions.strict.default;
+    } else if (!isBoolean(schema.strict)) {
+      throw 'Parse Server option schema.strict must be a boolean.';
+    }
+    if (schema.deleteExtraFields === undefined) {
+      schema.deleteExtraFields = SchemaOptions.deleteExtraFields.default;
+    } else if (!isBoolean(schema.deleteExtraFields)) {
+      throw 'Parse Server option schema.deleteExtraFields must be a boolean.';
+    }
+    if (schema.recreateModifiedFields === undefined) {
+      schema.recreateModifiedFields = SchemaOptions.recreateModifiedFields.default;
+    } else if (!isBoolean(schema.recreateModifiedFields)) {
+      throw 'Parse Server option schema.recreateModifiedFields must be a boolean.';
+    }
+    if (schema.lockSchemas === undefined) {
+      schema.lockSchemas = SchemaOptions.lockSchemas.default;
+    } else if (!isBoolean(schema.lockSchemas)) {
+      throw 'Parse Server option schema.lockSchemas must be a boolean.';
+    }
+    if (schema.beforeMigration === undefined) {
+      schema.beforeMigration = null;
+    } else if (schema.beforeMigration !== null && typeof schema.beforeMigration !== 'function') {
+      throw 'Parse Server option schema.beforeMigration must be a function.';
+    }
+    if (schema.afterMigration === undefined) {
+      schema.afterMigration = null;
+    } else if (schema.afterMigration !== null && typeof schema.afterMigration !== 'function') {
+      throw 'Parse Server option schema.afterMigration must be a function.';
+    }
+  }
+
   static validatePagesOptions(pages) {
     if (Object.prototype.toString.call(pages) !== '[object Object]') {
       throw 'Parse Server option pages must be an object.';

src/Controllers/SchemaController.js

@@ -831,7 +831,11 @@ export default class SchemaController {
         const existingFields = schema.fields;
         Object.keys(submittedFields).forEach(name => {
           const field = submittedFields[name];
-          if (existingFields[name] && field.__op !== 'Delete') {
+          if (
+            existingFields[name] &&
+            existingFields[name].type !== field.type &&
+            field.__op !== 'Delete'
+          ) {
             throw new Parse.Error(255, `Field ${name} exists, cannot update.`);
           }
           if (!existingFields[name] && field.__op === 'Delete') {
@@ -1057,7 +1061,12 @@ export default class SchemaController {
   // object if the provided className-fieldName-type tuple is valid.
   // The className must already be validated.
   // If 'freeze' is true, refuse to update the schema for this field.
-  enforceFieldExists(className: string, fieldName: string, type: string | SchemaField) {
+  enforceFieldExists(
+    className: string,
+    fieldName: string,
+    type: string | SchemaField,
+    isValidation?: boolean
+  ) {
     if (fieldName.indexOf('.') > 0) {
       // subdocument key (x.y) => ok if x is of type 'object'
       fieldName = fieldName.split('.')[0];
@@ -1101,7 +1110,14 @@ export default class SchemaController {
           )} but got ${typeToString(type)}`
         );
       }
-      return undefined;
+      // If type options do not change
+      // we can safely return
+      if (isValidation || JSON.stringify(expectedType) === JSON.stringify(type)) {
+        return undefined;
+      }
+      // Field options are may be changed
+      // ensure to have an update to date schema field
+      return this._dbAdapter.updateFieldOptions(className, fieldName, type);
     }
 
     return this._dbAdapter
@@ -1236,7 +1252,7 @@ export default class SchemaController {
         // Every object has ACL implicitly.
         continue;
       }
-      promises.push(schema.enforceFieldExists(className, fieldName, expected));
+      promises.push(schema.enforceFieldExists(className, fieldName, expected, true));
     }
     const results = await Promise.all(promises);
     const enforceFields = results.filter(result => !!result);

src/Options/Definitions.js

@@ -446,6 +446,45 @@ module.exports.SecurityOptions = {
     default: false,
   },
 };
+module.exports.SchemaOptions = {
+  definitions: {
+    help: 'The schema definitions.',
+    default: [],
+  },
+  strict: {
+    env: 'PARSE_SERVER_SCHEMA_STRICT',
+    help: 'Is true if Parse Server should exit if schema update fail.',
+    action: parsers.booleanParser,
+    default: true,
+  },
+  deleteExtraFields: {
+    env: 'PARSE_SERVER_SCHEMA_DELETE_EXTRA_FIELDS',
+    help:
+      'Is true if Parse Server should delete any fields not defined in a schema definition. This should only be used during development.',
+    action: parsers.booleanParser,
+    default: false,
+  },
+  recreateModifiedFields: {
+    env: 'PARSE_SERVER_SCHEMA_RECREATE_MODIFIED_FIELDS',
+    help:
+      'Is true if Parse Server should recreate any fields that are different between the current database schema and theschema definition. This should only be used during development.',
+    action: parsers.booleanParser,
+    default: false,
+  },
+  lockSchemas: {
+    env: 'PARSE_SERVER_SCHEMA_LOCK',
+    help:
+      'Is true if Parse Server will reject any attempts to modify the schema while the server is running.',
+    action: parsers.booleanParser,
+    default: false,
+  },
+  beforeMigration: {
+    help: 'Execute a callback before running schema migrations.',
+  },
+  afterMigration: {
+    help: 'Execute a callback after running schema migrations.',
+  },
+};
 module.exports.PagesOptions = {
   customRoutes: {
     env: 'PARSE_SERVER_PAGES_CUSTOM_ROUTES',

src/Options/index.js

@@ -1,3 +1,4 @@
+// @flow
 import { AnalyticsAdapter } from '../Adapters/Analytics/AnalyticsAdapter';
 import { FilesAdapter } from '../Adapters/Files/FilesAdapter';
 import { LoggerAdapter } from '../Adapters/Logger/LoggerAdapter';
@@ -7,8 +8,8 @@ import { MailAdapter } from '../Adapters/Email/MailAdapter';
 import { PubSubAdapter } from '../Adapters/PubSub/PubSubAdapter';
 import { WSSAdapter } from '../Adapters/WebSocketServer/WSSAdapter';
 import { CheckGroup } from '../Security/CheckGroup';
+import type { SchemaOptions } from '../SchemaMigrations/Migrations';
 
-// @flow
 type Adapter<T> = string | any | T;
 type NumberOrBoolean = number | boolean;
 type NumberOrString = number | string;
@@ -241,6 +242,8 @@ export interface ParseServerOptions {
   playgroundPath: ?string;
   /* Callback when server has started */
   serverStartComplete: ?(error: ?Error) => void;
+  /* Rest representation on Parse.Schema https://docs.parseplatform.org/rest/guide/#adding-a-schema */
+  schema: ?SchemaOptions;
   /* Callback when server has closed */
   serverCloseComplete: ?() => void;
   /* The security options to identify and report weak security settings.

src/ParseServer.js

@@ -44,6 +44,7 @@ import { ParseGraphQLServer } from './GraphQL/ParseGraphQLServer';
 import { SecurityRouter } from './Routers/SecurityRouter';
 import CheckRunner from './Security/CheckRunner';
 import Deprecator from './Deprecator/Deprecator';
+import { DefinedSchemas } from './SchemaMigrations/DefinedSchemas';
 
 // Mutate the Parse object to add the Cloud Code handlers
 addParseCloud();
@@ -68,6 +69,7 @@ class ParseServer {
       javascriptKey,
       serverURL = requiredParameter('You must provide a serverURL!'),
       serverStartComplete,
+      schema,
     } = options;
     // Initialize the node client SDK automatically
     Parse.initialize(appId, javascriptKey || 'unused', masterKey);
@@ -84,7 +86,10 @@ class ParseServer {
     databaseController
       .performInitialization()
       .then(() => hooksController.load())
-      .then(() => {
+      .then(async () => {
+        if (schema) {
+          await new DefinedSchemas(schema, this.config).execute();
+        }
         if (serverStartComplete) {
           serverStartComplete();
         }