Super scary extension development improvments.

src/extension-support/extension-manager.js

@@ -130,6 +130,42 @@ class ExtensionManager {
         dispatch.setService('extensions', createExtensionService(this)).catch(e => {
             log.error(`ExtensionManager was unable to register extension service: ${JSON.stringify(e)}`);
         });
+
+        // Allow for sandboxed extensions, and worker extensions to access some of our APIs.
+        // TODO: This works outside of extensions. We may be able to use port's to limit this
+        //       to extensions. if we even want to.
+        // Extensions are'nt ran when using the test's / CLI so we can test if addEventListener exists,
+        // and only then if it does do we attempt to add the message listener.
+        if (global.addEventListener) global.addEventListener('message', this._messageListener);
+    }
+
+    /**
+     * Callback for when a MessageEvent is received.
+     * @private
+     */
+    _messageListener ({data}) {
+        try {
+            data = JSON.parse(data);
+        } catch {
+            return;
+        }
+        // Make sure this is coming from someone who knows the API. eg- us. (doesn't have to be perfect)
+        if (data.TW_extensionAPI !== true) return;
+        // Validation of the message.
+        if ((typeof data.TW_command) !== 'object') return;
+        const post = data.TW_command;
+        if (typeof post.type !== 'string') return;
+        if (!Array.isArray(post.args)) return;
+        switch (post.type) {
+        case 'refreshBlocks':
+            this.refreshBlocks(post.args[0]);
+            break;
+        case 'loadExtensionIdSync':
+            this.loadExtensionIdSync(post.args[0]);
+            break;
+        default:
+            console.warn('Unknown extension API call: ', data.TW_command);
+        }
     }
 
     /**

src/extension-support/extension-worker.js

@@ -96,7 +96,31 @@ Object.assign(global.Scratch, ScratchCommon, {
  */
 const extensionWorker = new ExtensionWorker();
 global.Scratch.extensions = {
+    worker: true, // TW: To match the other stuff like unsandboxed.
     register: extensionWorker.register.bind(extensionWorker)
 };
 
+// TW: Allow for some specific VM APIs that are considered safe,
+//     to be used by extensions in this context.
+global.Scratch.extensions.refresh = id => {
+    if (!global.parent) return;
+    global.parent.postMessage(JSON.stringify({
+        TW_extensionAPI: true,
+        TW_command: {
+            type: 'refreshBlocks',
+            args: [id]
+        }
+    }), '*');
+};
+global.Scratch.extensions.loadBuiltIn = id => {
+    if (!global.parent) return;
+    global.parent.postMessage(JSON.stringify({
+        TW_extensionAPI: true,
+        TW_command: {
+            type: 'loadExtensionIdSync',
+            args: [id]
+        }
+    }), '*');
+};
+
 global.ScratchExtensions = createScratchX(global.Scratch);

src/extension-support/tools.js

@@ -0,0 +1,46 @@
+/**
+ * @fileoverview Responsible for giving unsandboxed extensions their extra tools.
+ * This is exposed on the VM for ease of use and to prevent duplicate code.
+ */
+
+class ExtensionTools {
+    // Internal functions.
+    static xmlEscape = require('../util/xml-escape');
+    static uid = require('../util/uid');
+    static fetchStatic = require('../util/tw-static-fetch');
+    static fetchWithTimeout = require('../util/fetch-with-timeout').fetchWithTimeout;
+    static setFetchWithTimeoutFetchFn = require('../util/fetch-with-timeout').setFetch;
+    static isNotActuallyZero = require('../util/is-not-actually-zero');
+    static getMonitorID = require('../util/get-monitor-id');
+    static maybeFormatMessage = require('../util/maybe-format-message');
+    static newBlockIds = require('../util/new-block-ids');
+    static hasOwn = (object, key) => Object.prototype.hasOwnProperty.call(object, key);
+    // External classes / functions.
+    static nanolog = require('@turbowarp/nanolog');
+    static log = require('../util/log');
+    static buffers = require('buffer');
+    static TextEncoder = require('text-encoding').TextEncoder;
+    static TextDecoder = require('text-encoding').TextDecoder;
+    static twjson = require('@turbowarp/json');
+    // Internal classes.
+    static math = require('../util/math-util');
+    static assets = require('../util/tw-asset-util');
+    static base64 = require('../util/base64-util');
+    static strings = require('../util/string-util');
+    static variables = require('../util/variable-util');
+    static asyncLimiter = require('../util/async-limiter');
+    static clone = require('../util/clone');
+    static sanitizer = require('../util/value-sanitizer');
+    static jsonrpc = require('../util/jsonrpc');
+    static color = require('../util/color');
+    static rateLimiter = require('../util/rateLimiter');
+    static scratchLinkWebSocket = require('../util/scratch-link-websocket');
+    static taskQueue = require('../util/task-queue');
+    static timer = require('../util/timer');
+    static sharedDispatch = require('../dispatch/shared-dispatch');
+    // Instanced dispatchers.
+    static centralDispatch = require('../dispatch/central-dispatch');
+    static workerDispatch = require('../dispatch/worker-dispatch');
+}
+
+module.exports = ExtensionTools;

src/extension-support/tw-unsandboxed-extension-runner.js

@@ -152,8 +152,22 @@ const setupUnsandboxedExtensionAPI = vm => new Promise(resolve => {
         link.remove();
     };
 
+    // Mappings to extensionManager functions for parity with worker and sandboxed extensions.
+    Scratch.extensions.refresh = id => vm.extensionManager.refreshBlocks(id);
+    Scratch.extensions.loadBuiltIn = id => vm.extensionManager.loadExtensionIdSync(id);
+
     Scratch.translate = createTranslate(vm);
 
+    // Make the tools export lazy to save resources when none of the extra tools are used.
+    Object.defineProperty(Scratch, 'tools', {
+        get () {
+            return require('../extension-support/tools');
+        },
+        set () {
+            throw new Error('Not writable, nice try.');
+        }
+    });
+
     global.Scratch = Scratch;
     global.ScratchExtensions = createScratchX(Scratch);
 

src/util/cast.js

@@ -1,4 +1,6 @@
-const Color = require('../util/color');
+const Color = require('./color');
+const Sanitizer = require('./value-sanitizer');
+const isNotActuallyZero = require('./is-not-actually-zero');
 
 /**
  * @fileoverview
@@ -11,27 +13,6 @@ const Color = require('../util/color');
  * Use when coercing a value before computation.
  */
 
-/**
- * Used internally by compare()
- * @param {*} val A value that evaluates to 0 in JS string-to-number conversation such as empty string, 0, or tab.
- * @returns {boolean} True if the value should not be treated as the number zero.
- */
-const isNotActuallyZero = val => {
-    if (typeof val !== 'string') return false;
-    for (let i = 0; i < val.length; i++) {
-        const code = val.charCodeAt(i);
-        // '0'.charCodeAt(0) === 48
-        // '\t'.charCodeAt(0) === 9
-        // We include tab for compatibility with scratch-www's broken trim() polyfill.
-        // https://github.com/TurboWarp/scratch-vm/issues/115
-        // https://scratch.mit.edu/projects/788261699/
-        if (code === 48 || code === 9) {
-            return false;
-        }
-    }
-    return true;
-};
-
 class Cast {
     /**
      * Scratch cast to number.
@@ -234,6 +215,156 @@ class Cast {
         }
         return index;
     }
+
+    // TW: More casting tool's meant for extensions.
+
+    /**
+     * Convert's a translatable menu value to a index or a string.
+     * @param {string|number} value Item in valid format.
+     * @param {number} [count] Modulo value. this is optional.
+     * @param {string[]} [valid] Option valid options. in all lowercase.
+     * @returns {string} The casted option.
+     * NOTE: If you use valid, make sure you handle translation support correctly;
+     *       make sure the items are all lowercase.
+     */
+    asNumberedItem (value, count, valid) {
+        // eslint-disable-next-line spaced-comment
+        /**!
+         * This code is modified and borrowed from the following code.
+         * @see https://raw.githubusercontent.com/surv-is-a-dev/surv-is-a-dev.github.io/fa453c76e2bcc2ceed87cc4c9af4ee2951886139/static/0001tt.txt
+         * It is used as a reference to handle the numbered dropdown values.
+         */
+        if (typeof value === 'number') {
+            value = ((+value % (1 + count || 1)) || 1);
+            if (!valid) return value.toString();
+            // Disable === null and === checks because `== null` passes if the value is
+            // null or undefined, which is what we want.
+            // eslint-disable-next-line no-eq-null, eqeqeq
+            if (valid[value] == null) return '1'; // Fallback to 1 if the value is null / undefined.
+            return value.toString(); // The index is valid so we can just return it.
+        }
+        value = this.toString(value).toLowerCase();
+        if (value[0] !== '(') return value;
+        const match = value.match(/^\([0-9]+\) ?/); // The space is optional for the sake of ease.
+        if (match && match[0]) {
+            let v = parseInt(match[0].trim().slice(1, -1), 10);
+            if (count) v = ((v % (1 + count || 1)) || 1);
+            if (!valid) return v.toString();
+            // See above.
+            // eslint-disable-next-line no-eq-null, eqeqeq
+            if (valid[value] == null) return '1';
+            return v.toString();
+        }
+        if (valid.indexOf(value.toLowerCase()) === -1) return '1'; // Fallback to 1 if the item is not valid.
+        return value;
+    }
+
+    /**
+     * Determine if a Scratch argument number represents a big integer. (BigInt)
+     * This treats normal integers as valid BigInts. @see {isInt}
+     * @param {*} val Value to check.
+     * @return {boolean} True if number looks like an integer.
+     */
+    static isBigInt (val) {
+        return (typeof val === 'bigint') || this.isInt(val);
+    }
+
+    /**
+     * Scratch cast to BigInt.
+     * Treats NaN-likes as 0. Floats are truncated.
+     * @param {*} value Value to cast to BigInt.
+     * @return {bigint} The Scratch-casted BigInt value.
+     */
+    static toBigInt (value) {
+        // If the value is already a BigInt then we don't have to do anything.
+        if (typeof value === 'bigint') return value;
+        // Handle NaN like value's as BigInt will throw an error if it cannot coerce the value.
+        if (isNaN(value)) return 0n;
+        // Same with floats.
+        if (!this.isBigInt(value)) value = Math.trunc(value);
+        // eslint-disable-next-line no-undef
+        return BigInt(value);
+    }
+
+    /**
+     * Scratch cast to Object.
+     * @param {*} value Value to cast to Object.
+     * @param {boolean} [noBad] Should null and undefined be disabled? Defaults to false.
+     * @param {boolean} [nullAssign] See {Sanitizer.object}. Defaults to false.
+     * @return {!object} The Scratch-casted Object value.
+     */
+    static toObjectLike (value, noBad = false, nullAssign = false) {
+        // eslint-disable-next-line no-eq-null, eqeqeq
+        if (value == null && noBad) return nullAssign ? Object.create(null) : {};
+        if (typeof value === 'object') return noBad ? Sanitizer.value(value, '', nullAssign) : value;
+        if (typeof value !== 'string' || value === '') return nullAssign ? Object.create(null) : {};
+        try {
+            if (noBad) {
+                value = Sanitizer.parseJSON(value, '', nullAssign);
+            } else value = JSON.parse(value);
+        } catch {
+            value = nullAssign ? Object.create(null) : {};
+        }
+        return this.toObjectLike(value, noBad, nullAssign);
+    }
+
+    /**
+     * Scratch cast to an Object.
+     * Treats null, undefined and arrays as empty objects.
+     * @param {*} value Value to cast to Object.
+     * @return {!object} The Scratch-casted Object value.
+     */
+    static toObject (value) {
+        if (typeof value === 'object') {
+            // eslint-disable-next-line no-eq-null, eqeqeq
+            if (Array.isArray(value) || value == null) return Object.create(null);
+            return Sanitizer.object(value, '', true); // This doesn't take into account for other Object typed values.
+        }
+        return this.toObject(this.toObjectLike(value, true, true));
+    }
+
+    /**
+     * Scratch cast to an Array.
+     * Treats null, undefined and objects as empty arrays.
+     * @param {*} value Value to cast to Array.
+     * @return {array} The Scratch-casted Array value.
+     */
+    static toArray (value) {
+        if (Array.isArray(value)) return Sanitizer.array(value, '');
+        // eslint-disable-next-line no-eq-null, eqeqeq
+        if (typeof value === 'object' && value != null) {
+            try {
+                value = Array.from(value);
+            } catch {
+                value = [];
+            }
+            return this.toArray(value); // Just in case.
+        }
+        return this.toArray(this.toObjectLike(value, true, true));
+    }
+
+    /**
+     * Scratch cast to a Map.
+     * Treats null and undefined as empty maps.
+     * @param {*} value Value to cast to Map.
+     * @return {map} The Scratch-casted Map value.
+     * NOTE: This is an alternative to `toObject`.
+     */
+    static toMap (value) {
+        if (value instanceof Map) return Sanitizer.map(value, '', true);
+        // This is done to handle null / undefined values popping up in our values.
+        value = this.toObjectLike(Sanitizer.value(value, '', true), true);
+        try {
+            if (!Array.isArray(value)) {
+                if (typeof value === 'object') value = Object.entries(value);
+                else value = [];
+            }
+            value = this.toArray(value); // Cast the value to an array.
+            return Sanitizer.map(new Map(value), '', true);
+        } catch {
+            return new Map();
+        }
+    }
 }
 
 module.exports = Cast;

src/util/is-not-actually-zero.js

@@ -0,0 +1,22 @@
+/**
+ * Used internally by compare()
+ * @param {*} val A value that evaluates to 0 in JS string-to-number conversation such as empty string, 0, or tab.
+ * @returns {boolean} True if the value should not be treated as the number zero.
+ */
+const isNotActuallyZero = val => {
+    if (typeof val !== 'string') return false;
+    for (let i = 0; i < val.length; i++) {
+        const code = val.charCodeAt(i);
+        // '0'.charCodeAt(0) === 48
+        // '\t'.charCodeAt(0) === 9
+        // We include tab for compatibility with scratch-www's broken trim() polyfill.
+        // https://github.com/TurboWarp/scratch-vm/issues/115
+        // https://scratch.mit.edu/projects/788261699/
+        if (code === 48 || code === 9) {
+            return false;
+        }
+    }
+    return true;
+};
+
+module.exports = isNotActuallyZero;