Add support for the "loudness" block and "loudness greater than" trigger

src/Loudness.js

@@ -0,0 +1,74 @@
+import Sound from "./Sound.js";
+
+const IGNORABLE_ERROR = ["NotAllowedError", "NotFoundError"];
+
+// https://github.com/LLK/scratch-audio/blob/develop/src/Loudness.js
+export default class LoudnessHandler {
+  constructor() {
+    // TODO: use a TypeScript enum
+    this.connectionState = "NOT_CONNECTED";
+  }
+
+  get audioContext() {
+    return Sound.audioContext;
+  }
+
+  async connect() {
+    // If we're in the middle of connecting, or failed to connect,
+    // don't attempt to connect again
+    if (this.connectionState !== "NOT_CONNECTED") return;
+    this.connectionState = "CONNECTING";
+
+    try {
+      const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+      // Chrome blocks usage of audio until the user interacts with the page.
+      // By calling `resume` here, we will wait until that happens.
+      await Sound.audioContext.resume();
+      this.hasConnected = true;
+      this.audioStream = stream;
+      const mic = this.audioContext.createMediaStreamSource(stream);
+      this.analyser = this.audioContext.createAnalyser();
+      mic.connect(this.analyser);
+      this.micDataArray = new Float32Array(this.analyser.fftSize);
+      this.connectionState = "CONNECTED";
+    } catch (e) {
+      this.connectionState = "ERROR";
+      if (IGNORABLE_ERROR.includes(e.name)) {
+        console.warn("Mic is not available.");
+      } else {
+        throw e;
+      }
+    }
+  }
+
+  get loudness() {
+    if (this.connectionState !== "CONNECTED" || !this.audioStream.active) {
+      return -1;
+    }
+
+    this.analyser.getFloatTimeDomainData(this.micDataArray);
+    let sum = 0;
+    // compute the RMS of the sound
+    for (let i = 0; i < this.micDataArray.length; i++) {
+      sum += Math.pow(this.micDataArray[i], 2);
+    }
+    let rms = Math.sqrt(sum / this.micDataArray.length);
+    // smoothe the value with the last one, if it is descending
+    if (this._lastValue) {
+      rms = Math.max(rms, this._lastValue * 0.6);
+    }
+    this._lastValue = rms;
+
+    // scale the measurement so it's more sensitive to quieter sounds
+    rms *= 1.63;
+    rms = Math.sqrt(rms);
+    rms = Math.round(rms * 100);
+    rms = Math.min(rms, 100);
+    return rms;
+  }
+
+  getLoudness() {
+    this.connect();
+    return this.loudness;
+  }
+}

src/Project.js

@@ -1,6 +1,8 @@
 import Trigger from "./Trigger.js";
 import Renderer from "./Renderer.js";
 import Input from "./Input.js";
+import LoudnessHandler from "./Loudness.js";
+import Sound from "./Sound.js";
 
 export default class Project {
   constructor(stage, sprites = {}, { frameRate = 30 } = {}) {
@@ -19,6 +21,10 @@ export default class Project {
       this.fireTrigger(Trigger.KEY_PRESSED, { key });
     });
 
+    this.loudnessHandler = new LoudnessHandler();
+    // Only update loudness once per step.
+    this._cachedLoudness = null;
+
     this.runningTriggers = [];
     // Used to keep track of what edge-activated trigger predicates evaluted to
     // on the previous step.
@@ -40,6 +46,14 @@ export default class Project {
   attach(renderTarget) {
     this.renderer.setRenderTarget(renderTarget);
     this.renderer.stage.addEventListener("click", () => {
+      // Chrome requires a user gesture on the page before we can start the
+      // audio context.
+      // When we click the stage, that counts as a user gesture, so try
+      // resuming the audio context.
+      if (Sound.audioContext.state === "suspended") {
+        Sound.audioContext.resume();
+      }
+
       let clickedSprite = this.renderer.pick(this.spritesAndClones, {
         x: this.input.mouse.x,
         y: this.input.mouse.y
@@ -60,6 +74,13 @@ export default class Project {
   }
 
   greenFlag() {
+    // Chrome requires a user gesture on the page before we can start the
+    // audio context.
+    // When greenFlag is triggered, it's likely that the cause of it was some
+    // kind of button click, so try resuming the audio context.
+    if (Sound.audioContext.state === "suspended") {
+      Sound.audioContext.resume();
+    }
     this.fireTrigger(Trigger.GREEN_FLAG);
     this.input.focus();
   }
@@ -89,6 +110,9 @@ export default class Project {
         case Trigger.TIMER_GREATER_THAN:
           predicate = this.timer > trigger.option("VALUE", target);
           break;
+        case Trigger.LOUDNESS_GREATER_THAN:
+          predicate = this.loudness > trigger.option("VALUE", target);
+          break;
         default:
           throw new Error(`Unimplemented trigger ${trigger.trigger}`);
       }
@@ -107,6 +131,7 @@ export default class Project {
   }
 
   step() {
+    this._cachedLoudness = null;
     this._stepEdgeActivatedTriggers();
 
     // Step all triggers
@@ -235,4 +260,11 @@ export default class Project {
   async askAndWait(question) {
     this.answer = await this.renderer.displayAskBox(question);
   }
+
+  get loudness() {
+    if (this._cachedLoudness === null) {
+      this._cachedLoudness = this.loudnessHandler.getLoudness();
+    }
+    return this._cachedLoudness;
+  }
 }

src/Sprite.js

@@ -313,6 +313,10 @@ class SpriteBase {
   get answer() {
     return this._project.answer;
   }
+
+  get loudness() {
+    return this._project.loudness;
+  }
 }
 
 export class Sprite extends SpriteBase {

src/Trigger.js

@@ -3,6 +3,7 @@ const KEY_PRESSED = Symbol("KEY_PRESSED");
 const BROADCAST = Symbol("BROADCAST");
 const CLICKED = Symbol("CLICKED");
 const CLONE_START = Symbol("CLONE_START");
+const LOUDNESS_GREATER_THAN = Symbol("LOUDNESS_GREATER_THAN");
 const TIMER_GREATER_THAN = Symbol("TIMER_GREATER_THAN");
 
 export default class Trigger {
@@ -22,7 +23,10 @@ export default class Trigger {
   }
 
   get isEdgeActivated() {
-    return this.trigger === TIMER_GREATER_THAN;
+    return (
+      this.trigger === TIMER_GREATER_THAN ||
+      this.trigger === LOUDNESS_GREATER_THAN
+    );
   }
 
   // Evaluate the given trigger option, whether it's a value or a function that
@@ -82,6 +86,9 @@ export default class Trigger {
   static get CLONE_START() {
     return CLONE_START;
   }
+  static get LOUDNESS_GREATER_THAN() {
+    return LOUDNESS_GREATER_THAN;
+  }
   static get TIMER_GREATER_THAN() {
     return TIMER_GREATER_THAN;
   }