Adding video motion sensing features

src/blocks.js

@@ -1710,6 +1710,38 @@ SyntaxElementMorph.prototype.labelPart = function (spec) {
                     new Point() : this.embossing;
             part.drawNew();
             break;
+        // Video motion
+        case '%vid': // video modes
+            part = new InputSlotMorph(
+                null,
+                false, {
+                    on: ['on'],
+                    off: ['off'],
+                    'on-flipped': ['on-flipped']
+                },
+                true
+            );
+            break;
+        case '%motype':
+            part = new InputSlotMorph(
+                null,
+                false, {
+                    'motion': ['motion'],
+                    'direction': ['direction']
+                },
+                true // read-only
+            );
+            part.setContents(['motion']);
+            break;
+        case '%on':
+            part = new InputSlotMorph(
+                null,
+                false, {
+                    'this sprite': ['this sprite'],
+                    'stage': ['stage']
+                },
+                true // read-only
+            );
         default:
             nop();
         }
@@ -2362,6 +2394,9 @@ SyntaxElementMorph.prototype.endLayout = function () {
     %f      - round function slot, unevaluated if replaced,
     %r      - round reporter slot
     %p      - hexagonal predicate slot
+    %vid    - chameleon colored rectangular drop-down for video modes
+    %motype - chameleon colored rectangular drop-down for motion type
+    %on     - chameleon colored rectangular drop-down for motion detection scope
 
     rings:
 

src/objects.js

@@ -1358,6 +1358,30 @@ SpriteMorph.prototype.initBlocks = function () {
             type: 'reporter',
             category: 'other',
             spec: 'code of %cmdRing'
+        },
+        // Video motion
+        doSetVideo: {
+            type: 'command',
+            category: 'sensing',
+            spec: 'turn video %vid',
+            defaults: ['on']
+        },
+        doSetVideoTransparency: {
+            type: 'command',
+            category: 'sensing',
+            spec: 'set video transparency to %n',
+            defaults: [50]
+        },
+        reportMotionOn: {
+            type: 'reporter',
+            category: 'sensing',
+            spec: 'video %motype on %on',
+            defaults: ['motion', 'this sprite']
+        },
+        reportMotionOnStage: {
+            type: 'reporter',
+            category: 'sensing',
+            spec: 'video %motype on stage'
         }
     };
 };
@@ -1627,6 +1651,10 @@ SpriteMorph.prototype.init = function (globals) {
     this.cachedPropagation = false; // not to be persisted
     this.inheritedAttributes = []; // 'x position', 'direction', 'size' etc...
 
+    this.motionAmount = 0;
+    this.motionDirection = 0;
+    this.frameNumber = 0;
+
     SpriteMorph.uber.init.call(this);
 
     this.cachedHSV = this.color.hsv();
@@ -2275,6 +2303,10 @@ SpriteMorph.prototype.blockTemplates = function (category) {
         blocks.push(watcherToggle('getTimer'));
         blocks.push(block('getTimer'));
         blocks.push('-');
+        blocks.push(block('doSetVideo'));
+        blocks.push(block('doSetVideoTransparency'));
+        blocks.push(block('reportMotionOn'));
+        blocks.push('-');
         blocks.push(block('reportAttributeOf'));
 
         if (SpriteMorph.prototype.enableFirstClass) {
@@ -6943,6 +6975,10 @@ StageMorph.prototype.init = function (globals) {
 
     this.remixID = null;
 
+    this.videoElement = null;
+    this.videoTransparency = 50;
+    this.videoMotion = null;
+
     StageMorph.uber.init.call(this);
 
     this.cachedHSV = this.color.hsv();
@@ -7047,7 +7083,10 @@ StageMorph.prototype.drawOn = function (aCanvas, aRect) {
             w,
             h
         );
-
+        // webcam
+        if (this.videoElement) {
+            this.drawVideo(context);
+        }
         // pen trails
         ws = w / this.scale;
         hs = h / this.scale;
@@ -7168,6 +7207,85 @@ StageMorph.prototype.colorFiltered = function (aColor, excludedSprite) {
     return morph;
 };
 
+// Video
+StageMorph.prototype.drawVideo = function(context) {
+    var w = this.dimensions.x * this.scale,
+        h = this.dimensions.y * this.scale;
+    context.save();
+    context.globalAlpha = 1 - (this.videoTransparency / 100);
+    if (!this.videoElement.isFlipped) {
+        context.translate(w, 0);
+        context.scale(-1, 1);
+    }
+    if (this.videoElement.width != this.dimensions.x || this.videoElement.height != this.dimensions.y) {
+        this.videoElement.width = this.dimensions.x;
+        this.videoElement.height = this.dimensions.y;
+        this.videoMotion.reset(this.dimensions.x, this.dimensions.y);
+    }
+    context.drawImage(
+        this.videoElement,
+        this.left() * (this.videoElement.isFlipped ? 1 : -1),
+        this.top(),
+        w,
+        h
+    );
+    context.restore();
+};
+
+StageMorph.prototype.startVideo = function(isFlipped) {
+    var myself = this;
+
+    function noCameraSupport() {
+        var dialog = new DialogBoxMorph();
+        dialog.inform(
+            localize('Camera not supported'),
+            localize('Please make sure your web browser is up to date\n' +
+                'and your camera is properly configured. \n\n' +
+                'Some browsers also require you to access Snap!\n' +
+                'through HTTPS to use the camera.\n\n' +
+                'Please replace the "http://" part of the address\n' +
+                'in your browser by "https://" and try again.'),
+            this.world
+        );
+        dialog.fixLayout();
+        dialog.drawNew();
+        if (myself.videoElement) {
+            myself.videoElement.remove();
+        }
+    }
+
+    if (!this.videoElement) {
+        this.videoElement = document.createElement('video');
+        this.videoElement.width = this.dimensions.x;
+        this.videoElement.height = this.dimensions.y;
+        this.videoElement.hidden = true;
+        document.body.appendChild(this.videoElement);
+    }
+    this.videoElement.isFlipped = isFlipped;
+    if (!this.videoMotion) {
+        this.videoMotion = new VideoMotion(this.dimensions.x, this.dimensions.y);
+    }
+    if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) {
+        navigator.mediaDevices.getUserMedia({ video: true })
+            .then(function(stream) {
+                myself.videoElement.srcObject = stream;
+                myself.videoElement.play().catch(noCameraSupport);
+                myself.videoElement.stream = stream;
+            })
+            .catch(noCameraSupport);
+    }
+};
+
+StageMorph.prototype.stopVideo = function() {
+    if (this.videoElement) {
+        this.videoElement.remove();
+        this.videoElement = null;
+        this.videoMotion = null;
+    }
+    this.changed();
+    this.drawNew();
+};
+
 // StageMorph pixel access:
 
 StageMorph.prototype.getPixelColor = function (aPoint) {
@@ -7335,6 +7453,20 @@ StageMorph.prototype.step = function () {
         });
         this.lastWatcherUpdate = Date.now();
     }
+
+    // video frame capture
+    if (this.videoElement) {
+        var context = newCanvas(this.dimensions, true).getContext('2d');
+        context.save();
+        if (!this.videoElement.isFlipped) {
+            context.translate(this.dimensions.x, 0);
+            context.scale(-1, 1);
+        }
+        context.drawImage(this.videoElement, 0, 0, this.videoElement.width, this.videoElement.height);
+        this.videoMotion.addFrame(context.getImageData(0, 0, this.videoElement.width, this.videoElement.height).data);
+        context.restore();
+        this.changed();
+    }
 };
 
 StageMorph.prototype.stepGenericConditions = function (stopAll) {
@@ -7850,6 +7982,10 @@ StageMorph.prototype.blockTemplates = function (category) {
         blocks.push(watcherToggle('getTimer'));
         blocks.push(block('getTimer'));
         blocks.push('-');
+        blocks.push(block('doSetVideo'));
+        blocks.push(block('doSetVideoTransparency'));
+        blocks.push(block('reportMotionOnStage'));
+        blocks.push('-');
         blocks.push(block('reportAttributeOf'));
 
         if (SpriteMorph.prototype.enableFirstClass) {

src/threads.js

@@ -43,7 +43,7 @@
         Variable
         VariableFrame
         JSCompiler
-
+        VideoMotion
 
     credits
     -------
@@ -70,6 +70,7 @@ var Context;
 var Variable;
 var VariableFrame;
 var JSCompiler;
+var VideoMotion;
 
 function snapEquals(a, b) {
     if (a instanceof List || (b instanceof List)) {
@@ -4388,6 +4389,69 @@ Process.prototype.reportDate = function (datefn) {
     return result;
 };
 
+// Video
+Process.prototype.doSetVideo = function(state) {
+    var stage,
+        inputState = this.inputOption(state);
+
+    if (this.homeContext.receiver) {
+        stage = this.homeContext.receiver.parentThatIsA(StageMorph);
+        if (stage) {
+            if (inputState === 'on') {
+                stage.startVideo();
+            } else if (inputState === 'off') {
+                stage.stopVideo();
+            } else if (inputState === 'on-flipped') {
+                stage.startVideo(true); // flipped
+            }
+        }
+    }
+};
+
+Process.prototype.doSetVideoTransparency = function(factor) {
+    var stage;
+    if (this.homeContext.receiver) {
+        stage = this.homeContext.receiver.parentThatIsA(StageMorph);
+        if (stage) {
+            stage.videoTransparency = Math.max(0, Math.min(100, factor));
+        }
+    }
+};
+
+Process.prototype.reportMotionOnStage = function(motionType) {
+    return this.reportMotionOn(motionType, 'stage');
+};
+
+Process.prototype.reportMotionOn = function(motionType, on) {
+    var stage = this.homeContext.receiver.parentThatIsA(StageMorph),
+        sprite = this.blockReceiver(),
+        result;
+    if (stage === null || !stage.videoElement) {
+        return null;
+    }
+    switch (this.inputOption(on)) {
+        case 'stage':
+            stage.videoMotion.getStageMotion();
+            if (this.inputOption(motionType) === 'direction') {
+                result = stage.videoMotion.motionDirection;
+            }
+            if (this.inputOption(motionType) === 'motion') {
+                result = stage.videoMotion.motionAmount;
+            }
+            break;
+        case 'this sprite':
+            stage.videoMotion.getLocalMotion(sprite);
+            if (this.inputOption(motionType) === 'direction') {
+                result = sprite.motionDirection;
+            }
+            if (this.inputOption(motionType) === 'motion') {
+                result = sprite.motionAmount;
+            }
+            break;
+    }
+    return result;
+};
+
 // Process code mapping
 
 /*
@@ -5706,3 +5770,372 @@ JSCompiler.prototype.compileInput = function (inp) {
         );
     }
 };
+
+// VideoMotion /////////////////////////////////////////////////////////////////
+
+/*
+ * Calculate, based on two consecutive video frames, the amount of movement and
+ * direction of this movement both on the stage and on the sprite.
+ * It's based on Scratch 3 (optical flow algorithm).
+ */
+function VideoMotion(width, height) {
+    this.width = width;
+    this.height = height;
+    this.frameNumber = 0;
+    this.winSize = 8;
+    this.lastAnalyzedFrame = 0;
+    this.motionAmount = 0;
+    this.motionDirection = 0;
+    this.imageBuffer = new ArrayBuffer(this.width * this.height * 2);
+    this.curr = new Uint8ClampedArray(
+        this.imageBuffer,
+        0,
+        this.width * this.height
+    );
+    this.prev = new Uint8ClampedArray(
+        this.imageBuffer,
+        this.width * this.height,
+        this.width * this.height
+    );
+    this.threshold = 30;
+    this.amountScale = 100;
+    this.toDegree = 180 / Math.PI;
+};
+
+/*
+ * Reset videoElement and videoMotion dimensions.
+ * This function is called when stage dimensions change.
+ */
+VideoMotion.prototype.reset = function(width, height){
+    this.width = width;
+    this.height = height;
+    this.frameNumber = 0;
+    this.lastAnalyzedFrame = 0;
+    this.imageBuffer = new ArrayBuffer(this.width * this.height * 2);
+    this.curr = new Uint8ClampedArray(
+        this.imageBuffer,
+        0,
+        this.width * this.height
+    );
+    this.prev = new Uint8ClampedArray(
+        this.imageBuffer,
+        this.width * this.height,
+        this.width * this.height
+    );
+};
+
+VideoMotion.prototype.addFrame = function(imageData) {
+    var i,
+        temp = this.prev,
+        frame = new Uint32Array(imageData.buffer.slice(0)); //ABGR
+    this.frameNumber++;
+    this.prev = this.curr;
+    this.curr = temp;
+    for (i = 0; i < frame.length; i++) {
+        this.curr[i] = frame[i] & 0xff;
+    }
+};
+
+VideoMotion.prototype.getStageMotion = function() {
+    var uu = 0, // Accumulate 2d motion vectors from groups
+        vv = 0, // of pixels and average it later.
+        n = 0,
+        vector = {
+            u: 0,
+            v: 0
+        },
+        i, j, k, l, address, nextAddress, maxAddress,
+        winStep = this.winSize * 2 + 1,
+        wmax = this.width - this.winSize - 1,
+        hmax = this.height - this.winSize - 1,
+    // Optical Flow vars
+        A2, A1B2, B1, C1, C2,
+        gradX, gradY, gradT;
+
+    if (!this.curr || !this.prev) {
+        this.motionAmount = this.motionDirection = -1;
+        // Don't have two frames to analyze yet
+        return;
+    }
+    // Return early if new data has not been received.
+    if (this.lastAnalyzedFrame === this.frameNumber) {
+        return;
+    }
+    this.lastAnalyzedFrame = this.frameNumber;
+    // Iterate over groups of cells building up the components to determine
+    // a motion vector for each cell instead of the whole frame to avoid
+    // integer overflows.
+    for (i = this.winSize + 1; i < hmax; i += winStep) {
+        for (j = this.winSize + 1; j < wmax; j += winStep) {
+            A2 = 0;
+            A1B2 = 0;
+            B1 = 0;
+            C1 = 0;
+            C2 = 0;
+            // This is a performance critical math region.
+            address = ((i - this.winSize) * this.width) + j - this.winSize;
+            nextAddress = address + winStep;
+            maxAddress = ((i + this.winSize) * this.width) + j + this.winSize;
+            for (; address <= maxAddress; address += this.width - winStep,
+                nextAddress += this.width) {
+                for (; address <= nextAddress; address += 1) {
+                    // The difference in color between the last frame and
+                    // the current frame.
+                    gradT = ((this.prev[address]) - (this.curr[address]));
+                    // The difference between the pixel to the left and the
+                    // pixel to the right.
+                    gradX = ((this.curr[address - 1]) - (this.curr[address + 1]));
+                    // The difference between the pixel above and the pixel
+                    // below.
+                    gradY = ((
+                        this.curr[address - this.width])
+                        - (this.curr[address + this.width]));
+                    // Add the combined values of this pixel to previously
+                    // considered pixels.
+                    A2 += gradX * gradX;
+                    A1B2 += gradX * gradY;
+                    B1 += gradY * gradY;
+                    C2 += gradX * gradT;
+                    C1 += gradY * gradT;
+                }
+            }
+            // Use the accumalated values from the for loop to determine a
+            // motion direction.
+            vector = this.getMotionVector(A2, A1B2, B1, C2, C1);
+            // If u and v are within negative winStep to positive winStep,
+            // add them to a sum that will later be averaged.
+            if (-winStep < vector.u
+                    && vector.u < winStep
+                    && -winStep < vector.v
+                    && vector.v < winStep) {
+                uu += vector.u;
+                vv += vector.v;
+                n++;
+            }
+        }
+    }
+    // Average the summed vector values of all of the motion groups.
+    uu /= n;
+    vv /= n;
+    // Scale the magnitude of the averaged UV vector.
+    this.motionAmount = Math.round(this.amountScale * Math.hypot(uu, vv));
+    if (this.motionAmount > this.threshold) {
+        this.motionDirection = (((Math.atan2(vv, uu) * this.toDegree + 270) % 360) - 180)
+            .toFixed(2); // Snap direction
+    }
+};
+
+/**
+ * Determine a motion vector combinations of the color component difference
+ * on the x axis, y axis, and temporal axis.
+ * A2 - a sum of x axis squared
+ * A1B2 - a sum of x axis times y axis
+ * B1 - a sum of y axis squared
+ * C2 - a sum of x axis times temporal axis
+ * C1 - a sum of y axis times temporal axis
+ * Returns a uv vector representing the motion for the given input
+ */
+VideoMotion.prototype.getMotionVector = function(A2, A1B2, B1, C2, C1) {
+    // Compare sums of X * Y and sums of X squared and Y squared.
+    var norm,
+        IGradNorm,
+        delta = ((A1B2 * A1B2) - (A2 * B1)),
+        deltaX, deltaY, Idelta,
+        motionVector = {
+            u: 0,
+            v: 0
+        };
+
+    if (delta) {
+        // System is not singular - solving by Kramer method.
+        deltaX = -((C1 * A1B2) - (C2 * B1));
+        deltaY = -((A1B2 * C2) - (A2 * C1));
+        Idelta = 8 / delta;
+        motionVector.u = deltaX * Idelta;
+        motionVector.v = deltaY * Idelta;
+    } else {
+        // Singular system - find optical flow in gradient direction.
+        norm = ((A1B2 + A2) * (A1B2 + A2)) + ((B1 + A1B2) * (B1 + A1B2));
+        if (norm) {
+            IGradNorm = 8 / norm;
+            motionVector.u = (A1B2 + A2) * (-(C1 + C2) * IGradNorm);
+            motionVector.v = (B1 + A1B2) * (-(C1 + C2) * IGradNorm);
+        } else {
+            motionVector.u = 0;
+            motionVector.v = 0;
+        }
+    }
+    return motionVector;
+};
+
+/**
+ * Calculate motion amount and direction values based on stored frames
+ * (current and previous) that overlaps a given sprite.
+ */
+VideoMotion.prototype.getLocalMotion = function(aSprite) {
+    var stage = aSprite.parentThatIsA(StageMorph),
+        activePixelNum = 0,
+        i, j,
+        spriteWidth = Math.floor(aSprite.width() / stage.scale),
+        winSize = this.winSize,
+        vector = {
+            u: 0,
+            v: 0
+        },
+        A2 = 0,
+        A1B2 = 0,
+        B1 = 0,
+        C1 = 0,
+        C2 = 0,
+        localThreshold = this.threshold / 3,
+        localMaxAmount = 100,
+        localAmountScale = this.amountScale * 2e-4,
+        scaleFactor = 0,
+        address = 0,
+        spriteImage,
+        cb,
+        pixel;
+
+    if (!this.curr || !this.prev) {
+        aSprite.motionAmount = aSprite.motionDirection = -1;
+        // Don't have two frames to analyze yet
+        return;
+    }
+    // Skip if the current frame has already been considered
+    // for this state.
+    if (aSprite.frameNumber !== this.frameNumber) {
+        spriteImage = getSpriteImgageData(aSprite);
+        // Consider only the area of the current frame overlapped
+        // with the given sprite.
+        cb = getClippedBounds(aSprite);
+        xmin = Math.max(
+            Math.floor((aSprite.left() - stage.left()) / stage.scale),
+            0);
+        ymin = Math.max(
+            Math.floor((aSprite.top() - stage.top()) / stage.scale),
+            0);
+        xmax = Math.min(cb.sw + xmin, stage.dimensions.x);
+        ymax = Math.min(cb.sh + ymin, stage.dimensions.y - 1);
+        // This is a performance critical math region.
+        pixel = cb.sy * spriteWidth + cb.sx;
+        for (i = ymin; i < ymax; i++, pixel += spriteWidth - cb.sw) { //rows
+            for (j = xmin; j < xmax; j++, ++pixel) { //cols
+                if (j > 0 && j < this.width && i > 0 && i < this.height
+                    && (spriteImage[pixel] >> 24 & 0xff) == 0xff) {
+                    address = (i * this.width) + j;
+                    // The difference in color between the last frame and
+                    // the current frame.
+                    gradT = ((this.prev[address]) - (this.curr[address]));
+                    // The difference between the pixel to the left and the
+                    // pixel to the right.
+                    gradX = ((this.curr[address - 1]) - (this.curr[address + 1]));
+                    // The difference between the pixel above and the pixel
+                    // below.
+                    gradY = (
+                        (this.curr[address - this.width])
+                        - (this.curr[address + this.width]));
+                    // Add the combined values of this pixel to previously
+                    // considered pixels.
+                    A2 += gradX * gradX;
+                    A1B2 += gradX * gradY;
+                    B1 += gradY * gradY;
+                    C2 += gradX * gradT;
+                    C1 += gradY * gradT;
+                    scaleFactor++;
+                }
+            }
+        }
+        // Use the accumalated values from the for loop to determine a
+        // motion direction.
+        vector = this.getMotionVector(A2, A1B2, B1, C2, C1);
+        if (scaleFactor) {
+            // Store the area of the sprite in pixels
+            activePixelNum = scaleFactor;
+            scaleFactor /= (2 * winSize * 2 * winSize);
+            vector.u = vector.u / scaleFactor;
+            vector.v = vector.v / scaleFactor;
+        }
+        // Scale the magnitude of the averaged UV vector and the number of
+        // overlapping solid pixels.
+        aSprite.motionAmount = Math.round(
+            localAmountScale * activePixelNum
+            * Math.hypot(vector.u, vector.v)
+        );
+        if (aSprite.motionAmount > localMaxAmount) {
+            // Clip all magnitudes greater than 100.
+            aSprite.motionAmount = Math.min(localMaxAmount, 100);
+        }
+        if (aSprite.motionAmount > localThreshold) {
+            // Snap direction.
+            aSprite.motionDirection = (((
+                Math.atan2(vector.v, vector.u)
+                * this.toDegree + 270) % 360) - 180)
+                .toFixed(2);
+        }
+        // Skip future calls on this state until a new frame is added.
+        aSprite.frameNumber = this.frameNumber;
+    }
+
+    /*
+     * Get sprite image data scaled to 1 an converted to ABGR array
+     */
+    function getSpriteImgageData(sprite) {
+        var stage = sprite.parentThatIsA(StageMorph),
+            newExtent = {
+                x: Math.floor(sprite.extent().x / stage.scale),
+                y: Math.floor(sprite.extent().y / stage.scale)
+            },
+            canvas = newCanvas(newExtent, true),
+            canvasContext,
+            imageData;
+        canvasContext = canvas.getContext("2d");
+        canvasContext.drawImage(
+            sprite.image,
+            0, 0, Math.floor(sprite.extent().x),
+            Math.floor(sprite.extent().y),
+            0, 0, newExtent.x, newExtent.y
+        );
+        imageData = canvas.getContext("2d")
+            .getImageData(0, 0, newExtent.x, newExtent.y).data;
+        return new Uint32Array(imageData.buffer.slice(0));
+    }
+
+    /*
+     * Return sprite's visible part bounds
+     */
+    function getClippedBounds(sprite) {
+        var stage = sprite.parentThatIsA(StageMorph),
+            scale = stage.scale,
+            bounds = {
+                sx: 0,
+                sy: 0,
+                sw: Math.floor(sprite.extent().x / scale),
+                sh: Math.floor(sprite.extent().y / scale)
+            };
+        // Clipping X
+        if (sprite.left() < stage.left()) { // sprite outer left stage
+            bounds.sw = Math.max(
+                Math.floor((sprite.right() - stage.left()) / scale),
+                0);
+            bounds.sx = Math.floor(sprite.width() / scale - bounds.sw);
+        }
+        if (sprite.right() > stage.right()) { // sprite outer right stage
+            bounds.sw = Math.max(
+                Math.floor((stage.right() - sprite.left()) / scale),
+                0);
+        }
+        //Clipping Y
+        if (sprite.top() < stage.top()) { // sprite upper top
+            bounds.sh = Math.max(
+                Math.floor((sprite.bottom() - stage.top()) / scale),
+                0);
+            bounds.sy = Math.floor(sprite.height() / scale - bounds.sh);
+        }
+        if (sprite.bottom() > stage.bottom()) { // sprite lower bottom
+            bounds.sh = Math.max(
+                Math.floor((stage.bottom() - sprite.top()) / scale),
+                0);
+        }
+        return bounds;
+    }
+};