kopia lustrzana https://github.com/backface/turtlestitch
moved video motion sensing code into its own file
rodzic
e364074a2b
commit
b8eaa514a7
|
@ -6,9 +6,9 @@
|
|||
<link rel="shortcut icon" href="src/favicon.ico">
|
||||
<script type="text/javascript" src="src/morphic.js?version=2019-02-07"></script>
|
||||
<script type="text/javascript" src="src/widgets.js?version=2019-04-05"></script>
|
||||
<script type="text/javascript" src="src/blocks.js?version=2019-05-03"></script>
|
||||
<script type="text/javascript" src="src/threads.js?version=2019-05-02"></script>
|
||||
<script type="text/javascript" src="src/objects.js?version=2019-05-03"></script>
|
||||
<script type="text/javascript" src="src/blocks.js?version=2019-05-07"></script>
|
||||
<script type="text/javascript" src="src/threads.js?version=2019-05-07"></script>
|
||||
<script type="text/javascript" src="src/objects.js?version=2019-05-07"></script>
|
||||
<script type="text/javascript" src="src/gui.js?version=2019-04-27"></script>
|
||||
<script type="text/javascript" src="src/paint.js?version=2019-02-22"></script>
|
||||
<script type="text/javascript" src="src/lists.js?version=2019-04-27"></script>
|
||||
|
@ -16,6 +16,7 @@
|
|||
<script type="text/javascript" src="src/tables.js?version=2019-02-07"></script>
|
||||
<script type="text/javascript" src="src/symbols.js?version=2019-03-07"></script>
|
||||
<script type="text/javascript" src="src/sketch.js?version=2019-02-22"></script>
|
||||
<script type="text/javascript" src="src/video.js?version=2019-05-07"></script>
|
||||
<script type="text/javascript" src="src/xml.js?version=2018-11-12"></script>
|
||||
<script type="text/javascript" src="src/store.js?version=2019-04-04"></script>
|
||||
<script type="text/javascript" src="src/locale.js?version=2019-05-03"></script>
|
||||
|
|
|
@ -148,7 +148,7 @@ CustomCommandBlockMorph, SymbolMorph, ToggleButtonMorph, DialMorph*/
|
|||
|
||||
// Global stuff ////////////////////////////////////////////////////////
|
||||
|
||||
modules.blocks = '2019-May-06';
|
||||
modules.blocks = '2019-May-07';
|
||||
|
||||
var SyntaxElementMorph;
|
||||
var BlockMorph;
|
||||
|
|
|
@ -84,7 +84,7 @@ BlockEditorMorph, BlockDialogMorph, PrototypeHatBlockMorph, localize,
|
|||
TableMorph, TableFrameMorph, normalizeCanvas, BooleanSlotMorph, HandleMorph,
|
||||
AlignmentMorph, Process, XML_Element, VectorPaintEditorMorph*/
|
||||
|
||||
modules.objects = '2019-May-06';
|
||||
modules.objects = '2019-May-07';
|
||||
|
||||
var SpriteMorph;
|
||||
var StageMorph;
|
||||
|
|
376
src/threads.js
376
src/threads.js
|
@ -43,7 +43,6 @@
|
|||
Variable
|
||||
VariableFrame
|
||||
JSCompiler
|
||||
VideoMotion
|
||||
|
||||
credits
|
||||
-------
|
||||
|
@ -62,7 +61,7 @@ StageMorph, SpriteMorph, StagePrompterMorph, Note, modules, isString, copy,
|
|||
isNil, WatcherMorph, List, ListWatcherMorph, alert, console, TableMorph, Color,
|
||||
TableFrameMorph, ColorSlotMorph, isSnapObject, Map, newCanvas, Symbol*/
|
||||
|
||||
modules.threads = '2019-May-06';
|
||||
modules.threads = '2019-May-07';
|
||||
|
||||
var ThreadManager;
|
||||
var Process;
|
||||
|
@ -70,7 +69,6 @@ var Context;
|
|||
var Variable;
|
||||
var VariableFrame;
|
||||
var JSCompiler;
|
||||
var VideoMotion;
|
||||
|
||||
function snapEquals(a, b) {
|
||||
if (a instanceof List || (b instanceof List)) {
|
||||
|
@ -4389,7 +4387,8 @@ Process.prototype.reportDate = function (datefn) {
|
|||
return result;
|
||||
};
|
||||
|
||||
// Video
|
||||
// Process video motion detection primitives
|
||||
|
||||
Process.prototype.doSetVideo = function(state) {
|
||||
var stage,
|
||||
inputState = this.inputOption(state);
|
||||
|
@ -5770,372 +5769,3 @@ JSCompiler.prototype.compileInput = function (inp) {
|
|||
);
|
||||
}
|
||||
};
|
||||
|
||||
// VideoMotion /////////////////////////////////////////////////////////////////
|
||||
|
||||
/*
|
||||
* Calculate, based on two consecutive video frames, the amount of movement and
|
||||
* direction of this movement both on the stage and on the sprite.
|
||||
* It's based on Scratch 3 (optical flow algorithm).
|
||||
*/
|
||||
function VideoMotion(width, height) {
|
||||
this.width = width;
|
||||
this.height = height;
|
||||
this.frameNumber = 0;
|
||||
this.winSize = 8;
|
||||
this.lastAnalyzedFrame = 0;
|
||||
this.motionAmount = 0;
|
||||
this.motionDirection = 0;
|
||||
this.imageBuffer = new ArrayBuffer(this.width * this.height * 2);
|
||||
this.curr = new Uint8ClampedArray(
|
||||
this.imageBuffer,
|
||||
0,
|
||||
this.width * this.height
|
||||
);
|
||||
this.prev = new Uint8ClampedArray(
|
||||
this.imageBuffer,
|
||||
this.width * this.height,
|
||||
this.width * this.height
|
||||
);
|
||||
this.threshold = 30;
|
||||
this.amountScale = 100;
|
||||
this.toDegree = 180 / Math.PI;
|
||||
}
|
||||
|
||||
/*
|
||||
* Reset videoElement and videoMotion dimensions.
|
||||
* This function is called when stage dimensions change.
|
||||
*/
|
||||
VideoMotion.prototype.reset = function(width, height){
|
||||
this.width = width;
|
||||
this.height = height;
|
||||
this.frameNumber = 0;
|
||||
this.lastAnalyzedFrame = 0;
|
||||
this.imageBuffer = new ArrayBuffer(this.width * this.height * 2);
|
||||
this.curr = new Uint8ClampedArray(
|
||||
this.imageBuffer,
|
||||
0,
|
||||
this.width * this.height
|
||||
);
|
||||
this.prev = new Uint8ClampedArray(
|
||||
this.imageBuffer,
|
||||
this.width * this.height,
|
||||
this.width * this.height
|
||||
);
|
||||
};
|
||||
|
||||
VideoMotion.prototype.addFrame = function(imageData) {
|
||||
var i,
|
||||
temp = this.prev,
|
||||
frame = new Uint32Array(imageData.buffer.slice(0)); //ABGR
|
||||
this.frameNumber++;
|
||||
this.prev = this.curr;
|
||||
this.curr = temp;
|
||||
for (i = 0; i < frame.length; i++) {
|
||||
this.curr[i] = frame[i] & 0xff;
|
||||
}
|
||||
};
|
||||
|
||||
VideoMotion.prototype.getStageMotion = function() {
|
||||
var uu = 0, // Accumulate 2d motion vectors from groups
|
||||
vv = 0, // of pixels and average it later.
|
||||
n = 0,
|
||||
vector = {
|
||||
u: 0,
|
||||
v: 0
|
||||
},
|
||||
i, j, address, nextAddress, maxAddress,
|
||||
winStep = this.winSize * 2 + 1,
|
||||
wmax = this.width - this.winSize - 1,
|
||||
hmax = this.height - this.winSize - 1,
|
||||
// Optical Flow vars
|
||||
A2, A1B2, B1, C1, C2,
|
||||
gradX, gradY, gradT;
|
||||
|
||||
if (!this.curr || !this.prev) {
|
||||
this.motionAmount = this.motionDirection = -1;
|
||||
// Don't have two frames to analyze yet
|
||||
return;
|
||||
}
|
||||
// Return early if new data has not been received.
|
||||
if (this.lastAnalyzedFrame === this.frameNumber) {
|
||||
return;
|
||||
}
|
||||
this.lastAnalyzedFrame = this.frameNumber;
|
||||
// Iterate over groups of cells building up the components to determine
|
||||
// a motion vector for each cell instead of the whole frame to avoid
|
||||
// integer overflows.
|
||||
for (i = this.winSize + 1; i < hmax; i += winStep) {
|
||||
for (j = this.winSize + 1; j < wmax; j += winStep) {
|
||||
A2 = 0;
|
||||
A1B2 = 0;
|
||||
B1 = 0;
|
||||
C1 = 0;
|
||||
C2 = 0;
|
||||
// This is a performance critical math region.
|
||||
address = ((i - this.winSize) * this.width) + j - this.winSize;
|
||||
nextAddress = address + winStep;
|
||||
maxAddress = ((i + this.winSize) * this.width) + j + this.winSize;
|
||||
for (; address <= maxAddress; address += this.width - winStep,
|
||||
nextAddress += this.width) {
|
||||
for (; address <= nextAddress; address += 1) {
|
||||
// The difference in color between the last frame and
|
||||
// the current frame.
|
||||
gradT = ((this.prev[address]) - (this.curr[address]));
|
||||
// The difference between the pixel to the left and the
|
||||
// pixel to the right.
|
||||
gradX = ((this.curr[address - 1]) - (this.curr[address + 1]));
|
||||
// The difference between the pixel above and the pixel
|
||||
// below.
|
||||
gradY = ((
|
||||
this.curr[address - this.width])
|
||||
- (this.curr[address + this.width]));
|
||||
// Add the combined values of this pixel to previously
|
||||
// considered pixels.
|
||||
A2 += gradX * gradX;
|
||||
A1B2 += gradX * gradY;
|
||||
B1 += gradY * gradY;
|
||||
C2 += gradX * gradT;
|
||||
C1 += gradY * gradT;
|
||||
}
|
||||
}
|
||||
// Use the accumalated values from the for loop to determine a
|
||||
// motion direction.
|
||||
vector = this.getMotionVector(A2, A1B2, B1, C2, C1);
|
||||
// If u and v are within negative winStep to positive winStep,
|
||||
// add them to a sum that will later be averaged.
|
||||
if (-winStep < vector.u
|
||||
&& vector.u < winStep
|
||||
&& -winStep < vector.v
|
||||
&& vector.v < winStep) {
|
||||
uu += vector.u;
|
||||
vv += vector.v;
|
||||
n++;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Average the summed vector values of all of the motion groups.
|
||||
uu /= n;
|
||||
vv /= n;
|
||||
// Scale the magnitude of the averaged UV vector.
|
||||
this.motionAmount = Math.round(this.amountScale * Math.hypot(uu, vv));
|
||||
if (this.motionAmount > this.threshold) {
|
||||
this.motionDirection = (((Math.atan2(vv, uu) * this.toDegree + 270) % 360) - 180)
|
||||
.toFixed(2); // Snap direction
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Determine a motion vector combinations of the color component difference
|
||||
* on the x axis, y axis, and temporal axis.
|
||||
* A2 - a sum of x axis squared
|
||||
* A1B2 - a sum of x axis times y axis
|
||||
* B1 - a sum of y axis squared
|
||||
* C2 - a sum of x axis times temporal axis
|
||||
* C1 - a sum of y axis times temporal axis
|
||||
* Returns a uv vector representing the motion for the given input
|
||||
*/
|
||||
VideoMotion.prototype.getMotionVector = function(A2, A1B2, B1, C2, C1) {
|
||||
// Compare sums of X * Y and sums of X squared and Y squared.
|
||||
var norm,
|
||||
IGradNorm,
|
||||
delta = ((A1B2 * A1B2) - (A2 * B1)),
|
||||
deltaX, deltaY, Idelta,
|
||||
motionVector = {
|
||||
u: 0,
|
||||
v: 0
|
||||
};
|
||||
|
||||
if (delta) {
|
||||
// System is not singular - solving by Kramer method.
|
||||
deltaX = -((C1 * A1B2) - (C2 * B1));
|
||||
deltaY = -((A1B2 * C2) - (A2 * C1));
|
||||
Idelta = 8 / delta;
|
||||
motionVector.u = deltaX * Idelta;
|
||||
motionVector.v = deltaY * Idelta;
|
||||
} else {
|
||||
// Singular system - find optical flow in gradient direction.
|
||||
norm = ((A1B2 + A2) * (A1B2 + A2)) + ((B1 + A1B2) * (B1 + A1B2));
|
||||
if (norm) {
|
||||
IGradNorm = 8 / norm;
|
||||
motionVector.u = (A1B2 + A2) * (-(C1 + C2) * IGradNorm);
|
||||
motionVector.v = (B1 + A1B2) * (-(C1 + C2) * IGradNorm);
|
||||
} else {
|
||||
motionVector.u = 0;
|
||||
motionVector.v = 0;
|
||||
}
|
||||
}
|
||||
return motionVector;
|
||||
};
|
||||
|
||||
/**
|
||||
* Calculate motion amount and direction values based on stored frames
|
||||
* (current and previous) that overlaps a given sprite.
|
||||
*/
|
||||
VideoMotion.prototype.getLocalMotion = function(aSprite) {
|
||||
var stage = aSprite.parentThatIsA(StageMorph),
|
||||
activePixelNum = 0,
|
||||
i, j, xmin, xmax, ymin, ymax, gradT, gradX, gradY,
|
||||
spriteWidth = Math.floor(aSprite.width() / stage.scale),
|
||||
winSize = this.winSize,
|
||||
vector = {
|
||||
u: 0,
|
||||
v: 0
|
||||
},
|
||||
A2 = 0,
|
||||
A1B2 = 0,
|
||||
B1 = 0,
|
||||
C1 = 0,
|
||||
C2 = 0,
|
||||
localThreshold = this.threshold / 3,
|
||||
localMaxAmount = 100,
|
||||
localAmountScale = this.amountScale * 2e-4,
|
||||
scaleFactor = 0,
|
||||
address = 0,
|
||||
spriteImage,
|
||||
cb,
|
||||
pixel;
|
||||
|
||||
if (!this.curr || !this.prev) {
|
||||
aSprite.motionAmount = aSprite.motionDirection = -1;
|
||||
// Don't have two frames to analyze yet
|
||||
return;
|
||||
}
|
||||
// Skip if the current frame has already been considered
|
||||
// for this state.
|
||||
if (aSprite.frameNumber !== this.frameNumber) {
|
||||
spriteImage = getSpriteImgageData(aSprite);
|
||||
// Consider only the area of the current frame overlapped
|
||||
// with the given sprite.
|
||||
cb = getClippedBounds(aSprite);
|
||||
xmin = Math.max(
|
||||
Math.floor((aSprite.left() - stage.left()) / stage.scale),
|
||||
0);
|
||||
ymin = Math.max(
|
||||
Math.floor((aSprite.top() - stage.top()) / stage.scale),
|
||||
0);
|
||||
xmax = Math.min(cb.sw + xmin, stage.dimensions.x);
|
||||
ymax = Math.min(cb.sh + ymin, stage.dimensions.y - 1);
|
||||
// This is a performance critical math region.
|
||||
pixel = cb.sy * spriteWidth + cb.sx;
|
||||
for (i = ymin; i < ymax; i++, pixel += spriteWidth - cb.sw) { //rows
|
||||
for (j = xmin; j < xmax; j++, ++pixel) { //cols
|
||||
if (j > 0 && j < this.width && i > 0 && i < this.height
|
||||
&& (spriteImage[pixel] >> 24 & 0xff) == 0xff) {
|
||||
address = (i * this.width) + j;
|
||||
// The difference in color between the last frame and
|
||||
// the current frame.
|
||||
gradT = ((this.prev[address]) - (this.curr[address]));
|
||||
// The difference between the pixel to the left and the
|
||||
// pixel to the right.
|
||||
gradX = ((this.curr[address - 1]) - (this.curr[address + 1]));
|
||||
// The difference between the pixel above and the pixel
|
||||
// below.
|
||||
gradY = (
|
||||
(this.curr[address - this.width])
|
||||
- (this.curr[address + this.width]));
|
||||
// Add the combined values of this pixel to previously
|
||||
// considered pixels.
|
||||
A2 += gradX * gradX;
|
||||
A1B2 += gradX * gradY;
|
||||
B1 += gradY * gradY;
|
||||
C2 += gradX * gradT;
|
||||
C1 += gradY * gradT;
|
||||
scaleFactor++;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Use the accumalated values from the for loop to determine a
|
||||
// motion direction.
|
||||
vector = this.getMotionVector(A2, A1B2, B1, C2, C1);
|
||||
if (scaleFactor) {
|
||||
// Store the area of the sprite in pixels
|
||||
activePixelNum = scaleFactor;
|
||||
scaleFactor /= (2 * winSize * 2 * winSize);
|
||||
vector.u = vector.u / scaleFactor;
|
||||
vector.v = vector.v / scaleFactor;
|
||||
}
|
||||
// Scale the magnitude of the averaged UV vector and the number of
|
||||
// overlapping solid pixels.
|
||||
aSprite.motionAmount = Math.round(
|
||||
localAmountScale * activePixelNum
|
||||
* Math.hypot(vector.u, vector.v)
|
||||
);
|
||||
if (aSprite.motionAmount > localMaxAmount) {
|
||||
// Clip all magnitudes greater than 100.
|
||||
aSprite.motionAmount = Math.min(localMaxAmount, 100);
|
||||
}
|
||||
if (aSprite.motionAmount > localThreshold) {
|
||||
// Snap direction.
|
||||
aSprite.motionDirection = (((
|
||||
Math.atan2(vector.v, vector.u)
|
||||
* this.toDegree + 270) % 360) - 180)
|
||||
.toFixed(2);
|
||||
}
|
||||
// Skip future calls on this state until a new frame is added.
|
||||
aSprite.frameNumber = this.frameNumber;
|
||||
}
|
||||
|
||||
/*
|
||||
* Get sprite image data scaled to 1 an converted to ABGR array
|
||||
*/
|
||||
function getSpriteImgageData(sprite) {
|
||||
var stage = sprite.parentThatIsA(StageMorph),
|
||||
newExtent = {
|
||||
x: Math.floor(sprite.extent().x / stage.scale),
|
||||
y: Math.floor(sprite.extent().y / stage.scale)
|
||||
},
|
||||
canvas = newCanvas(newExtent, true),
|
||||
canvasContext,
|
||||
imageData;
|
||||
canvasContext = canvas.getContext("2d");
|
||||
canvasContext.drawImage(
|
||||
sprite.image,
|
||||
0, 0, Math.floor(sprite.extent().x),
|
||||
Math.floor(sprite.extent().y),
|
||||
0, 0, newExtent.x, newExtent.y
|
||||
);
|
||||
imageData = canvas.getContext("2d")
|
||||
.getImageData(0, 0, newExtent.x, newExtent.y).data;
|
||||
return new Uint32Array(imageData.buffer.slice(0));
|
||||
}
|
||||
|
||||
/*
|
||||
* Return sprite's visible part bounds
|
||||
*/
|
||||
function getClippedBounds(sprite) {
|
||||
var stage = sprite.parentThatIsA(StageMorph),
|
||||
scale = stage.scale,
|
||||
bounds = {
|
||||
sx: 0,
|
||||
sy: 0,
|
||||
sw: Math.floor(sprite.extent().x / scale),
|
||||
sh: Math.floor(sprite.extent().y / scale)
|
||||
};
|
||||
// Clipping X
|
||||
if (sprite.left() < stage.left()) { // sprite outer left stage
|
||||
bounds.sw = Math.max(
|
||||
Math.floor((sprite.right() - stage.left()) / scale),
|
||||
0);
|
||||
bounds.sx = Math.floor(sprite.width() / scale - bounds.sw);
|
||||
}
|
||||
if (sprite.right() > stage.right()) { // sprite outer right stage
|
||||
bounds.sw = Math.max(
|
||||
Math.floor((stage.right() - sprite.left()) / scale),
|
||||
0);
|
||||
}
|
||||
//Clipping Y
|
||||
if (sprite.top() < stage.top()) { // sprite upper top
|
||||
bounds.sh = Math.max(
|
||||
Math.floor((sprite.bottom() - stage.top()) / scale),
|
||||
0);
|
||||
bounds.sy = Math.floor(sprite.height() / scale - bounds.sh);
|
||||
}
|
||||
if (sprite.bottom() > stage.bottom()) { // sprite lower bottom
|
||||
bounds.sh = Math.max(
|
||||
Math.floor((stage.bottom() - sprite.top()) / scale),
|
||||
0);
|
||||
}
|
||||
return bounds;
|
||||
}
|
||||
};
|
||||
|
|
|
@ -0,0 +1,415 @@
|
|||
/*
|
||||
|
||||
video.js
|
||||
|
||||
video motion detection for marphic.js and Snap!
|
||||
|
||||
written by Josep Ferràndiz i Farré
|
||||
https://github.com/jferran6
|
||||
|
||||
Copyright (C) 2019 by Josep Ferràndiz i Farré
|
||||
|
||||
This file is part of Snap!.
|
||||
|
||||
Snap! is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of
|
||||
the License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
prerequisites:
|
||||
--------------
|
||||
needs morphic.js
|
||||
|
||||
|
||||
prerequisites:
|
||||
--------------
|
||||
additional symbols have been contributed by members of the Snap!
|
||||
open-source community, especially by Bernat Romagosa
|
||||
|
||||
*/
|
||||
|
||||
/*global modules, StageMorph, newCanvas*/
|
||||
|
||||
// Global stuff ////////////////////////////////////////////////////////
|
||||
|
||||
modules.video = '2019-May-07';
|
||||
|
||||
var VideoMotion;
|
||||
|
||||
// VideoMotion /////////////////////////////////////////////////////////////////
|
||||
|
||||
function VideoMotion(width, height) {
|
||||
/*
|
||||
* Calculate, based on two consecutive video frames, the amount of movement and
|
||||
* direction of this movement both on the stage and on the sprite.
|
||||
* It's based on Scratch 3 (optical flow algorithm).
|
||||
*/
|
||||
this.width = width;
|
||||
this.height = height;
|
||||
this.frameNumber = 0;
|
||||
this.winSize = 8;
|
||||
this.lastAnalyzedFrame = 0;
|
||||
this.motionAmount = 0;
|
||||
this.motionDirection = 0;
|
||||
this.imageBuffer = new ArrayBuffer(this.width * this.height * 2);
|
||||
this.curr = new Uint8ClampedArray(
|
||||
this.imageBuffer,
|
||||
0,
|
||||
this.width * this.height
|
||||
);
|
||||
this.prev = new Uint8ClampedArray(
|
||||
this.imageBuffer,
|
||||
this.width * this.height,
|
||||
this.width * this.height
|
||||
);
|
||||
this.threshold = 30;
|
||||
this.amountScale = 100;
|
||||
this.toDegree = 180 / Math.PI;
|
||||
}
|
||||
|
||||
VideoMotion.prototype.reset = function(width, height){
|
||||
/*
|
||||
* Reset videoElement and videoMotion dimensions.
|
||||
* This function is called when stage dimensions change.
|
||||
*/
|
||||
this.width = width;
|
||||
this.height = height;
|
||||
this.frameNumber = 0;
|
||||
this.lastAnalyzedFrame = 0;
|
||||
this.imageBuffer = new ArrayBuffer(this.width * this.height * 2);
|
||||
this.curr = new Uint8ClampedArray(
|
||||
this.imageBuffer,
|
||||
0,
|
||||
this.width * this.height
|
||||
);
|
||||
this.prev = new Uint8ClampedArray(
|
||||
this.imageBuffer,
|
||||
this.width * this.height,
|
||||
this.width * this.height
|
||||
);
|
||||
};
|
||||
|
||||
VideoMotion.prototype.addFrame = function(imageData) {
|
||||
var i,
|
||||
temp = this.prev,
|
||||
frame = new Uint32Array(imageData.buffer.slice(0)); //ABGR
|
||||
this.frameNumber++;
|
||||
this.prev = this.curr;
|
||||
this.curr = temp;
|
||||
for (i = 0; i < frame.length; i++) {
|
||||
this.curr[i] = frame[i] & 0xff;
|
||||
}
|
||||
};
|
||||
|
||||
VideoMotion.prototype.getStageMotion = function() {
|
||||
var uu = 0, // Accumulate 2d motion vectors from groups
|
||||
vv = 0, // of pixels and average it later.
|
||||
n = 0,
|
||||
vector = {
|
||||
u: 0,
|
||||
v: 0
|
||||
},
|
||||
i, j, address, nextAddress, maxAddress,
|
||||
winStep = this.winSize * 2 + 1,
|
||||
wmax = this.width - this.winSize - 1,
|
||||
hmax = this.height - this.winSize - 1,
|
||||
// Optical Flow vars
|
||||
A2, A1B2, B1, C1, C2,
|
||||
gradX, gradY, gradT;
|
||||
|
||||
if (!this.curr || !this.prev) {
|
||||
this.motionAmount = this.motionDirection = -1;
|
||||
// Don't have two frames to analyze yet
|
||||
return;
|
||||
}
|
||||
// Return early if new data has not been received.
|
||||
if (this.lastAnalyzedFrame === this.frameNumber) {
|
||||
return;
|
||||
}
|
||||
this.lastAnalyzedFrame = this.frameNumber;
|
||||
// Iterate over groups of cells building up the components to determine
|
||||
// a motion vector for each cell instead of the whole frame to avoid
|
||||
// integer overflows.
|
||||
for (i = this.winSize + 1; i < hmax; i += winStep) {
|
||||
for (j = this.winSize + 1; j < wmax; j += winStep) {
|
||||
A2 = 0;
|
||||
A1B2 = 0;
|
||||
B1 = 0;
|
||||
C1 = 0;
|
||||
C2 = 0;
|
||||
// This is a performance critical math region.
|
||||
address = ((i - this.winSize) * this.width) + j - this.winSize;
|
||||
nextAddress = address + winStep;
|
||||
maxAddress = ((i + this.winSize) * this.width) + j + this.winSize;
|
||||
for (; address <= maxAddress; address += this.width - winStep,
|
||||
nextAddress += this.width) {
|
||||
for (; address <= nextAddress; address += 1) {
|
||||
// The difference in color between the last frame and
|
||||
// the current frame.
|
||||
gradT = ((this.prev[address]) - (this.curr[address]));
|
||||
// The difference between the pixel to the left and the
|
||||
// pixel to the right.
|
||||
gradX = ((this.curr[address - 1]) - (this.curr[address + 1]));
|
||||
// The difference between the pixel above and the pixel
|
||||
// below.
|
||||
gradY = ((
|
||||
this.curr[address - this.width])
|
||||
- (this.curr[address + this.width]));
|
||||
// Add the combined values of this pixel to previously
|
||||
// considered pixels.
|
||||
A2 += gradX * gradX;
|
||||
A1B2 += gradX * gradY;
|
||||
B1 += gradY * gradY;
|
||||
C2 += gradX * gradT;
|
||||
C1 += gradY * gradT;
|
||||
}
|
||||
}
|
||||
// Use the accumalated values from the for loop to determine a
|
||||
// motion direction.
|
||||
vector = this.getMotionVector(A2, A1B2, B1, C2, C1);
|
||||
// If u and v are within negative winStep to positive winStep,
|
||||
// add them to a sum that will later be averaged.
|
||||
if (-winStep < vector.u
|
||||
&& vector.u < winStep
|
||||
&& -winStep < vector.v
|
||||
&& vector.v < winStep) {
|
||||
uu += vector.u;
|
||||
vv += vector.v;
|
||||
n++;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Average the summed vector values of all of the motion groups.
|
||||
uu /= n;
|
||||
vv /= n;
|
||||
// Scale the magnitude of the averaged UV vector.
|
||||
this.motionAmount = Math.round(this.amountScale * Math.hypot(uu, vv));
|
||||
if (this.motionAmount > this.threshold) {
|
||||
this.motionDirection = (((Math.atan2(vv, uu) * this.toDegree + 270) % 360) - 180)
|
||||
.toFixed(2); // Snap direction
|
||||
}
|
||||
};
|
||||
|
||||
VideoMotion.prototype.getMotionVector = function(A2, A1B2, B1, C2, C1) {
|
||||
/**
|
||||
* Determine a motion vector combinations of the color component difference
|
||||
* on the x axis, y axis, and temporal axis.
|
||||
* A2 - a sum of x axis squared
|
||||
* A1B2 - a sum of x axis times y axis
|
||||
* B1 - a sum of y axis squared
|
||||
* C2 - a sum of x axis times temporal axis
|
||||
* C1 - a sum of y axis times temporal axis
|
||||
* Returns a uv vector representing the motion for the given input
|
||||
*/
|
||||
// Compare sums of X * Y and sums of X squared and Y squared.
|
||||
var norm,
|
||||
IGradNorm,
|
||||
delta = ((A1B2 * A1B2) - (A2 * B1)),
|
||||
deltaX, deltaY, Idelta,
|
||||
motionVector = {
|
||||
u: 0,
|
||||
v: 0
|
||||
};
|
||||
|
||||
if (delta) {
|
||||
// System is not singular - solving by Kramer method.
|
||||
deltaX = -((C1 * A1B2) - (C2 * B1));
|
||||
deltaY = -((A1B2 * C2) - (A2 * C1));
|
||||
Idelta = 8 / delta;
|
||||
motionVector.u = deltaX * Idelta;
|
||||
motionVector.v = deltaY * Idelta;
|
||||
} else {
|
||||
// Singular system - find optical flow in gradient direction.
|
||||
norm = ((A1B2 + A2) * (A1B2 + A2)) + ((B1 + A1B2) * (B1 + A1B2));
|
||||
if (norm) {
|
||||
IGradNorm = 8 / norm;
|
||||
motionVector.u = (A1B2 + A2) * (-(C1 + C2) * IGradNorm);
|
||||
motionVector.v = (B1 + A1B2) * (-(C1 + C2) * IGradNorm);
|
||||
} else {
|
||||
motionVector.u = 0;
|
||||
motionVector.v = 0;
|
||||
}
|
||||
}
|
||||
return motionVector;
|
||||
};
|
||||
|
||||
VideoMotion.prototype.getLocalMotion = function(aSprite) {
|
||||
/**
|
||||
* Calculate motion amount and direction values based on stored frames
|
||||
* (current and previous) that overlaps a given sprite.
|
||||
*/
|
||||
var stage = aSprite.parentThatIsA(StageMorph),
|
||||
activePixelNum = 0,
|
||||
i, j, xmin, xmax, ymin, ymax, gradT, gradX, gradY,
|
||||
spriteWidth = Math.floor(aSprite.width() / stage.scale),
|
||||
winSize = this.winSize,
|
||||
vector = {
|
||||
u: 0,
|
||||
v: 0
|
||||
},
|
||||
A2 = 0,
|
||||
A1B2 = 0,
|
||||
B1 = 0,
|
||||
C1 = 0,
|
||||
C2 = 0,
|
||||
localThreshold = this.threshold / 3,
|
||||
localMaxAmount = 100,
|
||||
localAmountScale = this.amountScale * 2e-4,
|
||||
scaleFactor = 0,
|
||||
address = 0,
|
||||
spriteImage,
|
||||
cb,
|
||||
pixel;
|
||||
|
||||
if (!this.curr || !this.prev) {
|
||||
aSprite.motionAmount = aSprite.motionDirection = -1;
|
||||
// Don't have two frames to analyze yet
|
||||
return;
|
||||
}
|
||||
// Skip if the current frame has already been considered
|
||||
// for this state.
|
||||
if (aSprite.frameNumber !== this.frameNumber) {
|
||||
spriteImage = getSpriteImgageData(aSprite);
|
||||
// Consider only the area of the current frame overlapped
|
||||
// with the given sprite.
|
||||
cb = getClippedBounds(aSprite);
|
||||
xmin = Math.max(
|
||||
Math.floor((aSprite.left() - stage.left()) / stage.scale),
|
||||
0);
|
||||
ymin = Math.max(
|
||||
Math.floor((aSprite.top() - stage.top()) / stage.scale),
|
||||
0);
|
||||
xmax = Math.min(cb.sw + xmin, stage.dimensions.x);
|
||||
ymax = Math.min(cb.sh + ymin, stage.dimensions.y - 1);
|
||||
// This is a performance critical math region.
|
||||
pixel = cb.sy * spriteWidth + cb.sx;
|
||||
for (i = ymin; i < ymax; i++, pixel += spriteWidth - cb.sw) { //rows
|
||||
for (j = xmin; j < xmax; j++, ++pixel) { //cols
|
||||
if (j > 0 && j < this.width && i > 0 && i < this.height
|
||||
&& (spriteImage[pixel] >> 24 & 0xff) == 0xff) {
|
||||
address = (i * this.width) + j;
|
||||
// The difference in color between the last frame and
|
||||
// the current frame.
|
||||
gradT = ((this.prev[address]) - (this.curr[address]));
|
||||
// The difference between the pixel to the left and the
|
||||
// pixel to the right.
|
||||
gradX = ((this.curr[address - 1]) - (this.curr[address + 1]));
|
||||
// The difference between the pixel above and the pixel
|
||||
// below.
|
||||
gradY = (
|
||||
(this.curr[address - this.width])
|
||||
- (this.curr[address + this.width]));
|
||||
// Add the combined values of this pixel to previously
|
||||
// considered pixels.
|
||||
A2 += gradX * gradX;
|
||||
A1B2 += gradX * gradY;
|
||||
B1 += gradY * gradY;
|
||||
C2 += gradX * gradT;
|
||||
C1 += gradY * gradT;
|
||||
scaleFactor++;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Use the accumalated values from the for loop to determine a
|
||||
// motion direction.
|
||||
vector = this.getMotionVector(A2, A1B2, B1, C2, C1);
|
||||
if (scaleFactor) {
|
||||
// Store the area of the sprite in pixels
|
||||
activePixelNum = scaleFactor;
|
||||
scaleFactor /= (2 * winSize * 2 * winSize);
|
||||
vector.u = vector.u / scaleFactor;
|
||||
vector.v = vector.v / scaleFactor;
|
||||
}
|
||||
// Scale the magnitude of the averaged UV vector and the number of
|
||||
// overlapping solid pixels.
|
||||
aSprite.motionAmount = Math.round(
|
||||
localAmountScale * activePixelNum
|
||||
* Math.hypot(vector.u, vector.v)
|
||||
);
|
||||
if (aSprite.motionAmount > localMaxAmount) {
|
||||
// Clip all magnitudes greater than 100.
|
||||
aSprite.motionAmount = Math.min(localMaxAmount, 100);
|
||||
}
|
||||
if (aSprite.motionAmount > localThreshold) {
|
||||
// Snap direction.
|
||||
aSprite.motionDirection = (((
|
||||
Math.atan2(vector.v, vector.u)
|
||||
* this.toDegree + 270) % 360) - 180)
|
||||
.toFixed(2);
|
||||
}
|
||||
// Skip future calls on this state until a new frame is added.
|
||||
aSprite.frameNumber = this.frameNumber;
|
||||
}
|
||||
|
||||
/*
|
||||
* Get sprite image data scaled to 1 an converted to ABGR array
|
||||
*/
|
||||
function getSpriteImgageData(sprite) {
|
||||
var stage = sprite.parentThatIsA(StageMorph),
|
||||
newExtent = {
|
||||
x: Math.floor(sprite.extent().x / stage.scale),
|
||||
y: Math.floor(sprite.extent().y / stage.scale)
|
||||
},
|
||||
canvas = newCanvas(newExtent, true),
|
||||
canvasContext,
|
||||
imageData;
|
||||
canvasContext = canvas.getContext("2d");
|
||||
canvasContext.drawImage(
|
||||
sprite.image,
|
||||
0, 0, Math.floor(sprite.extent().x),
|
||||
Math.floor(sprite.extent().y),
|
||||
0, 0, newExtent.x, newExtent.y
|
||||
);
|
||||
imageData = canvas.getContext("2d")
|
||||
.getImageData(0, 0, newExtent.x, newExtent.y).data;
|
||||
return new Uint32Array(imageData.buffer.slice(0));
|
||||
}
|
||||
|
||||
/*
|
||||
* Return sprite's visible part bounds
|
||||
*/
|
||||
function getClippedBounds(sprite) {
|
||||
var stage = sprite.parentThatIsA(StageMorph),
|
||||
scale = stage.scale,
|
||||
bounds = {
|
||||
sx: 0,
|
||||
sy: 0,
|
||||
sw: Math.floor(sprite.extent().x / scale),
|
||||
sh: Math.floor(sprite.extent().y / scale)
|
||||
};
|
||||
// Clipping X
|
||||
if (sprite.left() < stage.left()) { // sprite outer left stage
|
||||
bounds.sw = Math.max(
|
||||
Math.floor((sprite.right() - stage.left()) / scale),
|
||||
0);
|
||||
bounds.sx = Math.floor(sprite.width() / scale - bounds.sw);
|
||||
}
|
||||
if (sprite.right() > stage.right()) { // sprite outer right stage
|
||||
bounds.sw = Math.max(
|
||||
Math.floor((stage.right() - sprite.left()) / scale),
|
||||
0);
|
||||
}
|
||||
//Clipping Y
|
||||
if (sprite.top() < stage.top()) { // sprite upper top
|
||||
bounds.sh = Math.max(
|
||||
Math.floor((sprite.bottom() - stage.top()) / scale),
|
||||
0);
|
||||
bounds.sy = Math.floor(sprite.height() / scale - bounds.sh);
|
||||
}
|
||||
if (sprite.bottom() > stage.bottom()) { // sprite lower bottom
|
||||
bounds.sh = Math.max(
|
||||
Math.floor((stage.bottom() - sprite.top()) / scale),
|
||||
0);
|
||||
}
|
||||
return bounds;
|
||||
}
|
||||
};
|
Ładowanie…
Reference in New Issue