diff --git a/gulpfile.js b/gulpfile.js index 6ac27dc..22c9abd 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -6,38 +6,73 @@ var gulp = require("gulp"), addSrc = require("gulp-add-src"), concat = require("gulp-concat"), jshint = require("gulp-jshint"), - KarmaServer = require('karma').Server; + KarmaServer = require('karma').Server, + path = require("path"), + filter = require("gulp-filter"); -gulp.task("default", function() { - return gulp.src("untar-worker.js") +gulp.task("build:dev", function() { + var f = filter(['*', '!untar-worker.js'], { restore: true }); + + return gulp.src(["src/untar.js"]) .pipe(sourcemaps.init()) + .pipe(insert.append("\nworkerScriptUri = 'untar-worker.js';")) + .pipe(addSrc(["src/ProgressivePromise.js", "src/untar-worker.js"])) .pipe(jshint()) .pipe(jshint.reporter("default")) .pipe(jshint.reporter("fail")) + .pipe(insert.prepend('"use strict";\n')) + .pipe(f) + .pipe(umd({ + dependencies: function(file) { + if (path.basename(file.path) === "untar.js") { + return ["ProgressivePromise"]; + } else { + return []; + } + }, + exports: function(file) { + return path.basename(file.path, path.extname(file.path)); + }, + namespace: function(file) { + return path.basename(file.path, path.extname(file.path)); + } + })) + .pipe(f.restore) + .pipe(sourcemaps.write()) + .pipe(gulp.dest("build/dev")); +}); + +gulp.task("build:dist", function() { + return gulp.src("src/untar-worker.js") + .pipe(jshint()) + .pipe(jshint.reporter("default")) + .pipe(jshint.reporter("fail")) + .pipe(insert.prepend('"use strict";\n')) .pipe(uglify()) .pipe(insert.transform(function(contents, file) { - var str = ["\nvar workerScriptUri = URL.createObjectURL(createBlob([\""]; + var str = ["\nworkerScriptUri = URL.createObjectURL(createBlob([\""]; str.push(contents.replace(/"/g, '\\"')); str.push("\"]));"); return str.join(""); })) - .pipe(addSrc("untar.js")) + .pipe(addSrc(["src/ProgressivePromise.js", "src/untar.js"])) .pipe(jshint()) .pipe(jshint.reporter("default")) .pipe(jshint.reporter("fail")) .pipe(concat("untar.js")) + .pipe(insert.prepend('"use strict";\n')) .pipe(umd({ exports: function() { return "untar"; }, namespace: function() { return "untar"; } })) - .pipe(sourcemaps.write()) - .pipe(gulp.dest("build/dev")) .pipe(uglify()) .pipe(gulp.dest("build/dist")); }); -gulp.task("test", ["default"], function(done) { +gulp.task("default", ["build:dev", "build:dist"]); + +gulp.task("test", ["build:dev"], function(done) { new KarmaServer({ configFile: __dirname + '/karma.conf.js', singleRun: true diff --git a/karma.conf.js b/karma.conf.js index e2e2c52..efd2a53 100644 --- a/karma.conf.js +++ b/karma.conf.js @@ -16,9 +16,9 @@ module.exports = function(config) { // list of files / patterns to load in the browser files: [ 'https://www.promisejs.org/polyfills/promise-6.1.0.js', - 'test-main.js', {pattern: 'build/dev/**/*.js', included: false}, - {pattern: 'spec/**/*.*', included: false} + {pattern: 'spec/**/*.*', included: false}, + 'test-main.js' ], @@ -53,12 +53,12 @@ module.exports = function(config) { // enable / disable watching file and executing tests whenever any file changes - autoWatch: false, + autoWatch: true, // start these browsers // available browser launchers: https://npmjs.org/browse/keyword/karma-launcher - browsers: ['PhantomJS'], + browsers: ['Chrome'], browserNoActivityTimeout: 60000, diff --git a/package.json b/package.json index b392a05..a8891e3 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "test": "test" }, "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" + "test": "gulp test" }, "repository": { "type": "git", @@ -30,6 +30,7 @@ "gulp": "^3.9.0", "gulp-add-src": "^0.2.0", "gulp-concat": "^2.6.0", + "gulp-filter": "^3.0.1", "gulp-insert": "^0.5.0", "gulp-jshint": "^1.11.2", "gulp-sourcemaps": "^1.5.2", @@ -37,6 +38,7 @@ "gulp-umd": "^0.2.0", "jasmine-core": "^2.3.4", "karma": "^0.13.9", + "karma-chrome-launcher": "^0.2.0", "karma-jasmine": "^0.3.6", "karma-phantomjs-launcher": "^0.2.1", "karma-requirejs": "^0.2.2", diff --git a/spec/ProgressivePromise-spec.js b/spec/ProgressivePromise-spec.js new file mode 100644 index 0000000..128fcb4 --- /dev/null +++ b/spec/ProgressivePromise-spec.js @@ -0,0 +1,51 @@ +define(["ProgressivePromise"], function(ProgressivePromise) { + + describe("ProgressivePromise", function() { + it("should report progress events as they happen", function(done) { + var p = new ProgressivePromise(function(resolve, reject, progress) { + setTimeout(function() { progress(1); }, 5); + setTimeout(function() { progress(2); }, 10); + setTimeout(resolve, 15); + }); + + var r = []; + + p.progress(function(value) { + r.push(value); + }); + + p.then(function() { + if (r[0] === 1 && r[1] === 2) { + done(); + } else { + done.fail(); + } + }); + }); + + it("should report progress events after they've happened", function(done) { + var p = new ProgressivePromise(function(resolve, reject, progress) { + progress(1); + progress(2); + resolve(); + }); + + setTimeout(function() { + var r = []; + + p.progress(function(value) { + r.push(value); + }); + + p.then(function() { + if (r[0] === 1 && r[1] === 2) { + done(); + } else { + done.fail(); + } + }); + }, 5); + }); + }); + +}); diff --git a/spec/untar-spec.js b/spec/untar-spec.js index ed9803d..631300a 100644 --- a/spec/untar-spec.js +++ b/spec/untar-spec.js @@ -1,18 +1,37 @@ -define(["build/dev/untar"], function(untar) { +define(["untar"], function(untar) { + describe("untar", function() { - it("should unpack 3 files and a directory with 3 files", function(done) { - untar("/base/spec/data/test.tar", { - onExtract: function(file) { done(); } - }).then( + + console.log("untar: " + JSON.stringify(untar)); + + var fileNames = [ + "1.txt", + "2.txt", + "3.txt", + "directory/", + "directory/1.txt", + "directory/2.txt", + "directory/3.txt" + ]; + + it("should unpack 3 specific files and a directory with 3 specific files", function(done) { + var i = 0; + + untar("/base/spec/data/test.tar").then( function(files) { - expect(files.length).toBe(6); + expect(files.length).toBe(7); done(); }, function(err) { done.fail(JSON.stringify(err)); + }, + function(file) { + expect(file).toBeDefined(); + expect(file.name).toBe(fileNames[i]); + i += 1; } ); }, 20000); }); -}); +}); diff --git a/src/ProgressivePromise.js b/src/ProgressivePromise.js new file mode 100644 index 0000000..846fd22 --- /dev/null +++ b/src/ProgressivePromise.js @@ -0,0 +1,51 @@ +/* globals window: false, Promise: false */ + +/** +Returns a Promise decorated with a progress() event. +*/ +function ProgressivePromise(fn) { + if (typeof Promise !== "function") { + throw new Error("Promise implementation not available in this environment."); + } + + var progressCallbacks = []; + var progressHistory = []; + + function doProgress(value) { + for (var i = 0, l = progressCallbacks.length; i < l; ++i) { + progressCallbacks[i](value); + } + + progressHistory.push(value); + } + + var promise = new Promise(function(resolve, reject) { + fn(resolve, reject, doProgress); + }); + + promise.progress = function(cb) { + if (typeof cb !== "function") { + throw new Error("cb is not a function."); + } + + // Report the previous progress history + for (var i = 0, l = progressHistory.length; i < l; ++i) { + cb(progressHistory[i]); + } + + progressCallbacks.push(cb); + return promise; + }; + + promise.then = function(onSuccess, onFail, onProgress) { + Promise.prototype.then.call(promise, onSuccess, onFail); + + if (onProgress !== undefined) { + promise.progress(onProgress); + } + + return promise; + }; + + return promise; +} \ No newline at end of file diff --git a/src/createBlob.js b/src/createBlob.js new file mode 100644 index 0000000..a93f79f --- /dev/null +++ b/src/createBlob.js @@ -0,0 +1,20 @@ +"use strict"; + +var createBlob = (function() { + if (typeof window.Blob === "function") { + return function(dataArray) { return new Blob(dataArray); }; + } else { + var BBuilder = window.BlobBuilder || window.WebKitBlobBuilder; + + return function(dataArray) { + var builder = new BBuilder(); + + for (var i = 0; i < dataArray.length; ++i) { + var v = dataArray[i]; + builder.append(v); + } + + return builder.getBlob(); + }; + } +}()); \ No newline at end of file diff --git a/src/untar-worker.js b/src/untar-worker.js new file mode 100644 index 0000000..db2db5f --- /dev/null +++ b/src/untar-worker.js @@ -0,0 +1,181 @@ +/* globals postMessage: false, DataView: false, self: false, window: false, ArrayBuffer: false, Uint8Array: false */ + +function UntarWorker() { + +} + +UntarWorker.prototype = { + onmessage: function(msg) { + try { + if (msg.data.type === "extract") { + this.untarBuffer(msg.data.buffer); + } else { + throw new Error("Unknown message type: " + msg.data.type); + } + } catch (err) { + this.postError(err); + } + }, + + postError: function(err) { + this.postMessage({ type: "error", data: err }); + }, + + postLog: function(level, msg) { + this.postMessage({ type: "log", data: { level: level, msg: msg }}); + }, + + untarBuffer: function(arrayBuffer) { + try { + var tarFileStream = new UntarFileStream(arrayBuffer); + while (tarFileStream.hasNext()) { + var file = tarFileStream.next(); + + this.postMessage({ type: "extract", data: file }, [file.buffer]); + } + + this.postMessage({ type: "complete" }); + } catch (err) { + this.postError(err); + } + }, + + postMessage: function(msg, transfers) { + self.postMessage(msg, transfers); + } +}; + +if (typeof self !== "undefined") { + // We're running in a worker thread + var worker = new UntarWorker(); + self.onmessage = function(msg) { worker.onmessage(msg); }; +} + +function TarFile() { + +} + +function UntarStream(arrayBuffer) { + this._bufferView = new DataView(arrayBuffer); + this._position = 0; +} + +UntarStream.prototype = { + readString: function(charCount) { + //console.log("readString: position " + this.position() + ", " + charCount + " chars"); + var charSize = 1; + var byteCount = charCount * charSize; + + var charCodes = []; + + for (var i = 0; i < charCount; ++i) { + var charCode = this._bufferView.getUint8(this.position() + (i * charSize), true); + if (charCode !== 0) { + charCodes.push(charCode); + } else { + break; + } + } + + this.seek(byteCount); + + return String.fromCharCode.apply(null, charCodes); + }, + + readBuffer: function(byteCount) { + var buf; + + if (typeof ArrayBuffer.prototype.slice === "function") { + buf = this._bufferView.buffer.slice(this.position(), this.position() + byteCount); + } else { + buf = new ArrayBuffer(byteCount); + var target = new Uint8Array(buf); + var src = new Uint8Array(this._bufferView.buffer, this.position(), byteCount); + target.set(src); + } + + this.seek(byteCount); + return buf; + }, + + seek: function(byteCount) { + this._position += byteCount; + }, + + peekUint32: function() { + return this._bufferView.getUint32(this.position(), true); + }, + + position: function(newpos) { + if (newpos === undefined) { + return this._position; + } else { + this._position = newpos; + } + }, + + size: function() { + return this._bufferView.byteLength; + } +}; + +function UntarFileStream(arrayBuffer) { + this._stream = new UntarStream(arrayBuffer); +} + +UntarFileStream.prototype = { + hasNext: function() { + // A tar file ends with 4 zero bytes + return this._stream.position() + 4 < this._stream.size() && this._stream.peekUint32() !== 0; + }, + + next: function() { + var stream = this._stream; + var file = new TarFile(); + + var headerBeginPos = stream.position(); + var dataBeginPos = headerBeginPos + 512; + + // Read header + file.name = stream.readString(100); + file.mode = stream.readString(8); + file.uid = stream.readString(8); + file.gid = stream.readString(8); + file.size = parseInt(stream.readString(12), 8); + file.modificationTime = parseInt(stream.readString(12), 8); + file.checksum = stream.readString(8); + file.type = stream.readString(1); + file.linkname = stream.readString(1); + file.ustarFormat = stream.readString(6); + + if (file.ustarFormat === "ustar") { + file.version = stream.readString(2); + file.uname = stream.readString(32); + file.gname = stream.readString(32); + file.devmajor = stream.readString(8); + file.devminor = stream.readString(8); + file.namePrefix = stream.readString(155); + + if (file.namePrefix.length > 0) { + file.name = file.namePrefix + file.name; + } + } + + stream.position(dataBeginPos); + + // Normal file is either "\0" or 0. + if (file.type === "0" || file.type === "\0") { + file.buffer = stream.readBuffer(file.size); + } else if (file.type == 5) { + // Directory - should we do anything with this? Nope! + } else { + // We only care about real files, not symlinks. + } + + // File data is padded to reach a 512 byte boundary; skip the padded bytes. + var dataEndPos = dataBeginPos + (file.size > 0 ? file.size + (512 - file.size % 512) : 0); + stream.position(dataEndPos); + + return file; + } +}; \ No newline at end of file diff --git a/src/untar.js b/src/untar.js new file mode 100644 index 0000000..4bbd4b3 --- /dev/null +++ b/src/untar.js @@ -0,0 +1,82 @@ +/* globals window: false, Blob: false, Promise: false, console: false, Worker: false, ProgressivePromise: false */ + +var workerScriptUri; // Included at compile time + +var URL = window.URL || window.webkitURL; + +var createBlob = (function() { + if (typeof window.Blob === "function") { + return function(dataArray) { return new Blob(dataArray); }; + } else { + var BBuilder = window.BlobBuilder || window.WebKitBlobBuilder; + + return function(dataArray) { + var builder = new BBuilder(); + + for (var i = 0; i < dataArray.length; ++i) { + var v = dataArray[i]; + builder.append(v); + } + + return builder.getBlob(); + }; + } +}()); + +/** +Returns a ProgressivePromise. +*/ +function untar(arrayBuffer) { + if (!window.Worker) { + throw new Error("Worker implementation not available in this environment."); + } + + return new ProgressivePromise(function(resolve, reject, progress) { + var worker = new Worker(workerScriptUri); + + var files = []; + + worker.onmessage = function(message) { + message = message.data; + + switch (message.type) { + case "log": + console[message.data.level]("Worker: " + message.data.msg); + break; + case "extract": + var file = decorateExtractedFile(message.data); + files.push(file); + progress(file); + break; + case "complete": + resolve(files); + break; + case "error": + reject(message.data); + break; + default: + reject(new Error("Unknown message from worker: " + message.type)); + break; + } + }; + + //console.info("Sending arraybuffer to worker for extraction."); + worker.postMessage({ type: "extract", buffer: arrayBuffer }, [arrayBuffer]); + }); +} + +function decorateExtractedFile(file) { + file.blob = createBlob([file.buffer]); + delete file.buffer; + + var blobUrl; + file.getObjectUrl = function() { + if (!blobUrl) { + blobUrl = URL.createObjectURL(file.blob); + } + + return blobUrl; + }; + + return file; +} \ No newline at end of file diff --git a/test-main.js b/test-main.js index c2173ad..08a1383 100644 --- a/test-main.js +++ b/test-main.js @@ -8,13 +8,13 @@ Object.keys(window.__karma__.files).forEach(function(file) { // If you require sub-dependencies of test files to be loaded as-is (requiring file extension) // then do not normalize the paths var normalizedTestModule = file.replace(/^\/base\/|\.js$/g, ''); - allTestFiles.push(normalizedTestModule); + allTestFiles.push("../../" + normalizedTestModule); } }); require.config({ // Karma serves files under /base, which is the basePath from your config file - baseUrl: '/base', + baseUrl: '/base/build/dev', // dynamically load all test files deps: allTestFiles, diff --git a/untar-worker.js b/untar-worker.js deleted file mode 100644 index 833e196..0000000 --- a/untar-worker.js +++ /dev/null @@ -1,165 +0,0 @@ -/* globals postMessage: false, DataView: false, self: false, onmessage: true */ -/* jshint -W097 */ -"use strict"; - -onmessage = function(e) { - postMessage("test"); -}; - -self.onmessage = function(msg) { - postLog("info", "Received message."); - try { - if (msg.data.type === "extract") { - untarBuffer(msg.data.buffer); - } else { - throw new Error("Unknown message type."); - } - } catch (err) { - postError(err); - } -}; - -function postError(err) { - postMessage({ type: "error", data: err }); -} - -function postLog(level, msg) { - postMessage({ type: "log", data: { level: level, msg: msg }}); -} - -function untarBuffer(arrayBuffer) { - try { - postLog("info", "buffer size: " + arrayBuffer.byteLength); - var tarFileStream = new TarFileStream(arrayBuffer); - while (tarFileStream.hasNext()) { - var file = tarFileStream.next(); - - if (file.buffer) { - postMessage({ type: "extract", data: file }, [file.buffer]); - } - } - - postMessage({ type: "complete" }); - } catch (err) { - postError(err); - } -} - -function TarFile() { - -} - -TarFile.prototype = { - -}; - -function Stream(arrayBuffer) { - this._bufferView = new DataView(arrayBuffer); - this._position = 0; -} - -Stream.prototype = { - readString: function(charCount) { - var charSize = 1; - var byteCount = charCount * charSize; - - var charCodes = []; - - for (var i = 0; i < charCount; ++i) { - var charCode = this._bufferView.getUint8(this.position() + (i * charSize)); - if (charCode !== 0) { - charCodes.push(charCode); - } else { - break; - } - } - - this.seek(byteCount); - - return String.fromCharCode.apply(null, charCodes); - }, - - readBuffer: function(byteCount) { - return this._bufferView.buffer.slice(this._position, byteCount); - }, - - seek: function(byteCount) { - this._position += byteCount; - }, - - peekUint32: function() { - return this._bufferView.getUint32(this.position()); - }, - - position: function(newpos) { - if (newpos === undefined) { - return this._position; - } else { - this._position = newpos; - } - }, - - size: function() { - return this._bufferView.byteLength; - } -}; - -function TarFileStream(arrayBuffer) { - this._stream = new Stream(arrayBuffer); -} - -TarFileStream.prototype = { - hasNext: function() { - return this._stream.position() < this._stream.size() && this._stream.peekUint32() !== 0; - }, - - next: function() { - var stream = this._stream; - var file = new TarFile(); - - var headerBeginPos = stream.position; - var dataBeginPos = headerBeginPos + 512; - - // Read header - file.name = stream.readString(100); - file.mode = stream.readString(8); - file.uid = stream.readString(8); - file.gid = stream.readString(8); - file.size = parseInt(stream.readString(12), 8); - file.modificationTime = parseInt(stream.readString(12), 8); - file.checksum = stream.readString(8); - file.type = stream.readString(1); - file.linkname = stream.readString(1); - file.ustarFormat = stream.readString(6); - - if (file.ustarFormat === "ustar") { - file.version = stream.readString(2); - file.uname = stream.readString(32); - file.gname = stream.readString(32); - file.devmajor = stream.readString(8); - file.devminor = stream.readString(8); - file.namePrefix = stream.readString(155); - - if (file.namePrefix.length > 0) { - file.name = file.namePrefix + file.name; - } - } - - stream.position(dataBeginPos); - - // Normal file is either "\0" or 0. - if (file.type === 0 || file.type === "\0") { - file.buffer = stream.readBuffer(file.size); - } else if (file.type == 5) { - // Directory - should we do anything with this? Nope! - } else { - // We only care about real files, not symlinks. - } - - // File data is padded to reach a 512 byte boundary; skip the padded bytes. - var bytesToSkipCount = 512 - file.size % 512; - stream.seek(bytesToSkipCount); - - return file; - } -}; \ No newline at end of file diff --git a/untar.js b/untar.js deleted file mode 100644 index fa5809e..0000000 --- a/untar.js +++ /dev/null @@ -1,178 +0,0 @@ -/* globals window: false, Blob: false, Promise: false, console: false, XMLHttpRequest: false, Worker: false */ -/* jshint -W097 */ -"use strict"; - -var workerScriptUri; // Included at compile time - -var URL = window.URL || window.webkitURL; - -var createBlob = (function() { - if (typeof window.Blob === "function") { - return function(dataArray) { return new Blob(dataArray); }; - } else { - var BlobBuilder = window.BlobBuilder || window.WebKitBlobBuilder; - - return function(dataArray) { - var builder = new BlobBuilder(); - - for (var i = 0; i < dataArray.length; ++i) { - builder.append(dataArray[i]); - } - - return builder.getBlob(); - }; - } -}()); - -function createBlob(dataArray) { - if (typeof window.Blob === "function") { - return new Blob(dataArray); - } else { - var BlobBuilder = window.BlobBuilder || window.WebKitBlobBuilder; - var builder = new BlobBuilder(); - - for (var i = 0; i < dataArray.length; ++i) { - builder.append(dataArray[i]); - } - - return builder.getBlob(); - } -} - -function loadArrayBuffer(uri) { - console.info("loadArrayBuffer called"); - - return new Promise(function(resolve, reject) { - var request = new XMLHttpRequest(); - - /* - request.addEventListener("progress", function(e) { - postMessage({ type: "loading", data: e }); - }); - */ - - request.addEventListener("load", function(e) { - if (request.status >= 200 && request.status < 400) { - resolve(request.response); - } else { - reject(new Error(request.status + " " + request.statusText)); - } - }); - - request.addEventListener("error", function(err) { reject(err); }); - request.addEventListener("abort", function(err) { reject(err); }); - - request.open("GET", uri, true); - request.responseType = "arraybuffer"; - request.send(); - }); -} - -/** -source = ArrayBuffer or a url string. If an ArrayBuffer, it will be transfered to the web worker and will thus not be available in the window after. -options = { - onComplete, - onLoading, // When downloading the tar from a url. - onExtract, - onError -} -*/ -function untar(source, options) { - console.info("untar called"); - - if (typeof Promise !== "function") { - throw new Error("Promise implementation not available in this environment."); - } - - if (!window.Worker) { - throw new Error("Worker implementation not available in this environment."); - } - - options = options || {}; - - return new Promise(function(resolve, reject) { - var noop = function() { }; - var onComplete = options.onComplete || noop; - var onLoading = options.onProgress || noop; - var onExtract = options.onExtract || noop; - var onError = options.onError || noop; - - var worker = new Worker(workerScriptUri); - - var files = []; - var msgData; - - worker.onmessage = function(message) { - message = message.data; - - switch (message.type) { - case "log": - console[message.data.level]("Worker: " + message.data.msg); - break; - case "loading": - onLoading(message.data); - break; - case "extract": - msgData = new TarFile(message.data); - files.push(msgData); - onExtract(msgData); - break; - case "complete": - onComplete(files); - resolve(files); - break; - case "error": - msgData = message.data; - onError(msgData); - reject(msgData); - break; - default: - msgData = new Error("Unknown message from worker: " + message.type); - onError(msgData); - reject(msgData); - break; - } - }; - - if (typeof source === "string") { - loadArrayBuffer(source).then( - function(buffer) { - console.info("Loaded tar file, sending to worker for extraction."); - worker.postMessage({ type: "extract", buffer: buffer }, [buffer]); - }, - function(err) { - onError(err); - reject(err); - } - ); - } else { - console.info("Sending tar file to worker for extraction."); - worker.postMessage({ type: "extract", buffer: source }, [source]); - } - }); -} - -function TarFile(orig) { - this._blobUrl = null; - - for (var p in orig) { - switch (p) { - case "buffer": - this.blob = createBlob([orig.buffer]); - break; - default: - this[p] = orig[p]; - break; - } - } -} - -TarFile.prototype = { - getObjectUrl: function() { - if (!this._blobUrl) { - this._blobUrl = URL.createObjectURL(this.blob); - } - - return this._blobUrl; - } -};