From 9247ffca22e2f08348386669b0b30527400280a1 Mon Sep 17 00:00:00 2001 From: Qvazar Date: Mon, 14 Sep 2015 13:28:49 +0200 Subject: [PATCH] All tests passing, some documentation. --- README.md | 49 +++++++++- gulpfile.js | 6 +- karma.conf.js | 3 +- spec/empty-spec.js | 0 spec/untar-spec.js | 97 ++++++++++++------ spec/untar-worker-spec.js | 200 ++++++++++++++++++++++++++++++++++++++ src/untar-worker.js | 11 ++- src/untar.js | 32 +++--- 8 files changed, 337 insertions(+), 61 deletions(-) delete mode 100644 spec/empty-spec.js create mode 100644 spec/untar-worker-spec.js diff --git a/README.md b/README.md index 1eb3ed6..4eb5f9f 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,49 @@ # js-untar -Library for reading tar files in the browser. +Library for extracting tar files in the browser. + +## Documentation +Load the module with RequireJS or similar. Module is a function that returns a modified Promise with a progress callback. +This callback is executed every time a file is extracted. +The standard Promise.then method is also called when extraction is done, with all extracted files as argument. + +### Example: + + define(["untar"], function(untar) { + // Load the source ArrayBuffer from a XMLHttpRequest or any other way. + var sourceBuffer = ...; + + untar(sourceBuffer) + .progress(function(extractedFile) { + ... + }) + .then(function(extractedFiles) { + ... + }); + }); + +### File object +The returned file object has the following properties. Most of these are explained in the [Tar wikipedia entry](https://en.wikipedia.org/wiki/Tar_(computing)#File_format). + +* name = The full filename (including path and ustar filename prefix). +* mode +* uid +* gid +* size +* modificationTime +* checksum +* type +* linkname +* ustarFormat +* blob A [Blob](https://developer.mozilla.org/en-US/docs/Web/API/Blob) object with the contens of the file. +* getObjectUrl() + A unique [ObjectUrl](https://developer.mozilla.org/en-US/docs/Web/API/URL/createObjectURL) to the data can be retrieved with this method for easy usage of extracted data in tags etc. + document.getElementById("targetImageElement").src = file.getObjectUrl(); + +If the .tar file was in the ustar format (which most are), the following properties are also defined: + +* version +* uname +* gname +* devmajor +* devminor +* namePrefix diff --git a/gulpfile.js b/gulpfile.js index 22c9abd..0b08292 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -15,7 +15,7 @@ gulp.task("build:dev", function() { return gulp.src(["src/untar.js"]) .pipe(sourcemaps.init()) - .pipe(insert.append("\nworkerScriptUri = 'untar-worker.js';")) + .pipe(insert.append("\nworkerScriptUri = '/base/build/dev/untar-worker.js';")) .pipe(addSrc(["src/ProgressivePromise.js", "src/untar-worker.js"])) .pipe(jshint()) .pipe(jshint.reporter("default")) @@ -50,7 +50,7 @@ gulp.task("build:dist", function() { .pipe(insert.prepend('"use strict";\n')) .pipe(uglify()) .pipe(insert.transform(function(contents, file) { - var str = ["\nworkerScriptUri = URL.createObjectURL(createBlob([\""]; + var str = ["\nworkerScriptUri = URL.createObjectURL(new Blob([\""]; str.push(contents.replace(/"/g, '\\"')); str.push("\"]));"); @@ -72,7 +72,7 @@ gulp.task("build:dist", function() { gulp.task("default", ["build:dev", "build:dist"]); -gulp.task("test", ["build:dev"], function(done) { +gulp.task("test", ["build:dev", "build:dist"], function(done) { new KarmaServer({ configFile: __dirname + '/karma.conf.js', singleRun: true diff --git a/karma.conf.js b/karma.conf.js index efd2a53..ad44733 100644 --- a/karma.conf.js +++ b/karma.conf.js @@ -15,8 +15,7 @@ module.exports = function(config) { // list of files / patterns to load in the browser files: [ - 'https://www.promisejs.org/polyfills/promise-6.1.0.js', - {pattern: 'build/dev/**/*.js', included: false}, + {pattern: 'build/**/**/*.js', included: false}, {pattern: 'spec/**/*.*', included: false}, 'test-main.js' ], diff --git a/spec/empty-spec.js b/spec/empty-spec.js deleted file mode 100644 index e69de29..0000000 diff --git a/spec/untar-spec.js b/spec/untar-spec.js index 631300a..573ed70 100644 --- a/spec/untar-spec.js +++ b/spec/untar-spec.js @@ -1,37 +1,70 @@ -define(["untar"], function(untar) { +define(["untar", "../build/dist/untar"], function(untarDev, untarDist) { - describe("untar", function() { + function loadTestBuffer() { + return new Promise(function(resolve, reject) { + var r = new XMLHttpRequest(); - console.log("untar: " + JSON.stringify(untar)); - - var fileNames = [ - "1.txt", - "2.txt", - "3.txt", - "directory/", - "directory/1.txt", - "directory/2.txt", - "directory/3.txt" - ]; - - it("should unpack 3 specific files and a directory with 3 specific files", function(done) { - var i = 0; - - untar("/base/spec/data/test.tar").then( - function(files) { - expect(files.length).toBe(7); - done(); - }, - function(err) { - done.fail(JSON.stringify(err)); - }, - function(file) { - expect(file).toBeDefined(); - expect(file.name).toBe(fileNames[i]); - i += 1; + r.onload = function(e) { + if (r.status >= 200 && r.status < 400) { + var buffer = r.response; + resolve(buffer); + } else { + reject(r.status + " " + r.statusText); } - ); - }, 20000); - }); + } + + r.open("GET", "base/spec/data/test.tar"); + r.responseType = "arraybuffer"; + r.send(); + }); + } + + var fileNames = [ + "1.txt", + "2.txt", + "3.txt", + "directory/", + "directory/1.txt", + "directory/2.txt", + "directory/3.txt" + ]; + + var tests = function(untar) { + + return function() { + it("should unpack 3 specific files and a directory with 3 specific files", function(done) { + expect(typeof untar).toBe("function"); + + var i = 0; + var files = []; + + loadTestBuffer().then(function(buffer) { + untar(buffer).then( + function() { + expect(files.length).toBe(7); + done(); + }, + function(err) { + done.fail(err.message); + }, + function(file) { + expect(file).toBeDefined(); + expect(file.name).toBe(fileNames[i]); + files.push(file); + i += 1; + } + ); + }, done.fail); + }, 20000); + + it("should throw when not called with an ArrayBuffer", function() { + expect(untar).toThrow(); + expect(function() { untar("test"); }).toThrow(); + }); + } + }; + + describe("untarDev", tests(untarDev)); + describe("untarDist", tests(untarDist)); }); diff --git a/spec/untar-worker-spec.js b/spec/untar-worker-spec.js new file mode 100644 index 0000000..5b2011b --- /dev/null +++ b/spec/untar-worker-spec.js @@ -0,0 +1,200 @@ +define(["untar-worker"], function() { + + var untarWorker = new UntarWorker(); + + describe("untar-worker", function() { + var onmessage; + + var fileNames = [ + "1.txt", + "2.txt", + "3.txt", + "directory/", + "directory/1.txt", + "directory/2.txt", + "directory/3.txt" + ]; + + var fileContent = [ + "one", + "two", + "three", + "", + "one", + "two", + "three" + ]; + + function loadTestBuffer() { + return new Promise(function(resolve, reject) { + var r = new XMLHttpRequest(); + + r.onload = function(e) { + if (r.status >= 200 && r.status < 400) { + var buffer = r.response; + resolve(buffer); + } else { + reject(r.status + " " + r.statusText); + } + } + + r.open("GET", "base/spec/data/test.tar"); + r.responseType = "arraybuffer"; + r.send(); + }); + } + + beforeEach(function() { + onmessage = null; + untarWorker.postMessage = function(msg, transfers) { + if (typeof onmessage === "function") { + onmessage(msg, transfers); + } + }; + }); + + describe("UntarStream", function() { + var s; + + beforeEach(function(done) { + var n = new Uint32Array(1); + n[0] = 42; + var blob = new Blob([n.buffer, "String of 18 chars"]); + var fileReader = new FileReader(); + + fileReader.onload = function(e) { + var buf = fileReader.result; + s = new UntarStream(buf); + done(); + }; + + fileReader.readAsArrayBuffer(blob); + }); + + it("should peek at uint32", function() { + expect(s.peekUint32()).toBe(42); + }); + + it("should read a string", function() { + s.seek(4); + expect(s.readString(18)).toBe("String of 18 chars"); + }); + }); + + describe("UntarFileStream", function() { + var buffer; + var fileStream; + + beforeEach(function(done) { + loadTestBuffer().then(function(b) { + buffer = b; + fileStream = new UntarFileStream(b); + }).then(done, done.fail); + }); + + afterEach(function() { + buffer = null; + fileStream = null; + }); + + it("should use hasNext() to indicate more files", function() { + for (var i = 0; i < 7; ++i) { + expect(fileStream.hasNext()).toBe(true); + fileStream.next(); + } + + expect(fileStream.hasNext()).toBe(false); + }); + + it("should extract files in a specific order", function() { + var file; + var i = 0; + + while (fileStream.hasNext()) { + file = fileStream.next(); + expect(file.name).toBe(fileNames[i++]); + + if (i > fileNames.length) fail("i > fileNames.length"); + } + }); + + it("should extract the correct content", function() { + function readString(buffer) { + if (!buffer) { + return ""; + } + + //console.log("readString: position " + this.position() + ", " + charCount + " chars"); + var charCount = buffer.byteLength; + var charSize = 1; + var byteCount = charCount * charSize; + var bufferView = new DataView(buffer); + + var charCodes = []; + + for (var i = 0; i < charCount; ++i) { + var charCode = bufferView.getUint8(i * charSize, true); + charCodes.push(charCode); + } + + return String.fromCharCode.apply(null, charCodes); + } + + + var file; + var i = 0; + + while (fileStream.hasNext()) { + file = fileStream.next(); + + var content = readString(file.buffer); + expect(content).toBe(fileContent[i++]); + + if (i > fileContent.length) fail("i > fileContent.length"); + } + }); + }); + + describe("UntarWorker", function() { + var buffer; + var worker; + + beforeEach(function(done) { + worker = new UntarWorker(); + + loadTestBuffer().then(function(b) { + buffer = b; + }).then(done, done.fail); + }); + + it("receives messages to extract from a buffer", function() { + var filesExtracted = 0; + onmessage = function(msg, transfers) { + var file; + msg = msg.data; + switch (msg.type) { + case "extract": + file = msg.data; + expect(file.name).toBe(fileNames[filesExtracted++]); + expect(transfers[0]).toBe(file.buffer); + break; + case "complete": + expect(filesExtracted).toBe(7); + expect(msg.data.length).toBe(7); + + for (var x = 0; x < msg.data.length; ++x) { + expect(msg.data[x].name).toBe(fileNames[x]); + } + break; + case "error": + fail(msg.data); + break; + } + }; + + untarWorker.onmessage({type: "extract", buffer: buffer}); + }); + }); + }); + +}); diff --git a/src/untar-worker.js b/src/untar-worker.js index db2db5f..ff892d0 100644 --- a/src/untar-worker.js +++ b/src/untar-worker.js @@ -18,11 +18,13 @@ UntarWorker.prototype = { }, postError: function(err) { - this.postMessage({ type: "error", data: err }); + //console.info("postError(" + err.message + ")" + " " + JSON.stringify(err)); + this.postMessage({ type: "error", data: { message: err.message } }); }, postLog: function(level, msg) { - this.postMessage({ type: "log", data: { level: level, msg: msg }}); + console.info("postLog"); + this.postMessage({ type: "log", data: { level: level, msg: msg }}); }, untarBuffer: function(arrayBuffer) { @@ -41,6 +43,7 @@ UntarWorker.prototype = { }, postMessage: function(msg, transfers) { + console.info("postMessage(" + msg + ", " + JSON.stringify(transfers) + ")"); self.postMessage(msg, transfers); } }; @@ -172,6 +175,10 @@ UntarFileStream.prototype = { // We only care about real files, not symlinks. } + if (file.buffer === undefined) { + file.buffer = new ArrayBuffer(0); + } + // File data is padded to reach a 512 byte boundary; skip the padded bytes. var dataEndPos = dataBeginPos + (file.size > 0 ? file.size + (512 - file.size % 512) : 0); stream.position(dataEndPos); diff --git a/src/untar.js b/src/untar.js index 4bbd4b3..d0d9f97 100644 --- a/src/untar.js +++ b/src/untar.js @@ -4,29 +4,14 @@ var workerScriptUri; // Included at compile time var URL = window.URL || window.webkitURL; -var createBlob = (function() { - if (typeof window.Blob === "function") { - return function(dataArray) { return new Blob(dataArray); }; - } else { - var BBuilder = window.BlobBuilder || window.WebKitBlobBuilder; - - return function(dataArray) { - var builder = new BBuilder(); - - for (var i = 0; i < dataArray.length; ++i) { - var v = dataArray[i]; - builder.append(v); - } - - return builder.getBlob(); - }; - } -}()); - /** Returns a ProgressivePromise. */ function untar(arrayBuffer) { + if (!(arrayBuffer instanceof ArrayBuffer)) { + throw new TypeError("arrayBuffer is not an instance of ArrayBuffer."); + } + if (!window.Worker) { throw new Error("Worker implementation not available in this environment."); } @@ -36,6 +21,10 @@ function untar(arrayBuffer) { var files = []; + worker.onerror = function(err) { + reject(err); + }; + worker.onmessage = function(message) { message = message.data; @@ -52,7 +41,8 @@ function untar(arrayBuffer) { resolve(files); break; case "error": - reject(message.data); + //console.log("error message"); + reject(new Error(message.data.message)); break; default: reject(new Error("Unknown message from worker: " + message.type)); @@ -66,7 +56,7 @@ function untar(arrayBuffer) { } function decorateExtractedFile(file) { - file.blob = createBlob([file.buffer]); + file.blob = new Blob([file.buffer]); delete file.buffer; var blobUrl;