From 054bddd577d88559dbe92940cf40ff38e8e62f14 Mon Sep 17 00:00:00 2001 From: nightwing Date: Sun, 29 Nov 2015 06:00:05 +0400 Subject: [PATCH] use caches api for faster reloading in dev mode --- .../build_support/mini_require.js | 171 +++++++++++++++++- plugins/c9.ide.layout.classic/preload.js | 10 +- plugins/c9.static/cdn.js | 128 ++++++++++++- 3 files changed, 290 insertions(+), 19 deletions(-) diff --git a/node_modules/architect-build/build_support/mini_require.js b/node_modules/architect-build/build_support/mini_require.js index 87186077..77864a27 100644 --- a/node_modules/architect-build/build_support/mini_require.js +++ b/node_modules/architect-build/build_support/mini_require.js @@ -36,6 +36,14 @@ var define = function(name, deps, callback) { deps = null; } + if (nextModule) { + if (!name || name == nextModule.name) { + name = nextModule.name; + deps = deps || nextModule.deps; + nextModule = null; + } + } + if (!name) return defQueue.push([deps, callback]); @@ -60,6 +68,7 @@ var define = function(name, deps, callback) { define.lastModule = name; }; var defQueue = []; +var nextModule; var addToLoadQueue = function(missing, deps, callback, errback) { var toLoad = missing.length; var map = {}; @@ -275,12 +284,18 @@ var config = require.config = function(cfg) { config.paths[p] = cfg.paths[p]; }); + if (cfg.useCache && global.caches) { + config.useCache = true; + checkCache(); + } + if (cfg.baseUrlLoadBalancers) config.baseUrlLoadBalancers = cfg.baseUrlLoadBalancers; }; config.packages = Object.create(null); config.paths = Object.create(null); config.baseUrl = ""; +config.useCache = false; require.undef = function(module, recursive) { if (recursive) { @@ -347,7 +362,7 @@ require.toUrl = function(moduleName, ext, skipExt, skipBalancers) { if (!absRe.test(url)) { url = (config.baseUrl || require.MODULE_LOAD_URL + "/") + url; } - if (url[0] === "/" && config.baseUrlLoadBalancers && !skipBalancers) { + if (url[0] === "/" && config.baseUrlLoadBalancers && !skipBalancers && !config.useCache) { var n = Math.abs(hashCode(url)) % config.baseUrlLoadBalancers.length; url = config.baseUrlLoadBalancers[n] + url; } @@ -365,7 +380,7 @@ function hashCode(string) { return result; } -var loadScript = function(path, id, callback) { +var loadScriptWithTag = function(path, id, callback) { // TODO use importScripts for webworkers var head = document.head || document.documentElement; var s = document.createElement("script"); @@ -373,7 +388,7 @@ var loadScript = function(path, id, callback) { s.charset = "utf-8"; s.async = true; - if (path.lastIndexOf(require.MODULE_LOAD_URL, 0) == 0) + if (path.lastIndexOf(require.MODULE_LOAD_URL, 0) == 0 && path[0] != "/") s.crossOrigin = true; head.appendChild(s); @@ -393,6 +408,137 @@ var loadScript = function(path, id, callback) { }; }; +function loadText(path, cb) { + var xhr = new window.XMLHttpRequest(); + xhr.open("GET", path, true); + xhr.setRequestHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8"); + xhr.onload = function(e) { cb(null, xhr.responseText, xhr); }; + xhr.onabort = xhr.onerror = function(e) { cb(e); }; + xhr.send(""); +} + +/*** cache ***/ +var host = location.protocol + "//" + location.hostname + (location.port ? ":" + location.port : ""); +var loadScript = function(path, id, callback) { + if (!config.useCache) + return loadScriptWithTag(path, id, callback); + if (!/https?:/.test(path)) + path = host + path; + var cb = function(e, val, deps) { + if (e) console.error("Couldn't load module " + module, e); + + nextModule = { + name: id, + deps: deps + }; + window.eval(val + "\n//# sourceURL=" + path); + callback(null, id); + return define.loaded[id]; + }; + loadCached(path, cb); +}; + +var loadCached = function(path, cb) { + if (!config.useCache) + return loadText(path, cb); + function loadNew() { + loadText(path, function(e, val, xhr) { + var m = cb(e, val); + if (val) { + var ETAG = xhr.getResponseHeader("ETAG"); + var res = new Response(val); + res.headers.set("ETAG", ETAG); + var req = new Request(path); + req.headers.set("ETAG", ETAG); + if (m && m.deps) + res.headers.set("deps", m.deps.join(",")); + ideCache.put(req, res).catch(function() { + ideCache.delete(path); + }); + } + }); + } + if (ideCachePromiss) { + return ideCachePromiss.then(function(i) { + if (i) ideCache = i; + loadCached(path, cb); + }); + } + ideCache.match(path).then(function(e) { + if (!e) + return loadNew(); + e.text().then(function(val) { + var deps = e.headers.get("deps"); + if (typeof deps == "string") + deps = deps ? deps.split(",") : []; + + cb(null, val, deps); + }); + }); +}; + +var ideCache; +var ideCachePromiss; +function checkCache() { + var baseUrl; + ideCachePromiss = config.useCache && window.caches.open("ide").then(function(ideCache_) { + ideCache = ideCache_; + return ideCache.keys(); + }).then(function(keys) { + baseUrl = host + config.baseUrl; + var val = keys.map(function(r) { + var url = r.url; + if (url.startsWith(baseUrl)) + url = url.slice(baseUrl.length); + return r.headers.get("etag") + " " + url; + }).join("\n") + "\n"; + if (val.length == 1) { + ideCachePromiss = null; + return ideCache; + } + return new Promise(function(resolve) { + var checked = 0; + var buffer = ""; + var toDelete = [] + post("/static/__check__", val, function(t) { + var e = t.slice(checked); + checked = t.length; + var parts = (buffer + e).split("\n"); + buffer = parts.pop(); + for (var i = 0; i < parts.length; i++) { + if (parts[i]) { + var del = ideCache.delete(baseUrl + parts[i]); + toDelete.push(del); + console.log(parts[i]); + } + } + }, function(e, t) { + ideCachePromiss = null; + Promise.all(toDelete).then(function() { + resolve(ideCache); + }); + setTimeout(function() { + setTimeout(function() { + // TODO for now we do not support checking second time so we unset useCache after a while + config.useCache = false; + }, 5000); + }, 5000); + }); + }); + }); + return ideCachePromiss; +} + +function post(path, val, progress, cb) { + var xhr = new window.XMLHttpRequest(); + xhr.open("POST", path, true); + xhr.setRequestHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8"); + xhr.onload = function(e) { cb(null, xhr.responseText, xhr); }; + xhr.onreadystatechange = function(e) { progress(xhr.responseText, xhr); }; + xhr.onabort = xhr.onerror = function(e) { cb(e); }; + xhr.send(val); +} + require.load = function(module) { var i = module.indexOf("!") + 1; if (i) { @@ -447,12 +593,19 @@ require["text!"] = function(module, callback) { define("text!" + module, [], val); callback(); }; - var xhr = new window.XMLHttpRequest(); - xhr.open("GET", url + "?access_token=fake_token", true); - xhr.setRequestHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8"); - xhr.onload = function(e) { cb(null, xhr.responseText); }; - xhr.onabort = xhr.onerror = function(e) { cb(e); }; - xhr.send(""); + loadCached(url, cb); +}; +require["ace/requirejs/text!"] = function(module, callback) { + var url = require.toUrl(module); + if (define.fetchedUrls[url] & 2) + return false; + define.fetchedUrls[url] |= 2; + var cb = function(e, val) { + if (e) console.error("Couldn't load module " + module, e); + define("ace/requirejs/text!" + module, [], val); + callback(); + }; + loadCached(url, cb); }; /*** add global define ***/ diff --git a/plugins/c9.ide.layout.classic/preload.js b/plugins/c9.ide.layout.classic/preload.js index 1e132db0..665ebe3e 100644 --- a/plugins/c9.ide.layout.classic/preload.js +++ b/plugins/c9.ide.layout.classic/preload.js @@ -51,15 +51,13 @@ define(function(require, exports, module) { }); } else { var url = themePrefix + "/" + name + ".css"; - http.request(url, { - timeout: 2 * 60 * 1000 - }, function(err, data) { - if (err) - return callback(err, data); + require(["text!" + url], function(data) { // set sourceurl so that sourcemaps work when theme is inserted as a style tag data += "\n/*# sourceURL=" + url + " */"; themes[name] = data; - callback(err, data); + callback(null, data); + }, function(err) { + callback(err); }); } } diff --git a/plugins/c9.static/cdn.js b/plugins/c9.static/cdn.js index 1e141310..5de217a9 100644 --- a/plugins/c9.static/cdn.js +++ b/plugins/c9.static/cdn.js @@ -3,6 +3,7 @@ main.consumes = [ "connect", "connect.cors", + "connect.static", "cdn.build" ]; main.provides = []; @@ -12,6 +13,7 @@ module.exports = main; function main(options, imports, register) { var connect = imports.connect; var build = imports["cdn.build"]; + var connectStatic = imports["connect.static"]; var fs = require("fs"); var path = require("path"); @@ -28,7 +30,121 @@ function main(options, imports, register) { var section = api.section("static"); - //section.use(foreverCache()); + var resolveModulePath = require("architect-build/module-deps").resolveModulePath; + connectStatic.getRequireJsConfig().useCache = options.useBrowserCache; + section.post("/__check__", [function(req, res, next) { + req.params.hash = "any"; + next(); + }, prepare, function(req, res, next) { + function FsQ() { + this.buffer = []; + this.process = function() {}; + this.active = 0; + this.ended = false; + this.maxActive = 100; + } + FsQ.prototype.write = function(arr) { + this.buffer.push.apply(this.buffer, arr); + this.take(); + }; + FsQ.prototype.take = function(arr) { + while (this.buffer.length && this.active < this.maxActive) { + this.process(this.buffer.pop()); + this.active++; + } + }; + FsQ.prototype.oneDone = function() { + this.active--; + if (!this.active) + this.take(); + if (!this.active && this.ended) + this.end(); + }; + + res.writeHead(200, { + "Content-Type": "application/javascript", + "Cache-Control": "no-cache, no-store" + }); + req.setEncoding("utf8"); + + var buffer = ""; + var t = Date.now(); + var q = new FsQ(); + var skinChanged = false; + var requestedSkin = ""; + var requestedSkinChanged = false; + q.process = function(e) { + var parts = e.split(" "); + var id = parts[1]; + var etag = parts[0]; + var path = resolveModulePath(id, req.pathConfig.pathMap); + + if (path == id && !/^(\/|\w:)/.test(path)) { + path = build.cacheDir + "/" + path; + if (/^\w+\/skin\//.test(id)) + requestedSkin = id; + } + + fs.stat(path, function(err, s) { + if (!err) { + var mt = s.mtime.valueOf(); + var etagNew = '"' + s.size +"-" + mt + '"'; + if (etag !== etagNew) { + err = true; + } + } + + if (err) { + if (!skinChanged && /\.(css|less)/.test(id)) + skinChanged = true; + if (requestedSkin == id) + requestedSkinChanged = true; + res.write(id + "\n"); + } + q.oneDone(); + }); + }; + q.end = function() { + if (!q.buffer.length && !q.active) { + console.log(skinChanged, requestedSkinChanged, requestedSkin) + if (skinChanged && !requestedSkinChanged && requestedSkin) { + res.write(requestedSkin + "\n"); + console.info("Deleting old skin", requestedSkin); + return fs.unlink(build.cacheDir + "/" + requestedSkin, function() { + skinChanged = false; + q.end(); + }); + } + res.write("\n"); + res.end(); + console.info("Checking cache state took:", t - Date.now(), "ms"); + } + else { + q.ended = true; + } + }; + function onData(e) { + var parts = (buffer + e).split("\n"); + buffer = parts.pop(); + q.write(parts); + // console.log(i++); + } + function onEnd(e) { + console.log("end", t - Date.now()); + q.end(); + } + + if (req.body) { + // TODO disable automatic buffering in connect + onData(Object.keys(req.body)[0]); + onEnd(); + } else { + req.on("end", onEnd); + req.on("data", onData); + } + }]); + + // section.use(foreverCache()); section.use(imports["connect.cors"].cors("*")); section.use(connect.getModule().compress()); @@ -103,6 +219,8 @@ function main(options, imports, register) { type = "text/css"; res.setHeader("Content-Type", type); + var mtime = Date.now(); + res.setHeader("ETAG", '"' + Buffer.byteLength(code) + "-" + mtime + '"'); res.statusCode = 200; res.end(code); @@ -114,9 +232,11 @@ function main(options, imports, register) { atomic.writeFile(filename, code, "utf8", function(err) { if (err) - console.error("Caching file", filename, "failed", err); - else - console.log("File cached at", filename); + return console.error("Caching file", filename, "failed", err); + + console.log("File cached at", filename); + // set utime to have consistent etag + fs.utimes(filename, mtime, mtime, function() {}); }); }); });