miklo
/
TiddlyWiki5
kopia lustrzana https://github.com/miklobit/TiddlyWiki5
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Refactor sitemap handling so we can reuse it in the webserver

publishing-framework
jeremy@jermolene.com 2021-04-07 12:16:12 +01:00
rodzic a8c248eb3d
commit df2a3fdefd
11 zmienionych plików z 318 dodań i 176 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

1
core/language/en-GB/Help/listen.tid
Wyświetl plik

@ -23,6 +23,7 @@ All parameters are optional with safe defaults, and can be specified in any orde
* ''writers'' - comma separated list of principals allowed to write to this wiki
* ''csrf-disable'' - set to "yes" to disable CSRF checks (defaults to "no")
* ''sse-enabled'' - set to "yes" to enable Server-sent events (defaults to "no")
* ''sitemap'' - optional sitemap describing how the tiddlers will be served. See [[Publishing]] for more details
* ''root-tiddler'' - the tiddler to serve at the root (defaults to "$:/core/save/all")
* ''root-render-type'' - the content type to which the root tiddler should be rendered (defaults to "text/plain")
* ''root-serve-type'' - the content type with which the root tiddler should be served (defaults to "text/html")

189
core/modules/publisher-handler.js
Wyświetl plik

@ -61,49 +61,41 @@ PublishingJob.prototype.publish = function(callback) {
// Get the list of tiddlers to be exported, defaulting to all non-system tiddlers
this.exportList = this.publisherHandler.wiki.filterTiddlers(this.jobTiddler.fields["export-filter"] || "[!is[system]]");
// Get the job variables
this.jobVariables = this.extractVariables(this.jobTiddler);
this.jobVariables = this.jobTiddler.getFieldStrings({prefix: "var-"});
// Get publisher
this.publisher = this.getPublisher(this.jobTiddler.fields.publisher);
if(this.publisher) {
// Get the sitemap
this.sitemap = this.publisherHandler.wiki.getTiddler(this.jobTiddler.fields.sitemap);
if(this.sitemap) {
// Get the sitemap variables
this.sitemapVariables = this.extractVariables(this.sitemap);
// Collect the operations from each route
this.operations = [];
$tw.utils.each(this.sitemap.fields.list,function(routeTitle) {
var routeTiddler = self.publisherHandler.wiki.getTiddler(routeTitle);
if(routeTiddler) {
Array.prototype.push.apply(self.operations,self.getOperationsForRoute(routeTiddler));
}
});
// Display the progress modal
if($tw.modal) {
self.progressModal = $tw.modal.display(PUBLISHING_MODAL_TITLE,{
progress: true,
variables: {
currentTiddler: this.jobTitle,
totalFiles: this.operations.length + ""
},
onclose: function(event) {
if(event !== self) {
// The modal was closed other than by us programmatically
self.isCancelled = true;
}
this.sitemap = new $tw.Sitemap({
wiki: this.publisherHandler.wiki,
variables: this.publishVariables
});
this.sitemap.load(this.jobTiddler.fields.sitemap);
// Get the output operations
this.operations = this.sitemap.getAllFileDetails(this.exportList);
// Display the progress modal
if($tw.modal) {
this.progressModal = $tw.modal.display(PUBLISHING_MODAL_TITLE,{
progress: true,
variables: {
currentTiddler: this.jobTitle,
totalFiles: this.operations.length + ""
},
onclose: function(event) {
if(event !== self) {
// The modal was closed other than by us programmatically
self.isCancelled = true;
}
});
}
// Send the operations to the publisher
this.executeOperations(function(err) {
if(self.progressModal) {
self.progressModal.closeHandler(self);
}
callback(err);
});
} else {
return callback("Missing sitemap");
}
// Send the operations to the publisher
this.executeOperations(function(err) {
if(self.progressModal) {
self.progressModal.closeHandler(self);
}
callback(err);
});
} else {
return callback("Unrecognised publisher");
}
@ -125,104 +117,6 @@ PublishingJob.prototype.getPublisher = function(publisherName) {
return publisher && publisher.create(this.jobTiddler.fields,this.publisherHandler,this);
};
/*
Extract the variables from tiddler fields prefixed "var-"
*/
PublishingJob.prototype.extractVariables = function(tiddler) {
var variables = {};
$tw.utils.each(tiddler.getFieldStrings(),function(value,name) {
if(name.substring(0,4) === "var-") {
variables[name.substring(4)] = value;
}
});
return variables;
};
/*
Expand publish routes to separate operations
*/
PublishingJob.prototype.getOperationsForRoute = function(routeTiddler) {
var self = this,
operations = [],
routeFilter = routeTiddler.fields["route-tiddler-filter"] || "DUMMY_RESULT", // If no filter is provided, use a dummy filter that returns a single result
tiddlers = self.publisherHandler.wiki.filterTiddlers(routeFilter,null,self.publisherHandler.wiki.makeTiddlerIterator(this.exportList));
if(routeFilter) {
switch(routeTiddler.fields["route-type"]) {
case "save":
if(routeTiddler.fields["route-path"]) {
$tw.utils.each(tiddlers,function(title) {
operations.push({
"route-type": "save",
path: self.resolveParameterisedPath(routeTiddler.fields["route-path"],title),
title: title
});
});
}
break;
case "render":
if(routeTiddler.fields["route-path"] && routeTiddler.fields["route-template"]) {
var routeVariables = $tw.utils.extend({},this.publishVariables,this.jobVariables,this.sitemapVariables,this.extractVariables(routeTiddler));
$tw.utils.each(tiddlers,function(title) {
operations.push({
"route-type": "render",
path: self.resolveParameterisedPath(routeTiddler.fields["route-path"],title),
title: title,
template: routeTiddler.fields["route-template"],
variables: routeVariables
});
});
}
break;
}
}
return operations;
};
/*
Apply a tiddler to a parameterised path to create a usable path
*/
PublishingJob.prototype.resolveParameterisedPath = function(route,title) {
var self = this;
// Split the route on $$ markers
var tiddler = this.publisherHandler.wiki.getTiddler(title),
output = [];
$tw.utils.each(route.split(/(\$[a-z_]+\$)/),function(part) {
var match = part.match(/\$([a-z]+)_([a-z]+)\$/);
if(match) {
var value;
// Get the base value
switch(match[1]) {
case "uri":
case "title":
value = title;
break;
case "type":
value = tiddler.fields.type || "text/vnd.tiddlywiki";
break;
}
// Apply the encoding function
switch(match[2]) {
case "encoded":
value = encodeURIComponent(value);
break;
case "doubleencoded":
value = encodeURIComponent(encodeURIComponent(value));
break;
case "slugify":
value = self.publisherHandler.wiki.slugify(value);
break;
case "extension":
value = ($tw.config.contentTypeInfo[value] || {extension: "."}).extension.slice(1);
break;
}
output.push(value);
} else {
output.push(part);
}
});
return output.join("");
};
/*
Execute the operations for this job
*/
@ -249,7 +143,7 @@ PublishingJob.prototype.executeOperations = function(callback) {
});
} else {
// Execute this operation
var fileDetails = self.prepareOperation(self.operations[nextOperation]);
var fileDetails = self.operations[nextOperation]();
nextOperation += 1;
self.publisher.publishFile(fileDetails,function() {
$tw.utils.nextTick(performNextOperation);
@ -270,33 +164,6 @@ PublishingJob.prototype.executeOperations = function(callback) {
});
};
/*
Construct a file details object from an operation object
*/
PublishingJob.prototype.prepareOperation = function(operation) {
var tiddler = this.publisherHandler.wiki.getTiddler(operation.title),
fileDetails = {
path: operation.path
};
switch(operation["route-type"]) {
case "save":
fileDetails.text = tiddler.fields.text || "";
fileDetails.type = tiddler.fields.type || "";
fileDetails.isBase64 = ($tw.config.contentTypeInfo[tiddler.fields.type] || {}).encoding === "base64";
break;
case "render":
fileDetails.text = this.publisherHandler.wiki.renderTiddler("text/plain",operation.template,{
variables: $tw.utils.extend(
{currentTiddler: operation.title},
operation.variables
)
});
fileDetails.type = "text/html";
break;
}
return fileDetails;
};
PublishingJob.prototype.saveReport = function(report) {
// Create the report tiddler
var reportTitle = this.wiki.generateNewTitle("$:/temp/publish-report");

25
core/modules/server/server.js
Wyświetl plik

@ -58,11 +58,19 @@ function Server(options) {
// console.log("Loading server route " + title);
self.addAuthenticator(authenticatorDefinition.AuthenticatorClass);
});
// Load route handlers
$tw.modules.forEachModuleOfType("route", function(title,routeDefinition) {
// console.log("Loading server route " + title);
self.addRoute(routeDefinition);
});
// Load route handlers from sitemap if present, or just load all route modules
if(this.variables.sitemap) {
this.sitemap = new $tw.Sitemap({
wiki: this.wiki,
variables: {}
});
this.sitemap.load(this.variables.sitemap);
this.addRoutes(this.sitemap.getServerRoutes());
} else {
$tw.modules.forEachModuleOfType("route", function(title,routeDefinition) {
self.addRoute(routeDefinition);
});
}
// Initialise the http vs https
this.listenOptions = null;
this.protocol = "http";
@ -96,6 +104,13 @@ Server.prototype.get = function(name) {
return this.variables[name];
};
Server.prototype.addRoutes = function(routes) {
var self = this;
$tw.utils.each(routes,function(route) {
self.addRoute(route);
});
};
Server.prototype.addRoute = function(route) {
this.routes.push(route);
};

219
core/modules/sitemap.js 100644
Wyświetl plik

@ -0,0 +1,219 @@
/*\
title: $:/core/modules/sitemap.js
type: application/javascript
module-type: global
Sitemaps are used for static publishing and web serving
\*/
(function(){
/*jslint node: true, browser: true */
/*global $tw: false */
"use strict";
function Sitemap(options) {
options = options || {};
this.wiki = options.wiki;
this.routes = [];
this.variables = $tw.utils.extend({},options.variables);
}
Sitemap.prototype.load = function(sitemapTitle) {
var self = this;
// Get the sitemap
var sitemapTiddler = this.wiki.getTiddler(sitemapTitle);
if(sitemapTiddler) {
// Get the sitemap variables
$tw.utils.extend(this.variables,sitemapTiddler.getFieldStrings({prefix: "var-"}));
// Collect each route
$tw.utils.each(sitemapTiddler.fields.list,function(routeTitle) {
var routeTiddler = self.wiki.getTiddler(routeTitle);
if(routeTiddler) {
// Convert the path into a regexp and an array of {field:,function:} for each capture group
var regexpurgatedParameterisedPath = self.regexpurgateParameterisedPath(routeTiddler.fields["route-path"]);
self.routes.push({
params: routeTiddler.getFieldStrings({prefix: "route-"}),
variables: $tw.utils.extend({},self.variables,routeTiddler.getFieldStrings({prefix: "var-"})),
regexp: regexpurgatedParameterisedPath.regexp,
captureGroups: regexpurgatedParameterisedPath.captureGroups
});
}
});
}
console.log("routes",self.routes)
};
/*
Returns an array of functions that return {path:,text:,type:,isBase64:} for each path
*/
Sitemap.prototype.getAllFileDetails = function(exportTiddlers) {
var self = this,
output = [];
$tw.utils.each(this.routes,function(route) {
var routeFilter = route.params["tiddler-filter"] || "DUMMY_RESULT", // If no filter is provided, use a dummy filter that returns a single result
routeTiddlers = self.wiki.filterTiddlers(routeFilter,null,self.wiki.makeTiddlerIterator(exportTiddlers));
switch(route.params.type) {
case "raw":
$tw.utils.each(routeTiddlers,function(title) {
output.push(function() {
var tiddler = self.wiki.getTiddler(title);
return {
path: self.resolveParameterisedPath(route.params.path,title),
text: tiddler.fields.text || "",
type: tiddler.fields.type || "",
isBase64: ($tw.config.contentTypeInfo[tiddler.fields.type] || {}).encoding === "base64"
};
});
});
break;
case "render":
$tw.utils.each(routeTiddlers,function(title) {
output.push(function() {
var tiddler = self.wiki.getTiddler(title),
text = self.wiki.renderTiddler("text/plain",route.params.template,{
variables: $tw.utils.extend(
{currentTiddler: title},
$tw.utils.extend({},self.variables,route.variables)
)
});
return {
path: self.resolveParameterisedPath(route.params.path,title),
text: text,
type: route.params["output-type"] || "text/html",
isBase64: tiddler && (($tw.config.contentTypeInfo[tiddler.fields.type] || {}).encoding === "base64")
};
});
});
break;
}
});
return output;
};
/*
Returns an array of server routes {method:, path:, handler:}
*/
Sitemap.prototype.getServerRoutes = function() {
var self = this,
output = [];
$tw.utils.each(this.routes,function(route) {
output.push({
method: "GET",
path: route.regexp,
handler: function(request,response,state) {
// Locate the tiddler identified by the capture groups, if any
var title = null,
nextParam = 0;
$tw.utils.each(route.captureGroups,function(captureGroup) {
var param = state.params[nextParam++];
if(captureGroup.field === "title") {
switch(captureGroup.function) {
case "slugify":
var titles = self.wiki.unslugify(param);
if(titles && titles.length > 0) {
title = titles[0];
}
break;
}
}
})
// Return the rendering or raw tiddler
switch(route.params.type) {
case "render":
response.writeHead(200,{"Content-Type": route.params["output-type"] || "text/html"});
response.end(self.wiki.renderTiddler("text/plain",route.params.template,{
variables: {currentTiddler: title}
}));
break;
case "raw":
var tiddler = title && self.wiki.getTiddler(title);
if(tiddler) {
response.writeHead(200, {"Content-Type": tiddler.fields.type || "text/vnd.tiddlywiki"});
response.end(self.wiki.getTiddlerText(title),($tw.config.contentTypeInfo[tiddler.fields.type] || {encoding: "utf8"}).encoding);
} else {
response.writeHead(404);
response.end();
}
break;
}
}
});
});
return output;
};
/*
Apply a tiddler to a parameterised path to create a usable path
*/
Sitemap.prototype.resolveParameterisedPath = function(parameterisedPath,title) {
var self = this;
// Split the path on $*_*$ markers
var tiddler = this.wiki.getTiddler(title),
output = [];
$tw.utils.each(parameterisedPath.split(/(\$[a-z_]+\$)/),function(part) {
var match = part.match(/\$([a-z]+)_([a-z]+)\$/);
if(match) {
var value;
// Get the base value
switch(match[1]) {
case "uri":
case "title":
value = title;
break;
case "type":
value = tiddler.fields.type || "text/vnd.tiddlywiki";
break;
}
// Apply the encoding function
switch(match[2]) {
case "encoded":
value = encodeURIComponent(value);
break;
case "doubleencoded":
value = encodeURIComponent(encodeURIComponent(value));
break;
case "slugify":
value = self.wiki.slugify(value);
break;
case "extension":
value = ($tw.config.contentTypeInfo[value] || {extension: "."}).extension.slice(1);
break;
}
output.push(value);
} else {
output.push(part);
}
});
return output.join("");
};
/*
// Convert the path into a regexp and an array of {field:,function:} for each capture group
*/
Sitemap.prototype.regexpurgateParameterisedPath = function(parameterisedPath) {
var regexpParts = ["\\/"],
captureGroups = [];
$tw.utils.each(parameterisedPath.split(/(\$[a-z_]+\$)/),function(part) {
var match = part.match(/\$([a-z]+)_([a-z]+)\$/);
if(match) {
regexpParts.push("(.+)");
captureGroups.push({
field: match[1],
function: match[2]
});
} else {
regexpParts.push($tw.utils.escapeRegExp(part));
}
});
return {
regexp: new RegExp("^" + regexpParts.join("") + "$"),
captureGroups: captureGroups
};
};
exports.Sitemap = Sitemap;
})();

24
core/modules/tiddler.js
Wyświetl plik

@ -54,15 +54,27 @@ exports.getFieldList = function(field) {
/*
Get all the fields as a hashmap of strings. Options:
exclude: an array of field names to exclude
prefix: an optional field name prefix. Only fields with the prefix are included, and the prefix is stripped from the name
*/
exports.getFieldStrings = function(options) {
options = options || {};
var exclude = options.exclude || [];
var fields = {};
for(var field in this.fields) {
if($tw.utils.hop(this.fields,field)) {
if(exclude.indexOf(field) === -1) {
fields[field] = this.getFieldString(field);
var exclude = options.exclude || [],
fields = {},
field;
if(options.prefix) {
for(field in this.fields) {
if($tw.utils.hop(this.fields,field)) {
if(exclude.indexOf(field) === -1 && field.substring(0,options.prefix.length) === options.prefix) {
fields[field.substring(options.prefix.length)] = this.getFieldString(field);
}
}
}
} else {
for(field in this.fields) {
if($tw.utils.hop(this.fields,field)) {
if(exclude.indexOf(field) === -1) {
fields[field] = this.getFieldString(field);
}
}
}
}

19
core/modules/wiki.js
Wyświetl plik

@ -1538,5 +1538,24 @@ exports.slugify = function(title,options) {
return slug;
};
/*
Return an array of the titles that would generate a specified slug, if any. Options include:
*/
exports.unslugify = function(slug) {
var self = this,
slugToTitle = this.getGlobalCache("slugs",function() {
var map = {};
$tw.utils.each($tw.wiki.allTitles(),function(title) {
var slug = self.slugify(title);
if(!(slug in map)) {
map[slug] = [];
}
map[slug].push(title);
});
return map;
});
return slugToTitle[slug];
};
})();

2
core/ui/ControlPanel/Publishing.tid
Wyświetl plik

@ -54,7 +54,7 @@ Logs: <$view tiddler=<<job>> field="list"/>
<h2>Route: <$link to=<<route>>><$view tiddler=<<route>> field="caption"><$text text=<<route>>/></$view></$link></h2>
job-type: <$view tiddler=<<route>> field="route-type"/>
route type: <$view tiddler=<<route>> field="route-type"/>
path: <$edit-text tiddler=<<route>> size="50" field="route-path"/>

2
core/wiki/routes/StaticSite/Images.tid
Wyświetl plik

@ -1,6 +1,6 @@
title: $:/core/routes/StaticSite/Images
caption: Images
tags: $:/tags/Route
route-type: save
route-type: raw
route-path: images/$title_slugify$.$type_extension$
route-tiddler-filter: [is[image]]

1
core/wiki/routes/StaticSite/Styles.tid
Wyświetl plik

@ -4,3 +4,4 @@ tags: $:/tags/Route
route-type: render
route-path: static/static.css
route-template: $:/core/templates/static.template.css
route-output-type: text/css

4
editions/tw5.com/tiddlers/publishing/Publishing.tid
Wyświetl plik

@ -52,8 +52,8 @@ A ''route'' describes how a group of one or more files is to be created during t
* ''route-tiddler-filter'' -- a filter defining the tiddlers included in the route
* ''route-path'' - a parameterised path defining how the ''output path'' is derived from the field values of a particular tiddler
* ''route-template'' -- optional title of a tiddler used as a template for "render" route types
* ''route-type'' which can be set to "save" or "render":
** ''"save"'' indicates that the raw tiddler is to be saved, without any rendering
* ''route-type'' which can be set to "raw" or "render":
** ''"raw"'' indicates that the raw tiddler is to be saved, without any rendering
** ''"render"'' indicates that the tiddler is to be rendered through a specified template
* ''var-&lt;variable-name&gt;'' -- custom variables to be provided to the output template

8
editions/tw5.com/tiddlers/webserver/WebServer Parameter_ sitemap.tid 100644
Wyświetl plik

@ -0,0 +1,8 @@
caption: sitemap
created: 20210407121456100
modified: 20210407121456100
tags: [[WebServer Parameters]]
title: WebServer Parameter: sitemap
type: text/vnd.tiddlywiki
The optional [[web server configuration parameter|WebServer Parameters]] ''sitemap'' specifies the sitemap that determines how tiddlers are mapped to URLs. See [[Publishing]] for more details.