mirror
/
libresilient
kopia lustrzana https://gitlab.com/rysiekpl/libresilient
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

MIME type sniffing

merge-requests/23/head
Michał "rysiek" Woźniak 2023-11-10 21:08:03 +00:00
rodzic e6f03e7dbd
commit ac759fb3e2
16 zmienionych plików z 15138 dodań i 101 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

5
__tests__/config.json
Wyświetl plik

@ -1,6 +1,7 @@
{
"plugins": [{
"name": "dnslink-ipfs"
"name": "test-plugin"
}],
"loggedComponents": ["service-worker", "dnslink-ipfs"]
"useMimeSniffingLibrary": true,
"loggedComponents": ["service-worker", "test-plugin"]
}

405
__tests__/service-worker/service-worker.test.js
Wyświetl plik

@ -132,10 +132,17 @@ beforeAll(async ()=>{
}
/*
* mocking importScripts
* mocking importScriptsPrototype
*/
window.importScripts = (script) => {
let plugin = script.split('/')[2]
window.importScriptsPrototype = (script) => {
let plugin = null
try {
plugin = script.split('/')[2]
} catch (e) {}
if (plugin === null) {
// ignoring errors here — these happen when we're not actually loading a plugin
return false
}
window
.LibResilientPluginConstructors
.set(
@ -144,6 +151,10 @@ beforeAll(async ()=>{
)
}
window.LRLogPrototype = (component, ...items)=>{
console.debug(component + ' :: ', ...items)
}
/*
* mocking window.clients
*/
@ -288,6 +299,10 @@ beforeEach(async ()=>{
self.LibResilientPlugins = null
// postMessage spy
window.clients.prototypePostMessage = spy((msg)=>{console.log('*** got message', msg)})
// importScripts spy
window.importScripts = spy(window.importScriptsPrototype)
// LR.log spy
window.LR.log = spy(window.LRLogPrototype)
})
/**
@ -314,12 +329,11 @@ describe('service-worker', async () => {
window.LibResilientPluginConstructors = new Map()
window.LR = {
log: (component, ...items)=>{
console.debug(component + ' :: ', ...items)
}
log: null
}
window.fetch = null
window.importScripts = null
window.test_id = 0
@ -350,6 +364,7 @@ describe('service-worker', async () => {
assertEquals(self.LibResilientConfig.plugins, [{name: "fetch"},{name: "cache"}])
assertEquals(self.LibResilientConfig.loggedComponents, ['service-worker', 'fetch', 'cache'])
assertEquals(self.LibResilientConfig.normalizeQueryParams, true)
assertEquals(self.LibResilientConfig.useMimeSniffingLibrary, false)
assertSpyCalls(self.fetch, 1)
})
@ -369,6 +384,7 @@ describe('service-worker', async () => {
assertEquals(self.LibResilientConfig.plugins, [{name: "fetch"},{name: "cache"}])
assertEquals(self.LibResilientConfig.loggedComponents, ['service-worker', 'fetch', 'cache'])
assertEquals(self.LibResilientConfig.normalizeQueryParams, true)
assertEquals(self.LibResilientConfig.useMimeSniffingLibrary, false)
assertSpyCalls(self.fetch, 1)
})
@ -388,6 +404,7 @@ describe('service-worker', async () => {
assertEquals(self.LibResilientConfig.plugins, [{name: "fetch"},{name: "cache"}])
assertEquals(self.LibResilientConfig.loggedComponents, ['service-worker', 'fetch', 'cache'])
assertEquals(self.LibResilientConfig.normalizeQueryParams, true)
assertEquals(self.LibResilientConfig.useMimeSniffingLibrary, false)
assertSpyCalls(self.fetch, 1)
})
@ -407,6 +424,7 @@ describe('service-worker', async () => {
assertEquals(self.LibResilientConfig.plugins, [{name: "fetch"},{name: "cache"}])
assertEquals(self.LibResilientConfig.loggedComponents, ['service-worker', 'fetch', 'cache'])
assertEquals(self.LibResilientConfig.normalizeQueryParams, true)
assertEquals(self.LibResilientConfig.useMimeSniffingLibrary, false)
assertSpyCalls(self.fetch, 1)
})
@ -426,6 +444,7 @@ describe('service-worker', async () => {
assertEquals(self.LibResilientConfig.plugins, [{name: "fetch"},{name: "cache"}])
assertEquals(self.LibResilientConfig.loggedComponents, ['service-worker', 'fetch', 'cache'])
assertEquals(self.LibResilientConfig.normalizeQueryParams, true)
assertEquals(self.LibResilientConfig.useMimeSniffingLibrary, false)
assertSpyCalls(self.fetch, 1)
})
@ -445,12 +464,33 @@ describe('service-worker', async () => {
assertEquals(self.LibResilientConfig.plugins, [{name: "fetch"},{name: "cache"}])
assertEquals(self.LibResilientConfig.loggedComponents, ['service-worker', 'fetch', 'cache'])
assertEquals(self.LibResilientConfig.normalizeQueryParams, true)
assertEquals(self.LibResilientConfig.useMimeSniffingLibrary, false)
assertSpyCalls(self.fetch, 1)
})
it("should use default LibResilientConfig values when 'useMimeSniffingLibrary' field in config.json contains an invalid value", async () => {
let mock_response_data = {
data: JSON.stringify({loggedComponents: ['service-worker', 'fetch'], plugins: [{name: "fetch"}], defaultPluginTimeout: 5000, normalizeQueryParams: false, useMimeSniffingLibrary: "not a boolean"})
}
window.fetch = spy(window.getMockedFetch(mock_response_data))
await import("../../service-worker.js?" + window.test_id);
await self.dispatchEvent(new Event('install'))
await self.waitForSWInstall()
assertEquals(typeof self.LibResilientConfig, "object")
assertEquals(self.LibResilientConfig.defaultPluginTimeout, 10000)
assertEquals(self.LibResilientConfig.plugins, [{name: "fetch"},{name: "cache"}])
assertEquals(self.LibResilientConfig.loggedComponents, ['service-worker', 'fetch', 'cache'])
assertEquals(self.LibResilientConfig.normalizeQueryParams, true)
assertEquals(self.LibResilientConfig.useMimeSniffingLibrary, false)
assertSpyCalls(self.fetch, 1)
})
it("should use config values from a valid fetched config.json file, caching it", async () => {
let mock_response_data = {
data: JSON.stringify({loggedComponents: ['service-worker', 'cache'], plugins: [{name: "cache"}], defaultPluginTimeout: 5000, normalizeQueryParams: false})
data: JSON.stringify({loggedComponents: ['service-worker', 'cache'], plugins: [{name: "cache"}], defaultPluginTimeout: 5000, normalizeQueryParams: false, useMimeSniffingLibrary: true})
}
window.fetch = spy(window.getMockedFetch(mock_response_data))
@ -463,6 +503,7 @@ describe('service-worker', async () => {
assertEquals(self.LibResilientConfig.plugins, [{name: "cache"}])
assertEquals(self.LibResilientConfig.loggedComponents, ['service-worker', 'cache'])
assertEquals(self.LibResilientConfig.normalizeQueryParams, false)
assertEquals(self.LibResilientConfig.useMimeSniffingLibrary, true)
assertSpyCalls(self.fetch, 1)
// cacheConfigJSON() is called asynchronously in the Service Worker,
@ -1729,4 +1770,354 @@ describe('service-worker', async () => {
}]
})
})
it("guessMimeType() should correctly guess content type based on extension by default", async () => {
// set things up
await import("../../service-worker.js?" + window.test_id);
self.pluginName = "service-worker-test"
// extensions we support, with associated MIME types
let ext_to_mime = new Map([
['htm', 'text/html'],
['html', 'text/html'],
['css', 'text/css'],
['js', 'text/javascript'],
['json', 'application/json'],
['svg', 'image/svg+xml'],
['ico', 'image/x-icon'],
['gif', 'image/gif'],
['png', 'image/png'],
['jpg', 'image/jpeg'],
['jpeg', 'image/jpeg'],
['jpe', 'image/jpeg'],
['jfif', 'image/jpeg'],
['pjpeg', 'image/jpeg'],
['pjp', 'image/jpeg'],
['webp', 'image/webp'],
['avi', 'video/avi'],
['mp4', 'video/mp4'],
['mp2', 'video/mpeg'],
['mp3', 'audio/mpeg'],
['mpa', 'video/mpeg'],
['pdf', 'application/pdf'],
['txt', 'text/plain'],
['ics', 'text/calendar'],
['jsonld', 'application/ld+json'],
['mjs', 'text/javascript'],
['oga', 'audio/ogg'],
['ogv', 'video/ogg'],
['ogx', 'application/ogg'],
['opus', 'audio/opus'],
['otf', 'font/otf'],
['ts', 'video/mp2t'],
['ttf', 'font/ttf'],
['weba', 'audio/webm'],
['webm', 'video/webm'],
['webp', 'image/webp'],
['woff', 'font/woff'],
['woff2', 'font/woff2'],
['xhtml', 'application/xhtml+xml'],
['xml', 'application/xml']
])
// check'em all
for (let [ext, mime] of ext_to_mime.entries()) {
assertEquals(await self.guessMimeType(ext, null), mime)
}
})
it("should attempt to load the external MIME type sniffing library if 'useMimeSniffingLibrary' is set to true", async () => {
self.LibResilientConfig = {
plugins: [{
name: 'resolve-all',
}],
useMimeSniffingLibrary: true,
loggedComponents: ['service-worker']
}
window.fileType = {
fileTypeFromBuffer: (ext, content)=>{
console.log(`fileTypeFromBuffer(${ext}, ${content.length})`)
}
}
window.LibResilientPluginConstructors.set('resolve-all', ()=>{
return {
name: 'resolve-all',
description: 'Resolve all requests.',
version: '0.0.1',
fetch: fetch
}
})
await import("../../service-worker.js?" + window.test_id);
await self.dispatchEvent(new Event('install'))
await self.waitForSWInstall()
assertSpyCall(self.importScripts, 0, {args: ["./lib/file-type.js"]})
})
it("should default to extension-based MIME sniffing if 'useMimeSniffingLibrary' is set to true but loading the library failed", async () => {
self.LibResilientConfig = {
plugins: [{
name: 'resolve-all',
}],
useMimeSniffingLibrary: true,
loggedComponents: ['service-worker']
}
window.fileType = undefined
window.LibResilientPluginConstructors.set('resolve-all', ()=>{
return {
name: 'resolve-all',
description: 'Resolve all requests.',
version: '0.0.1',
fetch: fetch
}
})
await import("../../service-worker.js?" + window.test_id);
await self.dispatchEvent(new Event('install'))
await self.waitForSWInstall()
assertEquals(await self.guessMimeType("png", "test arg 2"), "image/png")
})
it("should call the external library function in 'guessMimeType()', passing the second argument, if 'useMimeSniffingLibrary' is set to true", async () => {
self.LibResilientConfig = {
plugins: [{
name: 'resolve-all',
}],
useMimeSniffingLibrary: true,
loggedComponents: ['service-worker']
}
window.fileType = {
fileTypeFromBuffer: spy(async (ext, content)=>{
return undefined
})
}
window.LibResilientPluginConstructors.set('resolve-all', ()=>{
return {
name: 'resolve-all',
description: 'Resolve all requests.',
version: '0.0.1',
fetch: fetch
}
})
await import("../../service-worker.js?" + window.test_id);
await self.dispatchEvent(new Event('install'))
await self.waitForSWInstall()
await self.guessMimeType("test arg 1", "test arg 2")
assertSpyCall(window.fileType.fileTypeFromBuffer, 0, {args: ["test arg 2"]})
})
it("should revert to guessing MIME type based on extension if the external library function in 'guessMimeType()' errors out", async () => {
self.LibResilientConfig = {
plugins: [{
name: 'resolve-all',
}],
useMimeSniffingLibrary: true,
loggedComponents: ['service-worker']
}
window.fileType = {
fileTypeFromBuffer: spy(async (ext, content)=>{
throw new Error('test error')
})
}
window.LibResilientPluginConstructors.set('resolve-all', ()=>{
return {
name: 'resolve-all',
description: 'Resolve all requests.',
version: '0.0.1',
fetch: fetch
}
})
await import("../../service-worker.js?" + window.test_id);
await self.dispatchEvent(new Event('install'))
await self.waitForSWInstall()
assertEquals(await self.guessMimeType("gif", "test arg 2"), "image/gif")
assertSpyCalls(window.fileType.fileTypeFromBuffer, 1)
})
it("should revert to guessing MIME type based on extension if the external library function in 'guessMimeType()' fails to guess the MIME type", async () => {
self.LibResilientConfig = {
plugins: [{
name: 'resolve-all',
}],
useMimeSniffingLibrary: true,
loggedComponents: ['service-worker']
}
window.fileType = {
fileTypeFromBuffer: spy(async (ext, content)=>{
return undefined
})
}
window.LibResilientPluginConstructors.set('resolve-all', ()=>{
return {
name: 'resolve-all',
description: 'Resolve all requests.',
version: '0.0.1',
fetch: fetch
}
})
await import("../../service-worker.js?" + window.test_id);
await self.dispatchEvent(new Event('install'))
await self.waitForSWInstall()
assertEquals(await self.guessMimeType("png", "test arg 2"), "image/png")
assertSpyCalls(window.fileType.fileTypeFromBuffer, 1)
})
it("should revert to guessing MIME type based on extension if the external library function in 'guessMimeType()' returns an unexpected value type", async () => {
self.LibResilientConfig = {
plugins: [{
name: 'resolve-all',
}],
useMimeSniffingLibrary: true,
loggedComponents: ['service-worker']
}
window.fileType = {
fileTypeFromBuffer: spy(async (ext, content)=>{
return "this should not be a string"
})
}
window.LibResilientPluginConstructors.set('resolve-all', ()=>{
return {
name: 'resolve-all',
description: 'Resolve all requests.',
version: '0.0.1',
fetch: fetch
}
})
await import("../../service-worker.js?" + window.test_id);
await self.dispatchEvent(new Event('install'))
await self.waitForSWInstall()
assertEquals(await self.guessMimeType("jpg", "test arg 2"), "image/jpeg")
assertSpyCalls(window.fileType.fileTypeFromBuffer, 1)
})
it("should revert to guessing MIME type based on extension if the external library function in 'guessMimeType()' returns an incorrectly formatted object", async () => {
self.LibResilientConfig = {
plugins: [{
name: 'resolve-all',
}],
useMimeSniffingLibrary: true,
loggedComponents: ['service-worker']
}
window.fileType = {
fileTypeFromBuffer: spy(async (ext, content)=>{
return {bad: "data"}
})
}
window.LibResilientPluginConstructors.set('resolve-all', ()=>{
return {
name: 'resolve-all',
description: 'Resolve all requests.',
version: '0.0.1',
fetch: fetch
}
})
await import("../../service-worker.js?" + window.test_id);
await self.dispatchEvent(new Event('install'))
await self.waitForSWInstall()
assertEquals(await self.guessMimeType("txt", "test arg 2"), "text/plain")
assertSpyCalls(window.fileType.fileTypeFromBuffer, 1)
})
it("should ignore extension if the external library function in 'guessMimeType()' returns valid data", async () => {
self.LibResilientConfig = {
plugins: [{
name: 'resolve-all',
}],
useMimeSniffingLibrary: true,
loggedComponents: ['service-worker']
}
window.fileType = {
fileTypeFromBuffer: spy(async (ext, content)=>{
return {ext: "txt", mime: "text/plain"}
})
}
window.LibResilientPluginConstructors.set('resolve-all', ()=>{
return {
name: 'resolve-all',
description: 'Resolve all requests.',
version: '0.0.1',
fetch: fetch
}
})
await import("../../service-worker.js?" + window.test_id);
await self.dispatchEvent(new Event('install'))
await self.waitForSWInstall()
assertEquals(await self.guessMimeType("png", "test arg 2"), "text/plain")
assertSpyCalls(window.fileType.fileTypeFromBuffer, 1)
})
it("should not guess a MIME type if both external library function and built-in extension-based heuristic fail", async () => {
self.LibResilientConfig = {
plugins: [{
name: 'resolve-all',
}],
useMimeSniffingLibrary: true,
loggedComponents: ['service-worker']
}
window.fileType = {
fileTypeFromBuffer: spy(async (ext, content)=>{
return {bad: "data"}
})
}
window.LibResilientPluginConstructors.set('resolve-all', ()=>{
return {
name: 'resolve-all',
description: 'Resolve all requests.',
version: '0.0.1',
fetch: fetch
}
})
await import("../../service-worker.js?" + window.test_id);
await self.dispatchEvent(new Event('install'))
await self.waitForSWInstall()
// if we're not sure, according to RFC 7231 we should not set Content-Type
// (or, set it to an empty string)
// https://www.rfc-editor.org/rfc/rfc7231#section-3.1.1.5
assertEquals(await self.guessMimeType("no-such-extension", "test arg 2"), "")
assertSpyCalls(window.fileType.fileTypeFromBuffer, 1)
})
})

2
config.json.example
Wyświetl plik

@ -19,5 +19,7 @@
"gunPubkey": "<your-gun-pubkey>"
}]
}],
"useMimeSniffingLibrary": false,
"normalizeQueryParams": true,
"loggedComponents": ["service-worker", "fetch", "cache", "any-of", "alt-fetch", "gun-ipfs"]
}

15
docs/ARCHITECTURE.md
Wyświetl plik

@ -164,3 +164,18 @@ The code in the browser window context is responsible for keeping a more permane
When the browser window context wants to message the service worker, it uses the [`Worker.postMessage()`](https://developer.mozilla.org/en-US/docs/Web/API/Worker/postMessage) call, with `clientId` field set to the relevant client ID if a response is expected. Service Worker then again responds using `Client.postMessage()` using the `clientId` field as source of the `Client ID`.
## MIME type guessing
Some plugins (for example, those based on IPFS, like [`dnslink-ipfs`](../plugins/dnslink-ipfs/)), receive content without a `Content-Type` header, because the transport simply does not support it. That's problematic, as the `Content-Type` header is used by the browser to decide what can be done with a file -- display it if it's an image, run it if it's a JavaScript file, and so on.
For the purpose of guessing the MIME type of a given piece of content (in order to populate the `Content-Type` header) the LibResilient's Service Worker implements the `guessMimeType()` function, available to any plugin.
By default the function attempts to guess the MIME type of content only by considering the file extension of the path used to retrieve it. There is a hard-coded list of extension-to-MIME-type mappings that should cover most file types relevant on the Web.
This might not be enough, however. So, the Service Worker can *optionally* load an external library that can establish a MIME type based on the actual content. Currently that is [`file-type`](https://github.com/sindresorhus/file-type/), distributed along with LibResilient in the `lib/` directory).
To enable content-based MIME type guessing, set the `useMimeSniffingLibrary` to `true` in `config.json`.
By default, content-based MIME guessing is *disabled*, because it is somewhat slower (content needs to be read, inspected, and compared to a lot of signatures), and relies on an external library that needs to be distributed along with LibResilient, while not being needed for most plugins, nor necessary for those that do actually need to guess content MIME types.
When enabled, content-based MIME guessing is attempted first for any given piece of content that requires it. If it fails, extension-based MIME guessing is then used.

5
lib/COPYRIGHT.md
Wyświetl plik

@ -13,3 +13,8 @@ The following files are part of the [GunDB](https://github.com/amark/gun) projec
- `gun.js`
- `sea.js`
- `webrtc.js`
## `file-type` project
The following files are generated based on the [`file-type`](https://github.com/sindresorhus/file-type/) project. The project itself is licensed under MIT license. The files below include bundled dependencies of the `file-type` project, all of which are either licensed under MIT or BSD 2-Clause licenses:
- `file-type.js`

14282
lib/file-type.js 100644
Wyświetl plik

Plik diff jest za duży Load Diff

9
plugins/dnslink-ipfs/__tests__/browser.test.js
Wyświetl plik

@ -118,11 +118,18 @@ beforeEach(()=>{
})
describe('browser: dnslink-ipfs plugin', async () => {
// we need access to the API
await import("../../../service-worker.js");
// API requires a pluginName
self.pluginName = 'dnslink-ipfs'
window.LibResilientPluginConstructors = new Map()
window.LR = {
log: (component, ...items)=>{
console.debug(component + ' :: ', ...items)
}
},
guessMimeType: self.guessMimeType
}
window.fetchResponse = []
window.resolvingFetch = null

42
plugins/dnslink-ipfs/index.js
Wyświetl plik

@ -85,34 +85,6 @@
}
LR.log(pluginName, "+-- starting DNSLink lookup of: '" + dnslinkAddr + "'");
/*
* naïvely assume content type based on file extension
* TODO: this needs a fix
*/
var contentType = '';
switch (dnslinkAddr.split('.').pop().toLowerCase()) {
case 'html':
case 'htm':
contentType = 'text/html';
break;
case 'css':
contentType = 'text/css';
break;
case 'js':
contentType = 'text/javascript';
break;
case 'svg':
contentType = 'image/svg+xml';
break;
case 'ico':
contentType = 'image/x-icon';
break;
case 'json':
contentType = 'application/json';
break;
}
LR.log(pluginName, " +-- guessed contentType : " + contentType);
// TODO: error handling!
return ipfs.name.resolve('/ipns/' + dnslinkAddr).next().then(ipfsaddr => {
@ -145,11 +117,17 @@
LR.log(pluginName, '+-- got a DNSLink-resolved IPFS-stored file; content is: ' + typeof content);
// let's guess the content type
let contentType = await LR.guessMimeType(
dnslinkAddr.split('.').pop().toLowerCase(),
content
)
// creating and populating the blob
var blob = new Blob(
[content],
{'type': contentType}
);
let blob = new Blob(
[content],
{'type': contentType}
);
return new Response(
blob,

19
plugins/gun-ipfs/__tests__/browser.test.js
Wyświetl plik

@ -140,11 +140,18 @@ beforeEach(()=>{
window.LR = {
log: spy((component, ...items)=>{
console.debug(component + ' :: ', ...items)
})
}),
guessMimeType: self.guessMimeType
}
})
describe('browser: gun-ipfs plugin', async () => {
// we need access to the API
await import("../../../service-worker.js");
// API requires a pluginName
self.pluginName = 'dnslink-ipfs'
window.LibResilientPluginConstructors = new Map()
window.fetchResponse = []
window.resolvingFetch = null
@ -217,7 +224,11 @@ describe('browser: gun-ipfs plugin', async () => {
]})
})
it("should correctly guess content types when fetching", async ()=>{
// TODO: currently disabled, because:
// 1. the plugin is in an unusable state anyway
// 2. most of this test needs to go into service worker test suite itself
// 3. fixing this test is substantial work, as it requires mocking IPFS.get() properly for every content type
it.ignore("should correctly guess content types when fetching", async ()=>{
let gunIpfsPlugin = LibResilientPluginConstructors.get('gun-ipfs')(LR, init)
let response = await gunIpfsPlugin.fetch('https://resilient.is/test/')
@ -327,7 +338,9 @@ describe('browser: gun-ipfs plugin', async () => {
{
args: [
"gun-ipfs",
" +-- guessed contentType : application/json"
"getGunData()\n",
"+-- gunUser : object\n",
"+-- gunaddr[] : resilient.is,/test.json"
]})
// TODO: implement actual content check once gun-ipfs plugin gets updated
// to work with current IPFS version

35
plugins/gun-ipfs/index.js
Wyświetl plik

@ -188,34 +188,6 @@ if (typeof window === 'undefined') {
`+-- gunUser : ${typeof gunUser}`
);
/*
* naïvely assume content type based on file extension
* TODO: this needs a fix
*/
var contentType = '';
switch (gunaddr.slice(-1)[0].split('.', -1)[1].toLowerCase()) {
case 'html':
case 'htm':
contentType = 'text/html';
break;
case 'css':
contentType = 'text/css';
break;
case 'js':
contentType = 'text/javascript';
break;
case 'json':
contentType = 'application/json';
break;
case 'svg':
contentType = 'image/svg+xml';
break;
case 'ico':
contentType = 'image/x-icon';
break;
}
LR.log(pluginName, " +-- guessed contentType : " + contentType);
const ipfsaddr = await getGunData(gunaddr)
LR.log(pluginName, `starting IPFS lookup of: '${ipfsaddr}'`);
LR.log(pluginName, `ipfs is: '${ipfs}'`);
@ -244,6 +216,13 @@ if (typeof window === 'undefined') {
LR.log(pluginName, `got a Gun-addressed IPFS-stored file: ${file.value.path}\n+-- content is: ${typeof content}`);
// let's guess the content type
let contentType = await LR.guessMimeType(
gunaddr.slice(-1)[0].split('.', -1)[1].toLowerCase(),
content
)
// creating and populating the blob
let blob = new Blob(
[content],
{ type: contentType }

33
plugins/ipns-ipfs/index.js
Wyświetl plik

@ -114,31 +114,6 @@
console.log(" +-- gun : " + typeof gun);
console.log(" +-- gunUser : " + typeof gunUser);
/*
* naïvely assume content type based on file extension
* TODO: this needs a fix
*/
var contentType = '';
switch (gunaddr.slice(-1)[0].split('.', -1)[1].toLowerCase()) {
case 'html':
case 'htm':
contentType = 'text/html';
break;
case 'css':
contentType = 'text/css';
break;
case 'js':
contentType = 'text/javascript';
break;
case 'svg':
contentType = 'image/svg+xml';
break;
case 'ico':
contentType = 'image/x-icon';
break;
}
console.log(" +-- guessed contentType : " + contentType);
return getGunData(gunaddr).then(ipfsaddr => {
console.log("3. Starting IPFS lookup of: '" + ipfsaddr + "'");
return ipfs.get(ipfsaddr).next();
@ -157,8 +132,14 @@
}
return content
}
return getContent(file.value.content).then((content)=>{
return getContent(file.value.content).then(async (content)=>{
console.log('4. Got a Gun-addressed IPFS-stored file: ' + file.value.path + '; content is: ' + typeof content);
// let's guess the content type
let contentType = await LR.guessMimeType(
gunaddr.slice(-1)[0].split('.', -1)[1].toLowerCase(),
content
)
// creating and populating the blob
var blob = new Blob(
[content],

6
plugins/test-plugin/README.md 100644
Wyświetl plik

@ -0,0 +1,6 @@
# Plugin: `test-plugin`
- **status**: permanently unstable
- **type**: whatever is needed
This is a plugin used for testing things out. Should never be used in production.

126
plugins/test-plugin/__tests__/browser.test.js 100644
Wyświetl plik

@ -0,0 +1,126 @@
import {
describe,
it,
afterEach,
beforeEach
} from "https://deno.land/std@0.183.0/testing/bdd.ts";
import {
assert,
assertRejects,
assertEquals
} from "https://deno.land/std@0.183.0/testing/asserts.ts";
import {
assertSpyCall,
assertSpyCalls,
spy,
} from "https://deno.land/std@0.183.0/testing/mock.ts";
beforeEach(()=>{
window.fetch = spy(window.resolvingFetch)
})
afterEach(()=>{
window.fetch = null
})
describe('browser: fetch plugin', async () => {
window.LibResilientPluginConstructors = new Map()
window.LR = {
log: (component, ...items)=>{
console.debug(component + ' :: ', ...items)
}
}
window.resolvingFetch = (url, init) => {
return Promise.resolve(
new Response(
new Blob(
[JSON.stringify({ test: "success" })],
{type: "application/json"}
),
{
status: 200,
statusText: "OK",
headers: {
'ETag': 'TestingETagHeader'
}
}
)
)
}
window.fetch = null
await import("../../../plugins/fetch/index.js");
it("should register in LibResilientPluginConstructors", () => {
assertEquals(LibResilientPluginConstructors.get('fetch')(LR).name, 'fetch');
});
it("should return data from fetch()", async () => {
const response = await LibResilientPluginConstructors.get('fetch')(LR).fetch('https://resilient.is/test.json');
assertSpyCalls(fetch, 1);
assertEquals(await response.json(), {test: "success"})
});
it("should pass the Request() init data to fetch()", async () => {
var initTest = {
method: "GET",
headers: new Headers({"x-stub": "STUB"}),
mode: "mode-stub",
credentials: "credentials-stub",
cache: "cache-stub",
referrer: "referrer-stub",
redirect: "redirect-stub",
integrity: "integrity-stub"
}
const response = await LibResilientPluginConstructors.get('fetch')(LR).fetch('https://resilient.is/test.json', initTest);
assertSpyCall(
fetch,
0,
{
args: [
'https://resilient.is/test.json',
initTest // TODO: does the initTest actually properly work here?
]
})
assertEquals(await response.json(), {test: "success"})
});
it("should set the LibResilient headers", async () => {
const response = await LibResilientPluginConstructors.get('fetch')(LR).fetch('https://resilient.is/test.json');
assertSpyCalls(fetch, 1);
assertEquals(await response.json(), {test: "success"})
assertEquals(response.headers.has('X-LibResilient-Method'), true)
assertEquals(response.headers.get('X-LibResilient-Method'), 'fetch')
assertEquals(response.headers.has('X-LibResilient-Etag'), true)
assertEquals(response.headers.get('X-LibResilient-ETag'), 'TestingETagHeader')
});
it("should throw an error when HTTP status is >= 400", async () => {
window.fetch = (url, init) => {
const response = new Response(
new Blob(
["Not Found"],
{type: "text/plain"}
),
{
status: 404,
statusText: "Not Found",
url: url
});
return Promise.resolve(response);
}
assertRejects(
async () => {
return await LibResilientPluginConstructors
.get('fetch')(LR)
.fetch('https://resilient.is/test.json') },
Error,
'HTTP Error: 404 Not Found'
)
});
})

18
plugins/test-plugin/index.html 100644
Wyświetl plik

@ -0,0 +1,18 @@
<!DOCTYPE html>
<html>
<head>
<script>
self.LibResilientPluginConstructors = self.LibResilientPluginConstructors || new Map()
self.log = console.log
</script>
<script src="./index.js"></script>
<script>
var theplugin = LibResilientPluginConstructors.get('test-plugin')(self)
</script>
</head>
<body>
<h1><code>test-plugin</code></h1>
<p>This is a simple debugging harness for the <code>test-plugin</code> plugin of LibResilient.</p>
<p>The plugin should now have been initialized in the <code>theplugin</code> global variable. Open your browser's JavaScript console to start playing with it.</p>
</body>
</html>

95
plugins/test-plugin/index.js 100644
Wyświetl plik

@ -0,0 +1,95 @@
/* ========================================================================= *\
|* === Testing plugin === *|
\* ========================================================================= */
/**
* this plugin does not implement any push method
*/
// no polluting of the global namespace please
(function(LRPC){
// this never changes
const pluginName = "test-plugin"
LRPC.set(pluginName, (LR, init={})=>{
/*
* plugin config settings
*/
// sane defaults
let defaultConfig = {}
// merge the defaults with settings from init
let config = {...defaultConfig, ...init}
/**
* getting content using regular HTTP(S) fetch()
*/
let fetchContent = (url, init={}) => {
LR.log(pluginName, `regular fetch: ${url}`)
// we really want to make fetch happen, Regina!
// TODO: this change should *probably* be handled on the Service Worker level
init.cache = 'reload'
// run built-in regular fetch()
return fetch(url, init)
.then(async (response) => {
// 4xx? 5xx? that's a paddlin'
if (response.status >= 400) {
// throw an Error to fall back to LibResilient:
throw new Error('HTTP Error: ' + response.status + ' ' + response.statusText);
}
// all good, it seems
LR.log(pluginName, `fetched successfully: ${response.url}`);
// we need to create a new Response object
// with all the headers added explicitly,
// since response.headers is immutable
var responseInit = {
status: response.status,
statusText: response.statusText,
headers: {},
url: response.url
};
response.headers.forEach(function(val, header){
responseInit.headers[header] = val;
});
// add the X-LibResilient-* headers to the mix
responseInit.headers['X-LibResilient-Method'] = pluginName
responseInit.headers['X-LibResilient-ETag'] = response.headers.get('ETag')
let ab = await response.clone()
.blob()
.then(blob=>{
return blob.arrayBuffer()
})
// let's guess the content type
let contentType = await LR.guessMimeType(
url.split('.').pop().toLowerCase(),
ab
)
LR.log(pluginName, '*** guessed content type:', contentType)
// return the new response, using the Blob from the original one
return response
.blob()
.then((blob) => {
return new Response(
blob,
responseInit
)
})
})
}
// return the plugin
return {
name: pluginName,
description: 'Just a regular HTTP(S) fetch()',
version: 'COMMIT_UNKNOWN',
fetch: fetchContent
}
})
// done with not polluting the global namespace
})(LibResilientPluginConstructors)

142
service-worker.js
Wyświetl plik

@ -13,9 +13,11 @@ if (!Array.isArray(self.LibResilientPlugins)) {
// which then can be modified via config.json
if (typeof self.LibResilientConfig !== 'object' || self.LibResilientConfig === null) {
self.LibResilientConfig = {
// how long do we wait before we decide that a plugin is unresponsive,
// and move on?
defaultPluginTimeout: 10000,
// plugins settings namespace
//
// this defines which plugins get loaded,
@ -30,6 +32,7 @@ if (typeof self.LibResilientConfig !== 'object' || self.LibResilientConfig === n
name: 'cache'
}
],
// which components should be logged?
// this is an array of strings, components not listed here
// will have their debug output disabled
@ -41,6 +44,7 @@ if (typeof self.LibResilientConfig !== 'object' || self.LibResilientConfig === n
'fetch',
'cache'
],
// should we normalize query params?
//
// this usually makes sense: a request to example.com/?a=a&b=b is
@ -48,7 +52,27 @@ if (typeof self.LibResilientConfig !== 'object' || self.LibResilientConfig === n
//
// but in case a given website does something weird with query params...
// ..normalization can be disabled here
normalizeQueryParams: true
normalizeQueryParams: true,
// do we want to use content-based MIME type detection using an external library?
//
// some plugins (for example, based on IPFS), receive content without a content type,
// because the transport simply does not support it. so we need to find a way to
// figure out the content type on our own -- that's done by the guessMimeType() function
//
// it can do this based on the extension of the file being requested,
// but that is a limited, imperfect approach.
//
// it can also load an external library (currently that's `file-type`), and guess
// the content type based on it. this approach is more exact, works for paths that
// do not have an "extension", and works for many more MIME types than the alternative.
//
// however, it is also slower (content needs to be read, inspected, and compared to a lot
// of signatures), and relies on an external library that needs to be distributed along
// with LibResilient.
//
// so, since it is not needed in case of most plugins, it is disabled by default.
useMimeSniffingLibrary: false
}
}
@ -69,6 +93,101 @@ self.log = function(component, ...items) {
}
}
// Map() of file extensions to MIME types for the guessing game below
// this is by no means complete, and focuses mainly on formats that
// are important on the Web
let ext_to_mime = new Map([
['htm', 'text/html'],
['html', 'text/html'],
['css', 'text/css'],
['js', 'text/javascript'],
['json', 'application/json'],
['svg', 'image/svg+xml'],
['ico', 'image/x-icon'],
['gif', 'image/gif'],
['png', 'image/png'],
['jpg', 'image/jpeg'],
['jpeg', 'image/jpeg'],
['jpe', 'image/jpeg'],
['jfif', 'image/jpeg'],
['pjpeg', 'image/jpeg'],
['pjp', 'image/jpeg'],
['webp', 'image/webp'],
['avi', 'video/avi'],
['mp4', 'video/mp4'],
['mp2', 'video/mpeg'],
['mp3', 'audio/mpeg'],
['mpa', 'video/mpeg'],
['pdf', 'application/pdf'],
['txt', 'text/plain'],
['ics', 'text/calendar'],
['jsonld', 'application/ld+json'],
['mjs', 'text/javascript'],
['oga', 'audio/ogg'],
['ogv', 'video/ogg'],
['ogx', 'application/ogg'],
['opus', 'audio/opus'],
['otf', 'font/otf'],
['ts', 'video/mp2t'],
['ttf', 'font/ttf'],
['weba', 'audio/webm'],
['webm', 'video/webm'],
['webp', 'image/webp'],
['woff', 'font/woff'],
['woff2', 'font/woff2'],
['xhtml', 'application/xhtml+xml'],
['xml', 'application/xml']
])
// preparing the variable for the MIME detection module
// in case we want to use it
let detectMimeFromBuffer = null
/**
* guess the MIME type, based on content and path extension
*
* important: according to RFC 7231 we should not set Content-Type if we're not sure!
* https://www.rfc-editor.org/rfc/rfc7231#section-3.1.1.5
*
* @param ext - the extension of the path content was fetched as
* @param content - the content itself
* @returns string containing the MIME type, or empty string if guessing failed
*/
self.guessMimeType = async function(ext, content) {
// if we have file-type library loaded, that means that useMimeSniffingLibrary config field is set to true
// and that we were able to load file-type.js
//
// in other words, we want to use it, we can use it -- so use it!
if (detectMimeFromBuffer !== null) {
let ft = undefined
try {
ft = await detectMimeFromBuffer(content)
} catch (e) {
self.log('service-worker', "+-- error while trying to guess MIME type based on content:", e);
}
// did we actually get anything?
if ( (ft !== undefined) && (typeof ft === "object") && ("mime" in ft) ) {
// yup!
self.log('service-worker', "+-- guessed MIME type based on content: " + ft.mime);
return ft.mime;
} else {
self.log('service-worker', "+-- unable to guess MIME type based on content.")
}
}
// an empty string is in our case equivalent to not setting the Content-Type
// as `new Blob()` with no `type` option set ends up having type set to an empty string
if (ext_to_mime.has(ext)) {
self.log('service-worker', "+-- guessed MIME type based on extension: " + ext_to_mime.get(ext));
return ext_to_mime.get(ext)
}
// if we're unable to guess the MIME type, we need to return an empty string
self.log('service-worker', " +-- unable to guess the MIME type");
return "";
}
/**
* verifying a config data object
@ -100,11 +219,18 @@ let verifyConfigData = (cdata) => {
}
// normalizeQueryParams is optional
if ("normalizeQueryParams" in cdata) {
if (cdata.normalizeQueryParams !== true && cdata.normalizeQueryParams !=- false) {
if (cdata.normalizeQueryParams !== true && cdata.normalizeQueryParams !== false) {
self.log('service-worker', 'fetched config contains invalid "normalizeQueryParams" data (boolean expected)')
return false;
}
}
// useMimeSniffingLibrary is optional
if ("useMimeSniffingLibrary" in cdata) {
if (cdata.useMimeSniffingLibrary !== true && cdata.useMimeSniffingLibrary !== false) {
self.log('service-worker', 'fetched config contains invalid "useMimeSniffingLibrary" data (boolean expected)')
return false;
}
}
// we're good
return true;
}
@ -214,6 +340,18 @@ let initServiceWorker = async () => {
self.log('service-worker', 'config loading failed, using defaults; error:', e)
}
// first let's deal with the easy part -- do we want to use MIME type guessing based on content?
if (self.LibResilientConfig.useMimeSniffingLibrary === true) {
// we do! load the external lib
self.importScripts(`./lib/file-type.js`)
if (typeof fileType !== 'undefined' && "fileTypeFromBuffer" in fileType) {
detectMimeFromBuffer = fileType.fileTypeFromBuffer
self.log('service-worker', 'loaded external MIME sniffing library')
} else {
self.log('service-worker', 'failed to load external MIME sniffing library!')
}
}
// create the LibResilientPluginConstructors map
// the global... hack is here so that we can run tests; not the most elegant
// TODO: find a better way