diff --git a/README.md b/README.md index e8edf5f..df8f1dc 100644 --- a/README.md +++ b/README.md @@ -59,22 +59,6 @@ npm run generate-dev-cert # Will generate a certificate for self-signing PDFs. F npm run dev # Starts the development server on port 3000 ``` -### Access keys -Create an access key to test with: - -```bash -$ uuidgen -BB67BBC4-1F4B-4353-8E6D-9927A10F4509 -``` - -And then add the key to `app/data/access-keys.json`: - -```json -{ - "BB67BBC4-1F4B-4353-8E6D-9927A10F4509": true -} -``` - ### Certificates history The _"Signatures Verification Page"_ page lists the certificates that were used for signing PDFs with the app. You may provide that history by creating two files under `/data`: diff --git a/app/const.js b/app/const.js index 18972cf..a5a3f74 100644 --- a/app/const.js +++ b/app/const.js @@ -46,13 +46,13 @@ export const STATIC_PATH = `${process.env.PWD}/app/static/`; * Maximum capture processes that can be run in parallel. * @constant */ -export const MAX_PARALLEL_CAPTURES_TOTAL = 200; +export const MAX_PARALLEL_CAPTURES_TOTAL = 50; /** - * Maximum capture processes that can be run in parallel for a given key. + * Maximum capture processes that can be run in parallel for a IP address. * @constant */ -export const MAX_PARALLEL_CAPTURES_PER_ACCESS_KEY = 20; +export const MAX_PARALLEL_CAPTURES_PER_IP = 2; /** * APP version. Pulled from `package.json` by default. diff --git a/app/server.js b/app/server.js index 53cc9df..c5b9e1d 100644 --- a/app/server.js +++ b/app/server.js @@ -8,24 +8,32 @@ import assert from "assert"; import nunjucks from "nunjucks"; -import { AccessKeys, CertsHistory, SuccessLog, TwitterCapture } from "./utils/index.js"; +import { IPBlockList, CertsHistory, SuccessLog, TwitterCapture } from "./utils/index.js"; import { TEMPLATES_PATH, STATIC_PATH, MAX_PARALLEL_CAPTURES_TOTAL, - MAX_PARALLEL_CAPTURES_PER_ACCESS_KEY, + MAX_PARALLEL_CAPTURES_PER_IP, } from "./const.js"; - /** * @type {SuccessLog} */ export const successLog = new SuccessLog(); /** - * @type {AccessKeys} + * @type {IPBlockList} */ -const accessKeys = new AccessKeys(); +const ipBlockList = new IPBlockList(); + +/** + * Fastify-cli options + * @constant + */ +export const options = { + trustProxy: true, + logger: true +} /** * Keeps track of how many capture processes are currently running. @@ -36,15 +44,15 @@ const accessKeys = new AccessKeys(); * @type {{ * currentTotal: number, * maxTotal: number, - * currentByAccessKey: object., - * maxPerAccessKey: number + * currentByIp: object., + * maxPerIp: number * }} */ export const CAPTURES_WATCH = { currentTotal: 0, maxTotal: MAX_PARALLEL_CAPTURES_TOTAL, - currentByAccessKey: {}, - maxPerAccessKey: MAX_PARALLEL_CAPTURES_PER_ACCESS_KEY, + currentByIp: {}, + maxPerIp: MAX_PARALLEL_CAPTURES_PER_IP, } export default async function (fastify, opts) { @@ -67,11 +75,10 @@ export default async function (fastify, opts) { fastify.post('/', capture); fastify.get('/check', check); - + fastify.get('/api/v1/hashes/check/:hash', checkHash); }; - /** * [GET] / * Shows the landing page and capture form. @@ -98,7 +105,6 @@ async function index(request, reply) { * Subject to captures rate limiting (see `CAPTURES_WATCH`). * * Body is expected as `application/x-www-form-urlencoded` with the following fields: - * - access-key * - url * - unfold-thread (optional) * @@ -110,17 +116,18 @@ async function index(request, reply) { */ async function capture(request, reply) { const data = request.body; - const accessKey = data["access-key"]; - + const ip = request.ip; + let why = null; + request.log.info(`Capture capacity: ${CAPTURES_WATCH.currentTotal} / ${CAPTURES_WATCH.maxTotal}.`); - + // - // Check access key + // Check that IP is not in block list // - if (!accessKeys.check(accessKey)) { + if (ipBlockList.check(ip)) { const html = nunjucks.render(`${TEMPLATES_PATH}index.njk`, { error: true, - errorReason: "ACCESS-KEY" + errorReason: "IP" }); return reply @@ -148,6 +155,25 @@ async function capture(request, reply) { .send(html); } + // + // Check "why" field + // + try { + why = data.why.trim(); + assert(why.length > 0); + } + catch(err) { + const html = nunjucks.render(`${TEMPLATES_PATH}index.njk`, { + error: true, + errorReason: "WHY" + }); + + return reply + .code(400) + .header('Content-Type', 'text/html; charset=utf-8') + .send(html); + } + // // Check that there is still capture capacity (total) // @@ -164,9 +190,9 @@ async function capture(request, reply) { } // - // Check that there is still capture capacity (for this access key) + // Check that there is still capture capacity (for this IP) // - if (CAPTURES_WATCH.currentByAccessKey[accessKey] >= CAPTURES_WATCH.maxPerAccessKey) { + if (CAPTURES_WATCH.currentByIp[ip] >= CAPTURES_WATCH.maxPerIp) { const html = nunjucks.render(`${TEMPLATES_PATH}index.njk`, { error: true, errorReason: "TOO-MANY-CAPTURES-USER" @@ -182,20 +208,20 @@ async function capture(request, reply) { // Process capture request // try { - // Add request to total and per-key counter + // Add request to total and per-IP counter CAPTURES_WATCH.currentTotal += 1; - if (accessKey in CAPTURES_WATCH.currentByAccessKey) { - CAPTURES_WATCH.currentByAccessKey[accessKey] += 1; + if (ip in CAPTURES_WATCH.currentByIp) { + CAPTURES_WATCH.currentByIp[ip] += 1; } else { - CAPTURES_WATCH.currentByAccessKey[accessKey] = 1; + CAPTURES_WATCH.currentByIp[ip] = 1; } const tweets = new TwitterCapture(data.url, {runBrowserBehaviors: "unfold-thread" in data}); const pdf = await tweets.capture(); - successLog.add(accessKey, pdf); + successLog.add(ip, why, pdf); // Generate a filename for the PDF based on url. // Example: harvardlil-status-123456789-2022-11-25.pdf @@ -232,8 +258,8 @@ async function capture(request, reply) { finally { CAPTURES_WATCH.currentTotal -= 1; - if (accessKey && accessKey in CAPTURES_WATCH.currentByAccessKey) { - CAPTURES_WATCH.currentByAccessKey[data["access-key"]] -= 1; + if (ip && ip in CAPTURES_WATCH.currentByIp) { + CAPTURES_WATCH.currentByIp[ip] -= 1; } } } diff --git a/app/server.test.js b/app/server.test.js index 09dee59..b8b9ede 100644 --- a/app/server.test.js +++ b/app/server.test.js @@ -4,7 +4,6 @@ * @author The Harvard Library Innovation Lab * @license MIT */ -import fs from "fs"; import assert from "assert"; import crypto from "crypto"; @@ -12,32 +11,19 @@ import { test } from "tap"; import Fastify from "fastify"; import isHtml from "is-html"; -import { AccessKeys } from "./utils/index.js"; - import server, { CAPTURES_WATCH, successLog } from "./server.js"; import { DATA_PATH, CERTS_PATH } from "./const.js"; -/** - * Access keys fixture. - * @type {{active: string[], inactive: string[]}} - */ -const ACCESS_KEYS = (() => { - const rawAccessKeys = JSON.parse(fs.readFileSync(AccessKeys.filepath)); - - const out = { active: [], inactive: [] }; - - for (let [key, value] of Object.entries(rawAccessKeys)) { - value === true ? out.active.push(key) : out.inactive.push(key); - } - - return out; -})(); - /** * Dummy url of a thread to capture. */ const THREAD_URL = "https://twitter.com/HarvardLIL/status/1595150565428039680"; +/** + * Dummy reason for capture + */ +const WHY = "Testing thread-keeper"; + test("Integration tests for server.js", async(t) => { // Do not run tests if `CERTS_PATH` and `DATA_PATH` do not point to a `fixtures` folder @@ -64,36 +50,42 @@ test("Integration tests for server.js", async(t) => { t.type(isHtml(response.body), true, "Server serves HTML."); }); - test("[POST] / returns HTTP 401 + HTML on failed access key check.", async (t) => { + test("[POST] / returns HTTP 401 + HTML on blocked IP check.", async (t) => { const app = Fastify({logger: false}); await server(app, {}); - const scenarios = [ - "FOO-BAR", // Invalid key - ACCESS_KEYS.inactive[0], // Inactive key - null // No key - ] + const params = new URLSearchParams(); + params.append("url", THREAD_URL); - for (const accessKey of scenarios) { - const params = new URLSearchParams(); + const response = await app.inject({ + method: "POST", + url: "/", + remoteAddress: "1.2.3.4", + headers: { + "Content-Type": "application/x-www-form-urlencoded", + }, + body: params.toString() + }); - if (accessKey) { - params.append("access-key", accessKey); - } + t.equal(response.statusCode, 401, "Server returns HTTP 401."); - const response = await app.inject({ - method: "POST", - url: "/", - headers: { "Content-Type": "application/x-www-form-urlencoded" }, - body: params.toString(), - }); + const body = `${response.body}`; + t.type(isHtml(body), true, "Server serves HTML"); + t.equal(body.includes(`data-reason="IP"`), true, "With error message."); + }); - t.equal(response.statusCode, 401, "Server returns HTTP 401."); + test("[POST] / lets IPs that are no longer blocked make requests.", async (t) => { + const app = Fastify({logger: false}); + await server(app, {}); - const body = `${response.body}`; - t.type(isHtml(body), true, "Server serves HTML"); - t.equal(body.includes(`data-reason="ACCESS-KEY"`), true, "With error message."); - } + const response = await app.inject({ + method: "POST", + url: "/", + remoteAddress: "4.3.2.1", + }); + + // Should fail because no URL were passed, not because IP was blocked + t.equal(response.statusCode, 400, "Server returns HTTP 400."); }); test("[POST] / returns HTTP 400 + HTML on failed url check.", async (t) => { @@ -110,8 +102,6 @@ test("Integration tests for server.js", async(t) => { for (const url of scenarios) { const params = new URLSearchParams(); - params.append("access-key", ACCESS_KEYS.active[0]); - if (url) { params.append("url", url); } @@ -123,7 +113,7 @@ test("Integration tests for server.js", async(t) => { body: params.toString(), }); - t.equal(response.statusCode, 400, "Server returns HTTP 401."); + t.equal(response.statusCode, 400, "Server returns HTTP 400."); const body = `${response.body}`; t.type(isHtml(body), true, "Server serves HTML"); @@ -131,6 +121,40 @@ test("Integration tests for server.js", async(t) => { } }); + test("[POST] / returns HTTP 400 + HTML on failed \"why\" check.", async (t) => { + const app = Fastify({logger: false}); + await server(app, {}); + + const scenarios = [ + null, // Nothing + " ", // Empty + "", + ] + + for (const why of scenarios) { + const params = new URLSearchParams(); + + params.append("url", THREAD_URL); + + if (why) { + params.append("why", why); + } + + const response = await app.inject({ + method: "POST", + url: "/", + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + body: params.toString(), + }); + + t.equal(response.statusCode, 400, "Server returns HTTP 400."); + + const body = `${response.body}`; + t.type(isHtml(body), true, "Server serves HTML"); + t.equal(body.includes(`data-reason="WHY"`), true, "With error message."); + } + }); + test("[POST] / returns HTTP 503 + HTML on failed server capacity check.", async (t) => { const app = Fastify({logger: false}); await server(app, {}); @@ -138,8 +162,8 @@ test("Integration tests for server.js", async(t) => { CAPTURES_WATCH.currentTotal = CAPTURES_WATCH.maxTotal; // Simulate peak const params = new URLSearchParams(); - params.append("access-key", ACCESS_KEYS.active[0]); params.append("url", THREAD_URL); + params.append("why", WHY); const response = await app.inject({ method: "POST", @@ -161,19 +185,18 @@ test("Integration tests for server.js", async(t) => { const app = Fastify({logger: false}); await server(app, {}); - const userKey = ACCESS_KEYS.active[0]; - - CAPTURES_WATCH.currentByAccessKey[userKey] = CAPTURES_WATCH.maxPerAccessKey; + const userIp = "127.0.0.1"; + CAPTURES_WATCH.currentByIp[userIp] = CAPTURES_WATCH.maxPerIp; const params = new URLSearchParams(); - params.append("access-key", userKey); params.append("url", THREAD_URL); + params.append("why", WHY); const response = await app.inject({ method: "POST", url: "/", headers: { "Content-Type": "application/x-www-form-urlencoded" }, - body: params.toString(), + body: params.toString() }); t.equal(response.statusCode, 429, "Server returns HTTP 503."); @@ -182,7 +205,7 @@ test("Integration tests for server.js", async(t) => { t.type(isHtml(body), true, "Server serves HTML"); t.equal(body.includes(`TOO-MANY-CAPTURES-USER`), true, "With error message."); - delete CAPTURES_WATCH.currentByAccessKey[userKey]; + delete CAPTURES_WATCH.currentByIp[userIp]; }); test("[POST] / returns HTTP 200 + PDF", async (t) => { @@ -190,8 +213,8 @@ test("Integration tests for server.js", async(t) => { await server(app, {}); const params = new URLSearchParams(); - params.append("access-key", ACCESS_KEYS.active[0]); params.append("url", THREAD_URL); + params.append("why", WHY); params.append("unfold-thread", "on"); const response = await app.inject({ @@ -246,7 +269,7 @@ test("Integration tests for server.js", async(t) => { // Add entry to success logs const toHash = Buffer.from(`${Date.now()}`); const hash = crypto.createHash('sha512').update(toHash).digest('base64'); - successLog.add(ACCESS_KEYS.active[0], toHash); + successLog.add("127.0.0.1", WHY, toHash); const response = await app.inject({ method: "GET", diff --git a/app/static/index.css b/app/static/index.css index 666fa37..2e72c1f 100644 --- a/app/static/index.css +++ b/app/static/index.css @@ -266,7 +266,7 @@ body#index > main form label { padding-bottom: 0.25rem; } -body#index > main form input { +body#index > main form input, body#index > main form textarea { display: block; width: 100%; padding: 0.5rem; diff --git a/app/static/index.js b/app/static/index.js index 576c26e..dc330a3 100644 --- a/app/static/index.js +++ b/app/static/index.js @@ -7,15 +7,15 @@ const formSubmitDialog = document.querySelector("dialog#form-submit"); document.querySelector("body#index form button").addEventListener("click", (e) => { e.preventDefault(); const url = document.querySelector("body#index form input[name='url']"); - const accessKey = document.querySelector("body#index form input[name='access-key']") + const why = document.querySelector("body#index form input[name='why']"); if (!url.checkValidity()) { url.reportValidity(); return; } - if (!accessKey.checkValidity()) { - accessKey.reportValidity(); + if (!why.checkValidity()) { + why.reportValidity(); return; } diff --git a/app/templates/index.njk b/app/templates/index.njk index 212bbe7..2b939ca 100644 --- a/app/templates/index.njk +++ b/app/templates/index.njk @@ -34,11 +34,8 @@
- - + +
@@ -66,20 +63,24 @@

Something went wrong

- {% if errorReason and errorReason == "ACCESS-KEY" %} -

The access key provided is invalid or no longer active.

+ {% if errorReason and errorReason == "IP" %} +

Your access to this service has been restricted.

{% endif %} {% if errorReason and errorReason == "URL" %}

The url provided is not a valid twitter.com url.

{% endif %} + {% if errorReason and errorReason == "WHY" %} +

The url provided is not a valid twitter.com url.

+ {% endif %} + {% if errorReason and errorReason == "TOO-MANY-CAPTURES-TOTAL" %}

Too many requests. Please retry in a minute.

{% endif %} {% if errorReason and errorReason == "TOO-MANY-CAPTURES-USER" %} -

Please wait until the capture requests you've started are completed before starting a new one.

+

Too many requests for your IP address. Please retry in a minute.

{% endif %} {% if errorReason and errorReason == "CAPTURE-ISSUE" %} @@ -95,9 +96,9 @@

This site is an experiment by the Harvard Library Innovation Lab to let you download signed PDFs of Twitter URLs. Here's an example PDF we made from this tweet.

-

Who can use it?

+

Who can use it?

-

To use our website you'll need to contact us for an API key. We're currently only able to share a limited number with people like journalists, internet scholars, and archivists. But you can also use our open source software to stand up an archive server of your own, and share it with your friends.

+

While anyone can use this website, it is experimental and running with limited server capacity. But you can also use our open source software to stand up an archive server of your own, and share it with your friends.

Why make a PDF archiving tool for Twitter?

diff --git a/app/utils/AccessKeys.js b/app/utils/AccessKeys.js deleted file mode 100644 index 818fb0b..0000000 --- a/app/utils/AccessKeys.js +++ /dev/null @@ -1,63 +0,0 @@ -/** - * thread-keeper - * @module utils.AccessKeys - * @author The Harvard Library Innovation Lab - * @license MIT - */ -import assert from "assert"; -import fs from "fs"; - -import { validate as uuidValidate } from 'uuid'; - -import { DATA_PATH } from "../const.js"; - -/** - * Utility class for handling access keys to the app. - * [!] For alpha launch only. - */ -export class AccessKeys { - /** - * Complete path to `access-keys.json`. - * @type {string} - */ - static filepath = `${DATA_PATH}access-keys.json`; - - /** - * Frozen hashmap of available access keys - * (app needs to be restarted for new keys to be taken into account, for now). - * @type {object.} - */ - #keys = {}; - - /** - * On init: - * - Create access keys file if it doesn't exist - * - Load keys from file into `this.#keys`. - */ - constructor() { - const filepath = AccessKeys.filepath; - - try { - const keys = fs.readFileSync(filepath); - this.#keys = Object.freeze(JSON.parse(keys)); - } - catch (err) { - fs.writeFileSync(filepath, "{}"); - } - } - - /** - * Checks that a given access key is valid and active. - * @param {string} accessKey - */ - check(accessKey) { - try { - assert(uuidValidate(accessKey)); - assert(this.#keys[accessKey] === true); - return true; - } - catch(err) { - return false; - } - } -} \ No newline at end of file diff --git a/app/utils/IPBlockList.js b/app/utils/IPBlockList.js new file mode 100644 index 0000000..44c807d --- /dev/null +++ b/app/utils/IPBlockList.js @@ -0,0 +1,61 @@ +/** + * thread-keeper + * @module utils.IPBlockList + * @author The Harvard Library Innovation Lab + * @license MIT + */ +import assert from "assert"; +import fs from "fs"; + +import { DATA_PATH } from "../const.js"; + +/** + * Utility class for handling the app's IP block list. + * [!] For alpha launch only. +*/ +export class IPBlockList { + /** + * Complete path to `ip-block-list.json`. + * @type {string} + */ + static filepath = `${DATA_PATH}ip-block-list.json`; + + /** + * Frozen hashmap of IPs that need blocking. + * (app needs to be restarted for new list to be taken into account, for now). + * @type {object.} + */ + #ips = {}; + + /** + * On init: + * - Create access keys file if it doesn't exist + * - Load IPS from file into `this.#ips`. + */ + constructor() { + const filepath = IPBlockList.filepath; + + try { + const ips = fs.readFileSync(filepath); + this.#ips = Object.freeze(JSON.parse(ips)); + } + catch (err) { + fs.writeFileSync(filepath, "{}"); + } + } + + /** + * Checks that a IP is on the blocklist and (still) blocked. + * @param {string} ip + */ + check(ip) { + try { + assert(ip in this.#ips) + assert(this.#ips[ip] === true); + return true; + } + catch(err) { + return false; + } + } + } \ No newline at end of file diff --git a/app/utils/SuccessLog.js b/app/utils/SuccessLog.js index 0bcae13..5ae3a8b 100644 --- a/app/utils/SuccessLog.js +++ b/app/utils/SuccessLog.js @@ -69,15 +69,25 @@ export class SuccessLog { * - Creates a success log entry * - Updates `this.#hashes` (so it doesn't need to reload from file) * - * @param {string} accessKey + * @param {string} ip + * @param {string} why - Reason for creating this archive * @param {Buffer} pdfBytes - Used to store a SHA512 hash of the PDF that was delivered */ - add(accessKey, pdfBytes) { + add(ip, why, pdfBytes) { // Calculate SHA512 hash of the PDF const hash = crypto.createHash('sha512').update(pdfBytes).digest('base64'); + // "why" field sanitization + why = why + .replaceAll("\t", " ") + .replaceAll("\n", " ") + .replaceAll("<", "") + .replaceAll(">", "") + .replaceAll("{", "") + .replaceAll("}", ""); + // Save entry - const entry = `${new Date().toISOString()}\t${accessKey}\tsha512-${hash}\n`; + const entry = `${new Date().toISOString()}\t${ip}\t${why}\tsha512-${hash}\n`; fs.appendFileSync(SuccessLog.filepath, entry); this.#hashes[`sha512-${hash}`] = true; } @@ -108,7 +118,7 @@ export class SuccessLog { * @returns {void} */ reset() { - fs.writeFileSync(SuccessLog.filepath, "date-time\taccess-key\thash\n"); + fs.writeFileSync(SuccessLog.filepath, "date-time\tip\twhy\thash\n"); this.#hashes = {}; } } diff --git a/app/utils/index.js b/app/utils/index.js index 61b1ed9..66d739e 100644 --- a/app/utils/index.js +++ b/app/utils/index.js @@ -4,9 +4,9 @@ * @author The Harvard Library Innovation Lab * @license MIT */ -import { AccessKeys } from "./AccessKeys.js"; +import { IPBlockList } from "./IPBlockList.js"; import { TwitterCapture } from "./TwitterCapture.js"; import { SuccessLog } from "./SuccessLog.js"; import { CertsHistory } from "./CertsHistory.js"; -export { AccessKeys, SuccessLog, CertsHistory, TwitterCapture }; \ No newline at end of file +export { IPBlockList, SuccessLog, CertsHistory, TwitterCapture }; \ No newline at end of file diff --git a/docs/const.md b/docs/const.md index e85e4c7..00e7950 100644 --- a/docs/const.md +++ b/docs/const.md @@ -14,7 +14,7 @@ thread-keeper * [.EXECUTABLES_FOLDER](#module_const.EXECUTABLES_FOLDER) * [.STATIC_PATH](#module_const.STATIC_PATH) * [.MAX_PARALLEL_CAPTURES_TOTAL](#module_const.MAX_PARALLEL_CAPTURES_TOTAL) - * [.MAX_PARALLEL_CAPTURES_PER_ACCESS_KEY](#module_const.MAX_PARALLEL_CAPTURES_PER_ACCESS_KEY) + * [.MAX_PARALLEL_CAPTURES_PER_IP](#module_const.MAX_PARALLEL_CAPTURES_PER_IP) * [.APP_VERSION](#module_const.APP_VERSION) @@ -59,10 +59,10 @@ Path to the "static" folder. Maximum capture processes that can be run in parallel. **Kind**: static constant of [const](#module_const) - + -### const.MAX\_PARALLEL\_CAPTURES\_PER\_ACCESS\_KEY -Maximum capture processes that can be run in parallel for a given key. +### const.MAX\_PARALLEL\_CAPTURES\_PER\_IP +Maximum capture processes that can be run in parallel for a IP address. **Kind**: static constant of [const](#module_const) diff --git a/docs/server.md b/docs/server.md index e660a0f..baeab01 100644 --- a/docs/server.md +++ b/docs/server.md @@ -9,9 +9,10 @@ thread-keeper * [server](#module_server) * _static_ * [.successLog](#module_server.successLog) : SuccessLog + * [.options](#module_server.options) * [.CAPTURES_WATCH](#module_server.CAPTURES_WATCH) : Object * _inner_ - * [~accessKeys](#module_server..accessKeys) : AccessKeys + * [~ipBlockList](#module_server..ipBlockList) : IPBlockList * [~index(request, reply)](#module_server..index) ⇒ Promise.<fastify.FastifyReply> * [~capture(request, reply)](#module_server..capture) ⇒ Promise.<fastify.FastifyReply> * [~check(request, reply)](#module_server..check) ⇒ Promise.<fastify.FastifyReply> @@ -20,6 +21,12 @@ thread-keeper ### server.successLog : SuccessLog +**Kind**: static constant of [server](#module_server) + + +### server.options +Fastify-cli options + **Kind**: static constant of [server](#module_server) @@ -30,9 +37,9 @@ May be used to redirect users if over capacity. [!] Only good for early prototyping. **Kind**: static constant of [server](#module_server) - + -### server~accessKeys : AccessKeys +### server~ipBlockList : IPBlockList **Kind**: inner constant of [server](#module_server) @@ -58,7 +65,6 @@ Returns to form with specific error code, passed as `errorReason`, otherwise. Subject to captures rate limiting (see `CAPTURES_WATCH`). Body is expected as `application/x-www-form-urlencoded` with the following fields: -- access-key - url - unfold-thread (optional) diff --git a/docs/utils/AccessKeys.md b/docs/utils/AccessKeys.md deleted file mode 100644 index 2625302..0000000 --- a/docs/utils/AccessKeys.md +++ /dev/null @@ -1,51 +0,0 @@ - - -## AccessKeys -thread-keeper - -**Author**: The Harvard Library Innovation Lab -**License**: MIT - -* [AccessKeys](#utils.module_AccessKeys) - * [.AccessKeys](#utils.module_AccessKeys.AccessKeys) - * [new exports.AccessKeys()](#new_utils.module_AccessKeys.AccessKeys_new) - * [.filepath](#utils.module_AccessKeys.AccessKeys+filepath) : string - * [.check(accessKey)](#utils.module_AccessKeys.AccessKeys+check) - - - -### AccessKeys.AccessKeys -Utility class for handling access keys to the app. -[!] For alpha launch only. - -**Kind**: static class of [AccessKeys](#utils.module_AccessKeys) - -* [.AccessKeys](#utils.module_AccessKeys.AccessKeys) - * [new exports.AccessKeys()](#new_utils.module_AccessKeys.AccessKeys_new) - * [.filepath](#utils.module_AccessKeys.AccessKeys+filepath) : string - * [.check(accessKey)](#utils.module_AccessKeys.AccessKeys+check) - - - -#### new exports.AccessKeys() -On init: -- Create access keys file if it doesn't exist -- Load keys from file into `this.#keys`. - - - -#### accessKeys.filepath : string -Complete path to `access-keys.json`. - -**Kind**: instance property of [AccessKeys](#utils.module_AccessKeys.AccessKeys) - - -#### accessKeys.check(accessKey) -Checks that a given access key is valid and active. - -**Kind**: instance method of [AccessKeys](#utils.module_AccessKeys.AccessKeys) - -| Param | Type | -| --- | --- | -| accessKey | string | - diff --git a/docs/utils/IPBlockList.md b/docs/utils/IPBlockList.md new file mode 100644 index 0000000..34b2c74 --- /dev/null +++ b/docs/utils/IPBlockList.md @@ -0,0 +1,51 @@ + + +## IPBlockList +thread-keeper + +**Author**: The Harvard Library Innovation Lab +**License**: MIT + +* [IPBlockList](#utils.module_IPBlockList) + * [.IPBlockList](#utils.module_IPBlockList.IPBlockList) + * [new exports.IPBlockList()](#new_utils.module_IPBlockList.IPBlockList_new) + * [.filepath](#utils.module_IPBlockList.IPBlockList+filepath) : string + * [.check(ip)](#utils.module_IPBlockList.IPBlockList+check) + + + +### IPBlockList.IPBlockList +Utility class for handling the app's IP block list. +[!] For alpha launch only. + +**Kind**: static class of [IPBlockList](#utils.module_IPBlockList) + +* [.IPBlockList](#utils.module_IPBlockList.IPBlockList) + * [new exports.IPBlockList()](#new_utils.module_IPBlockList.IPBlockList_new) + * [.filepath](#utils.module_IPBlockList.IPBlockList+filepath) : string + * [.check(ip)](#utils.module_IPBlockList.IPBlockList+check) + + + +#### new exports.IPBlockList() +On init: +- Create access keys file if it doesn't exist +- Load IPS from file into `this.#ips`. + + + +#### ipBlockList.filepath : string +Complete path to `ip-block-list.json`. + +**Kind**: instance property of [IPBlockList](#utils.module_IPBlockList.IPBlockList) + + +#### ipBlockList.check(ip) +Checks that a IP is on the blocklist and (still) blocked. + +**Kind**: instance method of [IPBlockList](#utils.module_IPBlockList.IPBlockList) + +| Param | Type | +| --- | --- | +| ip | string | + diff --git a/docs/utils/SuccessLog.md b/docs/utils/SuccessLog.md index d88fd07..9a9214f 100644 --- a/docs/utils/SuccessLog.md +++ b/docs/utils/SuccessLog.md @@ -10,7 +10,7 @@ thread-keeper * [.SuccessLog](#utils.module_SuccessLog.SuccessLog) * [new exports.SuccessLog()](#new_utils.module_SuccessLog.SuccessLog_new) * [.filepath](#utils.module_SuccessLog.SuccessLog+filepath) : string - * [.add(accessKey, pdfBytes)](#utils.module_SuccessLog.SuccessLog+add) + * [.add(ip, pdfBytes)](#utils.module_SuccessLog.SuccessLog+add) * [.findHashInLogs(hash)](#utils.module_SuccessLog.SuccessLog+findHashInLogs) ⇒ boolean * [.reset()](#utils.module_SuccessLog.SuccessLog+reset) ⇒ void @@ -24,7 +24,7 @@ Utility class for handling success logs. Keeps trace of the hashes of the PDFs t * [.SuccessLog](#utils.module_SuccessLog.SuccessLog) * [new exports.SuccessLog()](#new_utils.module_SuccessLog.SuccessLog_new) * [.filepath](#utils.module_SuccessLog.SuccessLog+filepath) : string - * [.add(accessKey, pdfBytes)](#utils.module_SuccessLog.SuccessLog+add) + * [.add(ip, pdfBytes)](#utils.module_SuccessLog.SuccessLog+add) * [.findHashInLogs(hash)](#utils.module_SuccessLog.SuccessLog+findHashInLogs) ⇒ boolean * [.reset()](#utils.module_SuccessLog.SuccessLog+reset) ⇒ void @@ -43,7 +43,7 @@ Complete path to `success-log.json`. **Kind**: instance property of [SuccessLog](#utils.module_SuccessLog.SuccessLog) -#### successLog.add(accessKey, pdfBytes) +#### successLog.add(ip, pdfBytes) Calculates hash of a PDF an: - Creates a success log entry - Updates `this.#hashes` (so it doesn't need to reload from file) @@ -52,7 +52,7 @@ Calculates hash of a PDF an: | Param | Type | Description | | --- | --- | --- | -| accessKey | string | | +| ip | string | | | pdfBytes | Buffer | Used to store a SHA512 hash of the PDF that was delivered | diff --git a/fixtures/data/access-keys.json b/fixtures/data/access-keys.json deleted file mode 100644 index 321c960..0000000 --- a/fixtures/data/access-keys.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "a9c2b7b6-0652-4207-bdce-0527ad28f3f9": true, - "60e605ec-3510-4358-a0a7-33f25b3d7b74": false -} \ No newline at end of file diff --git a/fixtures/data/ip-block-list.json b/fixtures/data/ip-block-list.json new file mode 100644 index 0000000..3054270 --- /dev/null +++ b/fixtures/data/ip-block-list.json @@ -0,0 +1,4 @@ +{ + "1.2.3.4": true, + "4.3.2.1": false +} \ No newline at end of file diff --git a/package.json b/package.json index eac3acc..74eaed8 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "type": "module", "scripts": { "start": "fastify start app/server.js -l warn", - "dev": "fastify start app/server.js -l info -w", + "dev": "fastify start app/server.js -l info -w -P", "postinstall": "cd scripts && bash download-yt-dlp.sh && bash pip-install.sh", "generate-dev-cert": "cd scripts && bash generate-dev-cert.sh", "docgen": "cd scripts && bash docgen.sh", diff --git a/scripts/docgen.sh b/scripts/docgen.sh index ec206dc..f0b8fef 100755 --- a/scripts/docgen.sh +++ b/scripts/docgen.sh @@ -3,7 +3,7 @@ jsdoc2md ../app/server.js > ../docs/server.md; jsdoc2md ../app/const.js > ../docs/const.md; jsdoc2md ../app/utils/index.js > ../docs/utils/index.md; -jsdoc2md ../app/utils/AccessKeys.js > ../docs/utils/AccessKeys.md; +jsdoc2md ../app/utils/IPBlockList.js > ../docs/utils/IPBlockList.md; jsdoc2md ../app/utils/SuccessLog.js > ../docs/utils/SuccessLog.md; jsdoc2md ../app/utils/TwitterCapture.js > ../docs/utils/TwitterCapture.md; jsdoc2md ../app/utils/CertsHistory.js > ../docs/utils/CertsHistory.md;