kopia lustrzana https://github.com/harvard-lil/archive.social
First implementation of IP rate limiting
Matteo Cargnelutti
rodzic
63e677a3ec
commit
e0c6134119
16
README.md
16
README.md
|
|
@ -59,22 +59,6 @@ npm run generate-dev-cert # Will generate a certificate for self-signing PDFs. F
|
|||
npm run dev # Starts the development server on port 3000
|
||||
```
|
||||
|
||||
### Access keys
|
||||
Create an access key to test with:
|
||||
|
||||
```bash
|
||||
$ uuidgen
|
||||
BB67BBC4-1F4B-4353-8E6D-9927A10F4509
|
||||
```
|
||||
|
||||
And then add the key to `app/data/access-keys.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"BB67BBC4-1F4B-4353-8E6D-9927A10F4509": true
|
||||
}
|
||||
```
|
||||
|
||||
### Certificates history
|
||||
|
||||
The _"Signatures Verification Page"_ page lists the certificates that were used for signing PDFs with the app. You may provide that history by creating two files under `/data`:
|
||||
|
|
|
|||
|
|
@ -46,13 +46,13 @@ export const STATIC_PATH = `${process.env.PWD}/app/static/`;
|
|||
* Maximum capture processes that can be run in parallel.
|
||||
* @constant
|
||||
*/
|
||||
export const MAX_PARALLEL_CAPTURES_TOTAL = 200;
|
||||
export const MAX_PARALLEL_CAPTURES_TOTAL = 50;
|
||||
|
||||
/**
|
||||
* Maximum capture processes that can be run in parallel for a given key.
|
||||
* Maximum capture processes that can be run in parallel for a IP address.
|
||||
* @constant
|
||||
*/
|
||||
export const MAX_PARALLEL_CAPTURES_PER_ACCESS_KEY = 20;
|
||||
export const MAX_PARALLEL_CAPTURES_PER_IP = 2;
|
||||
|
||||
/**
|
||||
* APP version. Pulled from `package.json` by default.
|
||||
|
|
|
|||
|
|
@ -8,24 +8,32 @@ import assert from "assert";
|
|||
|
||||
import nunjucks from "nunjucks";
|
||||
|
||||
import { AccessKeys, CertsHistory, SuccessLog, TwitterCapture } from "./utils/index.js";
|
||||
import { IPBlockList, CertsHistory, SuccessLog, TwitterCapture } from "./utils/index.js";
|
||||
import {
|
||||
TEMPLATES_PATH,
|
||||
STATIC_PATH,
|
||||
MAX_PARALLEL_CAPTURES_TOTAL,
|
||||
MAX_PARALLEL_CAPTURES_PER_ACCESS_KEY,
|
||||
MAX_PARALLEL_CAPTURES_PER_IP,
|
||||
} from "./const.js";
|
||||
|
||||
|
||||
/**
|
||||
* @type {SuccessLog}
|
||||
*/
|
||||
export const successLog = new SuccessLog();
|
||||
|
||||
/**
|
||||
* @type {AccessKeys}
|
||||
* @type {IPBlockList}
|
||||
*/
|
||||
const accessKeys = new AccessKeys();
|
||||
const ipBlockList = new IPBlockList();
|
||||
|
||||
/**
|
||||
* Fastify-cli options
|
||||
* @constant
|
||||
*/
|
||||
export const options = {
|
||||
trustProxy: true,
|
||||
logger: true
|
||||
}
|
||||
|
||||
/**
|
||||
* Keeps track of how many capture processes are currently running.
|
||||
|
|
@ -36,15 +44,15 @@ const accessKeys = new AccessKeys();
|
|||
* @type {{
|
||||
* currentTotal: number,
|
||||
* maxTotal: number,
|
||||
* currentByAccessKey: object.<string, number>,
|
||||
* maxPerAccessKey: number
|
||||
* currentByIp: object.<string, number>,
|
||||
* maxPerIp: number
|
||||
* }}
|
||||
*/
|
||||
export const CAPTURES_WATCH = {
|
||||
currentTotal: 0,
|
||||
maxTotal: MAX_PARALLEL_CAPTURES_TOTAL,
|
||||
currentByAccessKey: {},
|
||||
maxPerAccessKey: MAX_PARALLEL_CAPTURES_PER_ACCESS_KEY,
|
||||
currentByIp: {},
|
||||
maxPerIp: MAX_PARALLEL_CAPTURES_PER_IP,
|
||||
}
|
||||
|
||||
export default async function (fastify, opts) {
|
||||
|
|
@ -67,11 +75,10 @@ export default async function (fastify, opts) {
|
|||
fastify.post('/', capture);
|
||||
|
||||
fastify.get('/check', check);
|
||||
|
||||
|
||||
fastify.get('/api/v1/hashes/check/:hash', checkHash);
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* [GET] /
|
||||
* Shows the landing page and capture form.
|
||||
|
|
@ -98,7 +105,6 @@ async function index(request, reply) {
|
|||
* Subject to captures rate limiting (see `CAPTURES_WATCH`).
|
||||
*
|
||||
* Body is expected as `application/x-www-form-urlencoded` with the following fields:
|
||||
* - access-key
|
||||
* - url
|
||||
* - unfold-thread (optional)
|
||||
*
|
||||
|
|
@ -110,17 +116,18 @@ async function index(request, reply) {
|
|||
*/
|
||||
async function capture(request, reply) {
|
||||
const data = request.body;
|
||||
const accessKey = data["access-key"];
|
||||
|
||||
const ip = request.ip;
|
||||
let why = null;
|
||||
|
||||
request.log.info(`Capture capacity: ${CAPTURES_WATCH.currentTotal} / ${CAPTURES_WATCH.maxTotal}.`);
|
||||
|
||||
|
||||
//
|
||||
// Check access key
|
||||
// Check that IP is not in block list
|
||||
//
|
||||
if (!accessKeys.check(accessKey)) {
|
||||
if (ipBlockList.check(ip)) {
|
||||
const html = nunjucks.render(`${TEMPLATES_PATH}index.njk`, {
|
||||
error: true,
|
||||
errorReason: "ACCESS-KEY"
|
||||
errorReason: "IP"
|
||||
});
|
||||
|
||||
return reply
|
||||
|
|
@ -148,6 +155,25 @@ async function capture(request, reply) {
|
|||
.send(html);
|
||||
}
|
||||
|
||||
//
|
||||
// Check "why" field
|
||||
//
|
||||
try {
|
||||
why = data.why.trim();
|
||||
assert(why.length > 0);
|
||||
}
|
||||
catch(err) {
|
||||
const html = nunjucks.render(`${TEMPLATES_PATH}index.njk`, {
|
||||
error: true,
|
||||
errorReason: "WHY"
|
||||
});
|
||||
|
||||
return reply
|
||||
.code(400)
|
||||
.header('Content-Type', 'text/html; charset=utf-8')
|
||||
.send(html);
|
||||
}
|
||||
|
||||
//
|
||||
// Check that there is still capture capacity (total)
|
||||
//
|
||||
|
|
@ -164,9 +190,9 @@ async function capture(request, reply) {
|
|||
}
|
||||
|
||||
//
|
||||
// Check that there is still capture capacity (for this access key)
|
||||
// Check that there is still capture capacity (for this IP)
|
||||
//
|
||||
if (CAPTURES_WATCH.currentByAccessKey[accessKey] >= CAPTURES_WATCH.maxPerAccessKey) {
|
||||
if (CAPTURES_WATCH.currentByIp[ip] >= CAPTURES_WATCH.maxPerIp) {
|
||||
const html = nunjucks.render(`${TEMPLATES_PATH}index.njk`, {
|
||||
error: true,
|
||||
errorReason: "TOO-MANY-CAPTURES-USER"
|
||||
|
|
@ -182,20 +208,20 @@ async function capture(request, reply) {
|
|||
// Process capture request
|
||||
//
|
||||
try {
|
||||
// Add request to total and per-key counter
|
||||
// Add request to total and per-IP counter
|
||||
CAPTURES_WATCH.currentTotal += 1;
|
||||
|
||||
if (accessKey in CAPTURES_WATCH.currentByAccessKey) {
|
||||
CAPTURES_WATCH.currentByAccessKey[accessKey] += 1;
|
||||
if (ip in CAPTURES_WATCH.currentByIp) {
|
||||
CAPTURES_WATCH.currentByIp[ip] += 1;
|
||||
}
|
||||
else {
|
||||
CAPTURES_WATCH.currentByAccessKey[accessKey] = 1;
|
||||
CAPTURES_WATCH.currentByIp[ip] = 1;
|
||||
}
|
||||
|
||||
const tweets = new TwitterCapture(data.url, {runBrowserBehaviors: "unfold-thread" in data});
|
||||
const pdf = await tweets.capture();
|
||||
|
||||
successLog.add(accessKey, pdf);
|
||||
successLog.add(ip, why, pdf);
|
||||
|
||||
// Generate a filename for the PDF based on url.
|
||||
// Example: harvardlil-status-123456789-2022-11-25.pdf
|
||||
|
|
@ -232,8 +258,8 @@ async function capture(request, reply) {
|
|||
finally {
|
||||
CAPTURES_WATCH.currentTotal -= 1;
|
||||
|
||||
if (accessKey && accessKey in CAPTURES_WATCH.currentByAccessKey) {
|
||||
CAPTURES_WATCH.currentByAccessKey[data["access-key"]] -= 1;
|
||||
if (ip && ip in CAPTURES_WATCH.currentByIp) {
|
||||
CAPTURES_WATCH.currentByIp[ip] -= 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@
|
|||
* @author The Harvard Library Innovation Lab
|
||||
* @license MIT
|
||||
*/
|
||||
import fs from "fs";
|
||||
import assert from "assert";
|
||||
import crypto from "crypto";
|
||||
|
||||
|
|
@ -12,32 +11,19 @@ import { test } from "tap";
|
|||
import Fastify from "fastify";
|
||||
import isHtml from "is-html";
|
||||
|
||||
import { AccessKeys } from "./utils/index.js";
|
||||
|
||||
import server, { CAPTURES_WATCH, successLog } from "./server.js";
|
||||
import { DATA_PATH, CERTS_PATH } from "./const.js";
|
||||
|
||||
/**
|
||||
* Access keys fixture.
|
||||
* @type {{active: string[], inactive: string[]}}
|
||||
*/
|
||||
const ACCESS_KEYS = (() => {
|
||||
const rawAccessKeys = JSON.parse(fs.readFileSync(AccessKeys.filepath));
|
||||
|
||||
const out = { active: [], inactive: [] };
|
||||
|
||||
for (let [key, value] of Object.entries(rawAccessKeys)) {
|
||||
value === true ? out.active.push(key) : out.inactive.push(key);
|
||||
}
|
||||
|
||||
return out;
|
||||
})();
|
||||
|
||||
/**
|
||||
* Dummy url of a thread to capture.
|
||||
*/
|
||||
const THREAD_URL = "https://twitter.com/HarvardLIL/status/1595150565428039680";
|
||||
|
||||
/**
|
||||
* Dummy reason for capture
|
||||
*/
|
||||
const WHY = "Testing thread-keeper";
|
||||
|
||||
test("Integration tests for server.js", async(t) => {
|
||||
|
||||
// Do not run tests if `CERTS_PATH` and `DATA_PATH` do not point to a `fixtures` folder
|
||||
|
|
@ -64,36 +50,42 @@ test("Integration tests for server.js", async(t) => {
|
|||
t.type(isHtml(response.body), true, "Server serves HTML.");
|
||||
});
|
||||
|
||||
test("[POST] / returns HTTP 401 + HTML on failed access key check.", async (t) => {
|
||||
test("[POST] / returns HTTP 401 + HTML on blocked IP check.", async (t) => {
|
||||
const app = Fastify({logger: false});
|
||||
await server(app, {});
|
||||
|
||||
const scenarios = [
|
||||
"FOO-BAR", // Invalid key
|
||||
ACCESS_KEYS.inactive[0], // Inactive key
|
||||
null // No key
|
||||
]
|
||||
const params = new URLSearchParams();
|
||||
params.append("url", THREAD_URL);
|
||||
|
||||
for (const accessKey of scenarios) {
|
||||
const params = new URLSearchParams();
|
||||
const response = await app.inject({
|
||||
method: "POST",
|
||||
url: "/",
|
||||
remoteAddress: "1.2.3.4",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
},
|
||||
body: params.toString()
|
||||
});
|
||||
|
||||
if (accessKey) {
|
||||
params.append("access-key", accessKey);
|
||||
}
|
||||
t.equal(response.statusCode, 401, "Server returns HTTP 401.");
|
||||
|
||||
const response = await app.inject({
|
||||
method: "POST",
|
||||
url: "/",
|
||||
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
||||
body: params.toString(),
|
||||
});
|
||||
const body = `${response.body}`;
|
||||
t.type(isHtml(body), true, "Server serves HTML");
|
||||
t.equal(body.includes(`data-reason="IP"`), true, "With error message.");
|
||||
});
|
||||
|
||||
t.equal(response.statusCode, 401, "Server returns HTTP 401.");
|
||||
test("[POST] / lets IPs that are no longer blocked make requests.", async (t) => {
|
||||
const app = Fastify({logger: false});
|
||||
await server(app, {});
|
||||
|
||||
const body = `${response.body}`;
|
||||
t.type(isHtml(body), true, "Server serves HTML");
|
||||
t.equal(body.includes(`data-reason="ACCESS-KEY"`), true, "With error message.");
|
||||
}
|
||||
const response = await app.inject({
|
||||
method: "POST",
|
||||
url: "/",
|
||||
remoteAddress: "4.3.2.1",
|
||||
});
|
||||
|
||||
// Should fail because no URL were passed, not because IP was blocked
|
||||
t.equal(response.statusCode, 400, "Server returns HTTP 400.");
|
||||
});
|
||||
|
||||
test("[POST] / returns HTTP 400 + HTML on failed url check.", async (t) => {
|
||||
|
|
@ -110,8 +102,6 @@ test("Integration tests for server.js", async(t) => {
|
|||
for (const url of scenarios) {
|
||||
const params = new URLSearchParams();
|
||||
|
||||
params.append("access-key", ACCESS_KEYS.active[0]);
|
||||
|
||||
if (url) {
|
||||
params.append("url", url);
|
||||
}
|
||||
|
|
@ -123,7 +113,7 @@ test("Integration tests for server.js", async(t) => {
|
|||
body: params.toString(),
|
||||
});
|
||||
|
||||
t.equal(response.statusCode, 400, "Server returns HTTP 401.");
|
||||
t.equal(response.statusCode, 400, "Server returns HTTP 400.");
|
||||
|
||||
const body = `${response.body}`;
|
||||
t.type(isHtml(body), true, "Server serves HTML");
|
||||
|
|
@ -131,6 +121,40 @@ test("Integration tests for server.js", async(t) => {
|
|||
}
|
||||
});
|
||||
|
||||
test("[POST] / returns HTTP 400 + HTML on failed \"why\" check.", async (t) => {
|
||||
const app = Fastify({logger: false});
|
||||
await server(app, {});
|
||||
|
||||
const scenarios = [
|
||||
null, // Nothing
|
||||
" ", // Empty
|
||||
"",
|
||||
]
|
||||
|
||||
for (const why of scenarios) {
|
||||
const params = new URLSearchParams();
|
||||
|
||||
params.append("url", THREAD_URL);
|
||||
|
||||
if (why) {
|
||||
params.append("why", why);
|
||||
}
|
||||
|
||||
const response = await app.inject({
|
||||
method: "POST",
|
||||
url: "/",
|
||||
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
||||
body: params.toString(),
|
||||
});
|
||||
|
||||
t.equal(response.statusCode, 400, "Server returns HTTP 400.");
|
||||
|
||||
const body = `${response.body}`;
|
||||
t.type(isHtml(body), true, "Server serves HTML");
|
||||
t.equal(body.includes(`data-reason="WHY"`), true, "With error message.");
|
||||
}
|
||||
});
|
||||
|
||||
test("[POST] / returns HTTP 503 + HTML on failed server capacity check.", async (t) => {
|
||||
const app = Fastify({logger: false});
|
||||
await server(app, {});
|
||||
|
|
@ -138,8 +162,8 @@ test("Integration tests for server.js", async(t) => {
|
|||
CAPTURES_WATCH.currentTotal = CAPTURES_WATCH.maxTotal; // Simulate peak
|
||||
|
||||
const params = new URLSearchParams();
|
||||
params.append("access-key", ACCESS_KEYS.active[0]);
|
||||
params.append("url", THREAD_URL);
|
||||
params.append("why", WHY);
|
||||
|
||||
const response = await app.inject({
|
||||
method: "POST",
|
||||
|
|
@ -161,19 +185,18 @@ test("Integration tests for server.js", async(t) => {
|
|||
const app = Fastify({logger: false});
|
||||
await server(app, {});
|
||||
|
||||
const userKey = ACCESS_KEYS.active[0];
|
||||
|
||||
CAPTURES_WATCH.currentByAccessKey[userKey] = CAPTURES_WATCH.maxPerAccessKey;
|
||||
const userIp = "127.0.0.1";
|
||||
CAPTURES_WATCH.currentByIp[userIp] = CAPTURES_WATCH.maxPerIp;
|
||||
|
||||
const params = new URLSearchParams();
|
||||
params.append("access-key", userKey);
|
||||
params.append("url", THREAD_URL);
|
||||
params.append("why", WHY);
|
||||
|
||||
const response = await app.inject({
|
||||
method: "POST",
|
||||
url: "/",
|
||||
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
||||
body: params.toString(),
|
||||
body: params.toString()
|
||||
});
|
||||
|
||||
t.equal(response.statusCode, 429, "Server returns HTTP 503.");
|
||||
|
|
@ -182,7 +205,7 @@ test("Integration tests for server.js", async(t) => {
|
|||
t.type(isHtml(body), true, "Server serves HTML");
|
||||
t.equal(body.includes(`TOO-MANY-CAPTURES-USER`), true, "With error message.");
|
||||
|
||||
delete CAPTURES_WATCH.currentByAccessKey[userKey];
|
||||
delete CAPTURES_WATCH.currentByIp[userIp];
|
||||
});
|
||||
|
||||
test("[POST] / returns HTTP 200 + PDF", async (t) => {
|
||||
|
|
@ -190,8 +213,8 @@ test("Integration tests for server.js", async(t) => {
|
|||
await server(app, {});
|
||||
|
||||
const params = new URLSearchParams();
|
||||
params.append("access-key", ACCESS_KEYS.active[0]);
|
||||
params.append("url", THREAD_URL);
|
||||
params.append("why", WHY);
|
||||
params.append("unfold-thread", "on");
|
||||
|
||||
const response = await app.inject({
|
||||
|
|
@ -246,7 +269,7 @@ test("Integration tests for server.js", async(t) => {
|
|||
// Add entry to success logs
|
||||
const toHash = Buffer.from(`${Date.now()}`);
|
||||
const hash = crypto.createHash('sha512').update(toHash).digest('base64');
|
||||
successLog.add(ACCESS_KEYS.active[0], toHash);
|
||||
successLog.add("127.0.0.1", WHY, toHash);
|
||||
|
||||
const response = await app.inject({
|
||||
method: "GET",
|
||||
|
|
|
|||
|
|
@ -266,7 +266,7 @@ body#index > main form label {
|
|||
padding-bottom: 0.25rem;
|
||||
}
|
||||
|
||||
body#index > main form input {
|
||||
body#index > main form input, body#index > main form textarea {
|
||||
display: block;
|
||||
width: 100%;
|
||||
padding: 0.5rem;
|
||||
|
|
|
|||
|
|
@ -7,15 +7,15 @@ const formSubmitDialog = document.querySelector("dialog#form-submit");
|
|||
document.querySelector("body#index form button").addEventListener("click", (e) => {
|
||||
e.preventDefault();
|
||||
const url = document.querySelector("body#index form input[name='url']");
|
||||
const accessKey = document.querySelector("body#index form input[name='access-key']")
|
||||
const why = document.querySelector("body#index form input[name='why']");
|
||||
|
||||
if (!url.checkValidity()) {
|
||||
url.reportValidity();
|
||||
return;
|
||||
}
|
||||
|
||||
if (!accessKey.checkValidity()) {
|
||||
accessKey.reportValidity();
|
||||
if (!why.checkValidity()) {
|
||||
why.reportValidity();
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -34,11 +34,8 @@
|
|||
</fieldset>
|
||||
|
||||
<fieldset>
|
||||
<label for="access-key">Access key <a href="https://ocb.to/archive-social-form" title="Access key request form">(request access)</a></label>
|
||||
<input type="password"
|
||||
name="access-key"
|
||||
id="access-key"
|
||||
required>
|
||||
<label for="why">Reason for archiving <a href="#who-can-use-it">(why this question?)</a></label>
|
||||
<textarea required name="why" id="why" rows="2"></textarea>
|
||||
</fieldset>
|
||||
|
||||
<fieldset class="submit">
|
||||
|
|
@ -66,20 +63,24 @@
|
|||
<dialog id="form-error" data-reason="{{ errorReason }}">
|
||||
<h2>Something went wrong</h2>
|
||||
|
||||
{% if errorReason and errorReason == "ACCESS-KEY" %}
|
||||
<p>The access key provided is invalid or no longer active.</p>
|
||||
{% if errorReason and errorReason == "IP" %}
|
||||
<p>Your access to this service has been restricted.</p>
|
||||
{% endif %}
|
||||
|
||||
{% if errorReason and errorReason == "URL" %}
|
||||
<p>The url provided is not a valid twitter.com url.</p>
|
||||
{% endif %}
|
||||
|
||||
{% if errorReason and errorReason == "WHY" %}
|
||||
<p>The url provided is not a valid twitter.com url.</p>
|
||||
{% endif %}
|
||||
|
||||
{% if errorReason and errorReason == "TOO-MANY-CAPTURES-TOTAL" %}
|
||||
<p>Too many requests. Please retry in a minute.</p>
|
||||
{% endif %}
|
||||
|
||||
{% if errorReason and errorReason == "TOO-MANY-CAPTURES-USER" %}
|
||||
<p>Please wait until the capture requests you've started are completed before starting a new one.</p>
|
||||
<p>Too many requests for your IP address. Please retry in a minute.</p>
|
||||
{% endif %}
|
||||
|
||||
{% if errorReason and errorReason == "CAPTURE-ISSUE" %}
|
||||
|
|
@ -95,9 +96,9 @@
|
|||
|
||||
<p>This site is an experiment by the <a href="https://lil.law.harvard.edu">Harvard Library Innovation Lab</a> to let you download signed PDFs of Twitter URLs. <a href="/static/example.pdf">Here's an example PDF</a> we made from <a href="https://twitter.com/doctorow/status/1591759999323492358">this tweet</a>.</p>
|
||||
|
||||
<h2>Who can use it?</h2>
|
||||
<h2 id="who-can-use-it">Who can use it?</h2>
|
||||
|
||||
<p>To use our website <a href="https://ocb.to/archive-social-form">you'll need to contact us</a> for an API key. We're currently only able to share a limited number with people like journalists, internet scholars, and archivists. But you can also use <a href="https://github.com/harvard-lil/thread-keeper">our open source software</a> to stand up an archive server of your own, and share it with your friends.</p>
|
||||
<p>While anyone can use this website, it is experimental and running with limited server capacity. But you can also use <a href="https://github.com/harvard-lil/thread-keeper">our open source software</a> to stand up an archive server of your own, and share it with your friends.</p>
|
||||
|
||||
<h2>Why make a PDF archiving tool for Twitter?</h2>
|
||||
|
||||
|
|
|
|||
|
|
@ -1,63 +0,0 @@
|
|||
/**
|
||||
* thread-keeper
|
||||
* @module utils.AccessKeys
|
||||
* @author The Harvard Library Innovation Lab
|
||||
* @license MIT
|
||||
*/
|
||||
import assert from "assert";
|
||||
import fs from "fs";
|
||||
|
||||
import { validate as uuidValidate } from 'uuid';
|
||||
|
||||
import { DATA_PATH } from "../const.js";
|
||||
|
||||
/**
|
||||
* Utility class for handling access keys to the app.
|
||||
* [!] For alpha launch only.
|
||||
*/
|
||||
export class AccessKeys {
|
||||
/**
|
||||
* Complete path to `access-keys.json`.
|
||||
* @type {string}
|
||||
*/
|
||||
static filepath = `${DATA_PATH}access-keys.json`;
|
||||
|
||||
/**
|
||||
* Frozen hashmap of available access keys
|
||||
* (app needs to be restarted for new keys to be taken into account, for now).
|
||||
* @type {object.<string,boolean>}
|
||||
*/
|
||||
#keys = {};
|
||||
|
||||
/**
|
||||
* On init:
|
||||
* - Create access keys file if it doesn't exist
|
||||
* - Load keys from file into `this.#keys`.
|
||||
*/
|
||||
constructor() {
|
||||
const filepath = AccessKeys.filepath;
|
||||
|
||||
try {
|
||||
const keys = fs.readFileSync(filepath);
|
||||
this.#keys = Object.freeze(JSON.parse(keys));
|
||||
}
|
||||
catch (err) {
|
||||
fs.writeFileSync(filepath, "{}");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks that a given access key is valid and active.
|
||||
* @param {string} accessKey
|
||||
*/
|
||||
check(accessKey) {
|
||||
try {
|
||||
assert(uuidValidate(accessKey));
|
||||
assert(this.#keys[accessKey] === true);
|
||||
return true;
|
||||
}
|
||||
catch(err) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
/**
|
||||
* thread-keeper
|
||||
* @module utils.IPBlockList
|
||||
* @author The Harvard Library Innovation Lab
|
||||
* @license MIT
|
||||
*/
|
||||
import assert from "assert";
|
||||
import fs from "fs";
|
||||
|
||||
import { DATA_PATH } from "../const.js";
|
||||
|
||||
/**
|
||||
* Utility class for handling the app's IP block list.
|
||||
* [!] For alpha launch only.
|
||||
*/
|
||||
export class IPBlockList {
|
||||
/**
|
||||
* Complete path to `ip-block-list.json`.
|
||||
* @type {string}
|
||||
*/
|
||||
static filepath = `${DATA_PATH}ip-block-list.json`;
|
||||
|
||||
/**
|
||||
* Frozen hashmap of IPs that need blocking.
|
||||
* (app needs to be restarted for new list to be taken into account, for now).
|
||||
* @type {object.<string,boolean>}
|
||||
*/
|
||||
#ips = {};
|
||||
|
||||
/**
|
||||
* On init:
|
||||
* - Create access keys file if it doesn't exist
|
||||
* - Load IPS from file into `this.#ips`.
|
||||
*/
|
||||
constructor() {
|
||||
const filepath = IPBlockList.filepath;
|
||||
|
||||
try {
|
||||
const ips = fs.readFileSync(filepath);
|
||||
this.#ips = Object.freeze(JSON.parse(ips));
|
||||
}
|
||||
catch (err) {
|
||||
fs.writeFileSync(filepath, "{}");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks that a IP is on the blocklist and (still) blocked.
|
||||
* @param {string} ip
|
||||
*/
|
||||
check(ip) {
|
||||
try {
|
||||
assert(ip in this.#ips)
|
||||
assert(this.#ips[ip] === true);
|
||||
return true;
|
||||
}
|
||||
catch(err) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -69,15 +69,25 @@ export class SuccessLog {
|
|||
* - Creates a success log entry
|
||||
* - Updates `this.#hashes` (so it doesn't need to reload from file)
|
||||
*
|
||||
* @param {string} accessKey
|
||||
* @param {string} ip
|
||||
* @param {string} why - Reason for creating this archive
|
||||
* @param {Buffer} pdfBytes - Used to store a SHA512 hash of the PDF that was delivered
|
||||
*/
|
||||
add(accessKey, pdfBytes) {
|
||||
add(ip, why, pdfBytes) {
|
||||
// Calculate SHA512 hash of the PDF
|
||||
const hash = crypto.createHash('sha512').update(pdfBytes).digest('base64');
|
||||
|
||||
// "why" field sanitization
|
||||
why = why
|
||||
.replaceAll("\t", " ")
|
||||
.replaceAll("\n", " ")
|
||||
.replaceAll("<", "")
|
||||
.replaceAll(">", "")
|
||||
.replaceAll("{", "")
|
||||
.replaceAll("}", "");
|
||||
|
||||
// Save entry
|
||||
const entry = `${new Date().toISOString()}\t${accessKey}\tsha512-${hash}\n`;
|
||||
const entry = `${new Date().toISOString()}\t${ip}\t${why}\tsha512-${hash}\n`;
|
||||
fs.appendFileSync(SuccessLog.filepath, entry);
|
||||
this.#hashes[`sha512-${hash}`] = true;
|
||||
}
|
||||
|
|
@ -108,7 +118,7 @@ export class SuccessLog {
|
|||
* @returns {void}
|
||||
*/
|
||||
reset() {
|
||||
fs.writeFileSync(SuccessLog.filepath, "date-time\taccess-key\thash\n");
|
||||
fs.writeFileSync(SuccessLog.filepath, "date-time\tip\twhy\thash\n");
|
||||
this.#hashes = {};
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,9 +4,9 @@
|
|||
* @author The Harvard Library Innovation Lab
|
||||
* @license MIT
|
||||
*/
|
||||
import { AccessKeys } from "./AccessKeys.js";
|
||||
import { IPBlockList } from "./IPBlockList.js";
|
||||
import { TwitterCapture } from "./TwitterCapture.js";
|
||||
import { SuccessLog } from "./SuccessLog.js";
|
||||
import { CertsHistory } from "./CertsHistory.js";
|
||||
|
||||
export { AccessKeys, SuccessLog, CertsHistory, TwitterCapture };
|
||||
export { IPBlockList, SuccessLog, CertsHistory, TwitterCapture };
|
||||
|
|
@ -14,7 +14,7 @@ thread-keeper
|
|||
* [.EXECUTABLES_FOLDER](#module_const.EXECUTABLES_FOLDER)
|
||||
* [.STATIC_PATH](#module_const.STATIC_PATH)
|
||||
* [.MAX_PARALLEL_CAPTURES_TOTAL](#module_const.MAX_PARALLEL_CAPTURES_TOTAL)
|
||||
* [.MAX_PARALLEL_CAPTURES_PER_ACCESS_KEY](#module_const.MAX_PARALLEL_CAPTURES_PER_ACCESS_KEY)
|
||||
* [.MAX_PARALLEL_CAPTURES_PER_IP](#module_const.MAX_PARALLEL_CAPTURES_PER_IP)
|
||||
* [.APP_VERSION](#module_const.APP_VERSION)
|
||||
|
||||
<a name="module_const.CERTS_PATH"></a>
|
||||
|
|
@ -59,10 +59,10 @@ Path to the "static" folder.
|
|||
Maximum capture processes that can be run in parallel.
|
||||
|
||||
**Kind**: static constant of [<code>const</code>](#module_const)
|
||||
<a name="module_const.MAX_PARALLEL_CAPTURES_PER_ACCESS_KEY"></a>
|
||||
<a name="module_const.MAX_PARALLEL_CAPTURES_PER_IP"></a>
|
||||
|
||||
### const.MAX\_PARALLEL\_CAPTURES\_PER\_ACCESS\_KEY
|
||||
Maximum capture processes that can be run in parallel for a given key.
|
||||
### const.MAX\_PARALLEL\_CAPTURES\_PER\_IP
|
||||
Maximum capture processes that can be run in parallel for a IP address.
|
||||
|
||||
**Kind**: static constant of [<code>const</code>](#module_const)
|
||||
<a name="module_const.APP_VERSION"></a>
|
||||
|
|
|
|||
|
|
@ -9,9 +9,10 @@ thread-keeper
|
|||
* [server](#module_server)
|
||||
* _static_
|
||||
* [.successLog](#module_server.successLog) : <code>SuccessLog</code>
|
||||
* [.options](#module_server.options)
|
||||
* [.CAPTURES_WATCH](#module_server.CAPTURES_WATCH) : <code>Object</code>
|
||||
* _inner_
|
||||
* [~accessKeys](#module_server..accessKeys) : <code>AccessKeys</code>
|
||||
* [~ipBlockList](#module_server..ipBlockList) : <code>IPBlockList</code>
|
||||
* [~index(request, reply)](#module_server..index) ⇒ <code>Promise.<fastify.FastifyReply></code>
|
||||
* [~capture(request, reply)](#module_server..capture) ⇒ <code>Promise.<fastify.FastifyReply></code>
|
||||
* [~check(request, reply)](#module_server..check) ⇒ <code>Promise.<fastify.FastifyReply></code>
|
||||
|
|
@ -20,6 +21,12 @@ thread-keeper
|
|||
<a name="module_server.successLog"></a>
|
||||
|
||||
### server.successLog : <code>SuccessLog</code>
|
||||
**Kind**: static constant of [<code>server</code>](#module_server)
|
||||
<a name="module_server.options"></a>
|
||||
|
||||
### server.options
|
||||
Fastify-cli options
|
||||
|
||||
**Kind**: static constant of [<code>server</code>](#module_server)
|
||||
<a name="module_server.CAPTURES_WATCH"></a>
|
||||
|
||||
|
|
@ -30,9 +37,9 @@ May be used to redirect users if over capacity.
|
|||
[!] Only good for early prototyping.
|
||||
|
||||
**Kind**: static constant of [<code>server</code>](#module_server)
|
||||
<a name="module_server..accessKeys"></a>
|
||||
<a name="module_server..ipBlockList"></a>
|
||||
|
||||
### server~accessKeys : <code>AccessKeys</code>
|
||||
### server~ipBlockList : <code>IPBlockList</code>
|
||||
**Kind**: inner constant of [<code>server</code>](#module_server)
|
||||
<a name="module_server..index"></a>
|
||||
|
||||
|
|
@ -58,7 +65,6 @@ Returns to form with specific error code, passed as `errorReason`, otherwise.
|
|||
Subject to captures rate limiting (see `CAPTURES_WATCH`).
|
||||
|
||||
Body is expected as `application/x-www-form-urlencoded` with the following fields:
|
||||
- access-key
|
||||
- url
|
||||
- unfold-thread (optional)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,51 +0,0 @@
|
|||
<a name="utils.module_AccessKeys"></a>
|
||||
|
||||
## AccessKeys
|
||||
thread-keeper
|
||||
|
||||
**Author**: The Harvard Library Innovation Lab
|
||||
**License**: MIT
|
||||
|
||||
* [AccessKeys](#utils.module_AccessKeys)
|
||||
* [.AccessKeys](#utils.module_AccessKeys.AccessKeys)
|
||||
* [new exports.AccessKeys()](#new_utils.module_AccessKeys.AccessKeys_new)
|
||||
* [.filepath](#utils.module_AccessKeys.AccessKeys+filepath) : <code>string</code>
|
||||
* [.check(accessKey)](#utils.module_AccessKeys.AccessKeys+check)
|
||||
|
||||
<a name="utils.module_AccessKeys.AccessKeys"></a>
|
||||
|
||||
### AccessKeys.AccessKeys
|
||||
Utility class for handling access keys to the app.
|
||||
[!] For alpha launch only.
|
||||
|
||||
**Kind**: static class of [<code>AccessKeys</code>](#utils.module_AccessKeys)
|
||||
|
||||
* [.AccessKeys](#utils.module_AccessKeys.AccessKeys)
|
||||
* [new exports.AccessKeys()](#new_utils.module_AccessKeys.AccessKeys_new)
|
||||
* [.filepath](#utils.module_AccessKeys.AccessKeys+filepath) : <code>string</code>
|
||||
* [.check(accessKey)](#utils.module_AccessKeys.AccessKeys+check)
|
||||
|
||||
<a name="new_utils.module_AccessKeys.AccessKeys_new"></a>
|
||||
|
||||
#### new exports.AccessKeys()
|
||||
On init:
|
||||
- Create access keys file if it doesn't exist
|
||||
- Load keys from file into `this.#keys`.
|
||||
|
||||
<a name="utils.module_AccessKeys.AccessKeys+filepath"></a>
|
||||
|
||||
#### accessKeys.filepath : <code>string</code>
|
||||
Complete path to `access-keys.json`.
|
||||
|
||||
**Kind**: instance property of [<code>AccessKeys</code>](#utils.module_AccessKeys.AccessKeys)
|
||||
<a name="utils.module_AccessKeys.AccessKeys+check"></a>
|
||||
|
||||
#### accessKeys.check(accessKey)
|
||||
Checks that a given access key is valid and active.
|
||||
|
||||
**Kind**: instance method of [<code>AccessKeys</code>](#utils.module_AccessKeys.AccessKeys)
|
||||
|
||||
| Param | Type |
|
||||
| --- | --- |
|
||||
| accessKey | <code>string</code> |
|
||||
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
<a name="utils.module_IPBlockList"></a>
|
||||
|
||||
## IPBlockList
|
||||
thread-keeper
|
||||
|
||||
**Author**: The Harvard Library Innovation Lab
|
||||
**License**: MIT
|
||||
|
||||
* [IPBlockList](#utils.module_IPBlockList)
|
||||
* [.IPBlockList](#utils.module_IPBlockList.IPBlockList)
|
||||
* [new exports.IPBlockList()](#new_utils.module_IPBlockList.IPBlockList_new)
|
||||
* [.filepath](#utils.module_IPBlockList.IPBlockList+filepath) : <code>string</code>
|
||||
* [.check(ip)](#utils.module_IPBlockList.IPBlockList+check)
|
||||
|
||||
<a name="utils.module_IPBlockList.IPBlockList"></a>
|
||||
|
||||
### IPBlockList.IPBlockList
|
||||
Utility class for handling the app's IP block list.
|
||||
[!] For alpha launch only.
|
||||
|
||||
**Kind**: static class of [<code>IPBlockList</code>](#utils.module_IPBlockList)
|
||||
|
||||
* [.IPBlockList](#utils.module_IPBlockList.IPBlockList)
|
||||
* [new exports.IPBlockList()](#new_utils.module_IPBlockList.IPBlockList_new)
|
||||
* [.filepath](#utils.module_IPBlockList.IPBlockList+filepath) : <code>string</code>
|
||||
* [.check(ip)](#utils.module_IPBlockList.IPBlockList+check)
|
||||
|
||||
<a name="new_utils.module_IPBlockList.IPBlockList_new"></a>
|
||||
|
||||
#### new exports.IPBlockList()
|
||||
On init:
|
||||
- Create access keys file if it doesn't exist
|
||||
- Load IPS from file into `this.#ips`.
|
||||
|
||||
<a name="utils.module_IPBlockList.IPBlockList+filepath"></a>
|
||||
|
||||
#### ipBlockList.filepath : <code>string</code>
|
||||
Complete path to `ip-block-list.json`.
|
||||
|
||||
**Kind**: instance property of [<code>IPBlockList</code>](#utils.module_IPBlockList.IPBlockList)
|
||||
<a name="utils.module_IPBlockList.IPBlockList+check"></a>
|
||||
|
||||
#### ipBlockList.check(ip)
|
||||
Checks that a IP is on the blocklist and (still) blocked.
|
||||
|
||||
**Kind**: instance method of [<code>IPBlockList</code>](#utils.module_IPBlockList.IPBlockList)
|
||||
|
||||
| Param | Type |
|
||||
| --- | --- |
|
||||
| ip | <code>string</code> |
|
||||
|
||||
|
|
@ -10,7 +10,7 @@ thread-keeper
|
|||
* [.SuccessLog](#utils.module_SuccessLog.SuccessLog)
|
||||
* [new exports.SuccessLog()](#new_utils.module_SuccessLog.SuccessLog_new)
|
||||
* [.filepath](#utils.module_SuccessLog.SuccessLog+filepath) : <code>string</code>
|
||||
* [.add(accessKey, pdfBytes)](#utils.module_SuccessLog.SuccessLog+add)
|
||||
* [.add(ip, pdfBytes)](#utils.module_SuccessLog.SuccessLog+add)
|
||||
* [.findHashInLogs(hash)](#utils.module_SuccessLog.SuccessLog+findHashInLogs) ⇒ <code>boolean</code>
|
||||
* [.reset()](#utils.module_SuccessLog.SuccessLog+reset) ⇒ <code>void</code>
|
||||
|
||||
|
|
@ -24,7 +24,7 @@ Utility class for handling success logs. Keeps trace of the hashes of the PDFs t
|
|||
* [.SuccessLog](#utils.module_SuccessLog.SuccessLog)
|
||||
* [new exports.SuccessLog()](#new_utils.module_SuccessLog.SuccessLog_new)
|
||||
* [.filepath](#utils.module_SuccessLog.SuccessLog+filepath) : <code>string</code>
|
||||
* [.add(accessKey, pdfBytes)](#utils.module_SuccessLog.SuccessLog+add)
|
||||
* [.add(ip, pdfBytes)](#utils.module_SuccessLog.SuccessLog+add)
|
||||
* [.findHashInLogs(hash)](#utils.module_SuccessLog.SuccessLog+findHashInLogs) ⇒ <code>boolean</code>
|
||||
* [.reset()](#utils.module_SuccessLog.SuccessLog+reset) ⇒ <code>void</code>
|
||||
|
||||
|
|
@ -43,7 +43,7 @@ Complete path to `success-log.json`.
|
|||
**Kind**: instance property of [<code>SuccessLog</code>](#utils.module_SuccessLog.SuccessLog)
|
||||
<a name="utils.module_SuccessLog.SuccessLog+add"></a>
|
||||
|
||||
#### successLog.add(accessKey, pdfBytes)
|
||||
#### successLog.add(ip, pdfBytes)
|
||||
Calculates hash of a PDF an:
|
||||
- Creates a success log entry
|
||||
- Updates `this.#hashes` (so it doesn't need to reload from file)
|
||||
|
|
@ -52,7 +52,7 @@ Calculates hash of a PDF an:
|
|||
|
||||
| Param | Type | Description |
|
||||
| --- | --- | --- |
|
||||
| accessKey | <code>string</code> | |
|
||||
| ip | <code>string</code> | |
|
||||
| pdfBytes | <code>Buffer</code> | Used to store a SHA512 hash of the PDF that was delivered |
|
||||
|
||||
<a name="utils.module_SuccessLog.SuccessLog+findHashInLogs"></a>
|
||||
|
|
|
|||
|
|
@ -1,4 +0,0 @@
|
|||
{
|
||||
"a9c2b7b6-0652-4207-bdce-0527ad28f3f9": true,
|
||||
"60e605ec-3510-4358-a0a7-33f25b3d7b74": false
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"1.2.3.4": true,
|
||||
"4.3.2.1": false
|
||||
}
|
||||
|
|
@ -6,7 +6,7 @@
|
|||
"type": "module",
|
||||
"scripts": {
|
||||
"start": "fastify start app/server.js -l warn",
|
||||
"dev": "fastify start app/server.js -l info -w",
|
||||
"dev": "fastify start app/server.js -l info -w -P",
|
||||
"postinstall": "cd scripts && bash download-yt-dlp.sh && bash pip-install.sh",
|
||||
"generate-dev-cert": "cd scripts && bash generate-dev-cert.sh",
|
||||
"docgen": "cd scripts && bash docgen.sh",
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ jsdoc2md ../app/server.js > ../docs/server.md;
|
|||
jsdoc2md ../app/const.js > ../docs/const.md;
|
||||
|
||||
jsdoc2md ../app/utils/index.js > ../docs/utils/index.md;
|
||||
jsdoc2md ../app/utils/AccessKeys.js > ../docs/utils/AccessKeys.md;
|
||||
jsdoc2md ../app/utils/IPBlockList.js > ../docs/utils/IPBlockList.md;
|
||||
jsdoc2md ../app/utils/SuccessLog.js > ../docs/utils/SuccessLog.md;
|
||||
jsdoc2md ../app/utils/TwitterCapture.js > ../docs/utils/TwitterCapture.md;
|
||||
jsdoc2md ../app/utils/CertsHistory.js > ../docs/utils/CertsHistory.md;
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue