kopia lustrzana https://github.com/harvard-lil/archive.social
First implementation of IP rate limiting
rodzic
63e677a3ec
commit
e0c6134119
16
README.md
16
README.md
|
@ -59,22 +59,6 @@ npm run generate-dev-cert # Will generate a certificate for self-signing PDFs. F
|
||||||
npm run dev # Starts the development server on port 3000
|
npm run dev # Starts the development server on port 3000
|
||||||
```
|
```
|
||||||
|
|
||||||
### Access keys
|
|
||||||
Create an access key to test with:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
$ uuidgen
|
|
||||||
BB67BBC4-1F4B-4353-8E6D-9927A10F4509
|
|
||||||
```
|
|
||||||
|
|
||||||
And then add the key to `app/data/access-keys.json`:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"BB67BBC4-1F4B-4353-8E6D-9927A10F4509": true
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Certificates history
|
### Certificates history
|
||||||
|
|
||||||
The _"Signatures Verification Page"_ page lists the certificates that were used for signing PDFs with the app. You may provide that history by creating two files under `/data`:
|
The _"Signatures Verification Page"_ page lists the certificates that were used for signing PDFs with the app. You may provide that history by creating two files under `/data`:
|
||||||
|
|
|
@ -46,13 +46,13 @@ export const STATIC_PATH = `${process.env.PWD}/app/static/`;
|
||||||
* Maximum capture processes that can be run in parallel.
|
* Maximum capture processes that can be run in parallel.
|
||||||
* @constant
|
* @constant
|
||||||
*/
|
*/
|
||||||
export const MAX_PARALLEL_CAPTURES_TOTAL = 200;
|
export const MAX_PARALLEL_CAPTURES_TOTAL = 50;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Maximum capture processes that can be run in parallel for a given key.
|
* Maximum capture processes that can be run in parallel for a IP address.
|
||||||
* @constant
|
* @constant
|
||||||
*/
|
*/
|
||||||
export const MAX_PARALLEL_CAPTURES_PER_ACCESS_KEY = 20;
|
export const MAX_PARALLEL_CAPTURES_PER_IP = 2;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* APP version. Pulled from `package.json` by default.
|
* APP version. Pulled from `package.json` by default.
|
||||||
|
|
|
@ -8,24 +8,32 @@ import assert from "assert";
|
||||||
|
|
||||||
import nunjucks from "nunjucks";
|
import nunjucks from "nunjucks";
|
||||||
|
|
||||||
import { AccessKeys, CertsHistory, SuccessLog, TwitterCapture } from "./utils/index.js";
|
import { IPBlockList, CertsHistory, SuccessLog, TwitterCapture } from "./utils/index.js";
|
||||||
import {
|
import {
|
||||||
TEMPLATES_PATH,
|
TEMPLATES_PATH,
|
||||||
STATIC_PATH,
|
STATIC_PATH,
|
||||||
MAX_PARALLEL_CAPTURES_TOTAL,
|
MAX_PARALLEL_CAPTURES_TOTAL,
|
||||||
MAX_PARALLEL_CAPTURES_PER_ACCESS_KEY,
|
MAX_PARALLEL_CAPTURES_PER_IP,
|
||||||
} from "./const.js";
|
} from "./const.js";
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @type {SuccessLog}
|
* @type {SuccessLog}
|
||||||
*/
|
*/
|
||||||
export const successLog = new SuccessLog();
|
export const successLog = new SuccessLog();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @type {AccessKeys}
|
* @type {IPBlockList}
|
||||||
*/
|
*/
|
||||||
const accessKeys = new AccessKeys();
|
const ipBlockList = new IPBlockList();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fastify-cli options
|
||||||
|
* @constant
|
||||||
|
*/
|
||||||
|
export const options = {
|
||||||
|
trustProxy: true,
|
||||||
|
logger: true
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Keeps track of how many capture processes are currently running.
|
* Keeps track of how many capture processes are currently running.
|
||||||
|
@ -36,15 +44,15 @@ const accessKeys = new AccessKeys();
|
||||||
* @type {{
|
* @type {{
|
||||||
* currentTotal: number,
|
* currentTotal: number,
|
||||||
* maxTotal: number,
|
* maxTotal: number,
|
||||||
* currentByAccessKey: object.<string, number>,
|
* currentByIp: object.<string, number>,
|
||||||
* maxPerAccessKey: number
|
* maxPerIp: number
|
||||||
* }}
|
* }}
|
||||||
*/
|
*/
|
||||||
export const CAPTURES_WATCH = {
|
export const CAPTURES_WATCH = {
|
||||||
currentTotal: 0,
|
currentTotal: 0,
|
||||||
maxTotal: MAX_PARALLEL_CAPTURES_TOTAL,
|
maxTotal: MAX_PARALLEL_CAPTURES_TOTAL,
|
||||||
currentByAccessKey: {},
|
currentByIp: {},
|
||||||
maxPerAccessKey: MAX_PARALLEL_CAPTURES_PER_ACCESS_KEY,
|
maxPerIp: MAX_PARALLEL_CAPTURES_PER_IP,
|
||||||
}
|
}
|
||||||
|
|
||||||
export default async function (fastify, opts) {
|
export default async function (fastify, opts) {
|
||||||
|
@ -67,11 +75,10 @@ export default async function (fastify, opts) {
|
||||||
fastify.post('/', capture);
|
fastify.post('/', capture);
|
||||||
|
|
||||||
fastify.get('/check', check);
|
fastify.get('/check', check);
|
||||||
|
|
||||||
fastify.get('/api/v1/hashes/check/:hash', checkHash);
|
fastify.get('/api/v1/hashes/check/:hash', checkHash);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* [GET] /
|
* [GET] /
|
||||||
* Shows the landing page and capture form.
|
* Shows the landing page and capture form.
|
||||||
|
@ -98,7 +105,6 @@ async function index(request, reply) {
|
||||||
* Subject to captures rate limiting (see `CAPTURES_WATCH`).
|
* Subject to captures rate limiting (see `CAPTURES_WATCH`).
|
||||||
*
|
*
|
||||||
* Body is expected as `application/x-www-form-urlencoded` with the following fields:
|
* Body is expected as `application/x-www-form-urlencoded` with the following fields:
|
||||||
* - access-key
|
|
||||||
* - url
|
* - url
|
||||||
* - unfold-thread (optional)
|
* - unfold-thread (optional)
|
||||||
*
|
*
|
||||||
|
@ -110,17 +116,18 @@ async function index(request, reply) {
|
||||||
*/
|
*/
|
||||||
async function capture(request, reply) {
|
async function capture(request, reply) {
|
||||||
const data = request.body;
|
const data = request.body;
|
||||||
const accessKey = data["access-key"];
|
const ip = request.ip;
|
||||||
|
let why = null;
|
||||||
|
|
||||||
request.log.info(`Capture capacity: ${CAPTURES_WATCH.currentTotal} / ${CAPTURES_WATCH.maxTotal}.`);
|
request.log.info(`Capture capacity: ${CAPTURES_WATCH.currentTotal} / ${CAPTURES_WATCH.maxTotal}.`);
|
||||||
|
|
||||||
//
|
//
|
||||||
// Check access key
|
// Check that IP is not in block list
|
||||||
//
|
//
|
||||||
if (!accessKeys.check(accessKey)) {
|
if (ipBlockList.check(ip)) {
|
||||||
const html = nunjucks.render(`${TEMPLATES_PATH}index.njk`, {
|
const html = nunjucks.render(`${TEMPLATES_PATH}index.njk`, {
|
||||||
error: true,
|
error: true,
|
||||||
errorReason: "ACCESS-KEY"
|
errorReason: "IP"
|
||||||
});
|
});
|
||||||
|
|
||||||
return reply
|
return reply
|
||||||
|
@ -148,6 +155,25 @@ async function capture(request, reply) {
|
||||||
.send(html);
|
.send(html);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
//
|
||||||
|
// Check "why" field
|
||||||
|
//
|
||||||
|
try {
|
||||||
|
why = data.why.trim();
|
||||||
|
assert(why.length > 0);
|
||||||
|
}
|
||||||
|
catch(err) {
|
||||||
|
const html = nunjucks.render(`${TEMPLATES_PATH}index.njk`, {
|
||||||
|
error: true,
|
||||||
|
errorReason: "WHY"
|
||||||
|
});
|
||||||
|
|
||||||
|
return reply
|
||||||
|
.code(400)
|
||||||
|
.header('Content-Type', 'text/html; charset=utf-8')
|
||||||
|
.send(html);
|
||||||
|
}
|
||||||
|
|
||||||
//
|
//
|
||||||
// Check that there is still capture capacity (total)
|
// Check that there is still capture capacity (total)
|
||||||
//
|
//
|
||||||
|
@ -164,9 +190,9 @@ async function capture(request, reply) {
|
||||||
}
|
}
|
||||||
|
|
||||||
//
|
//
|
||||||
// Check that there is still capture capacity (for this access key)
|
// Check that there is still capture capacity (for this IP)
|
||||||
//
|
//
|
||||||
if (CAPTURES_WATCH.currentByAccessKey[accessKey] >= CAPTURES_WATCH.maxPerAccessKey) {
|
if (CAPTURES_WATCH.currentByIp[ip] >= CAPTURES_WATCH.maxPerIp) {
|
||||||
const html = nunjucks.render(`${TEMPLATES_PATH}index.njk`, {
|
const html = nunjucks.render(`${TEMPLATES_PATH}index.njk`, {
|
||||||
error: true,
|
error: true,
|
||||||
errorReason: "TOO-MANY-CAPTURES-USER"
|
errorReason: "TOO-MANY-CAPTURES-USER"
|
||||||
|
@ -182,20 +208,20 @@ async function capture(request, reply) {
|
||||||
// Process capture request
|
// Process capture request
|
||||||
//
|
//
|
||||||
try {
|
try {
|
||||||
// Add request to total and per-key counter
|
// Add request to total and per-IP counter
|
||||||
CAPTURES_WATCH.currentTotal += 1;
|
CAPTURES_WATCH.currentTotal += 1;
|
||||||
|
|
||||||
if (accessKey in CAPTURES_WATCH.currentByAccessKey) {
|
if (ip in CAPTURES_WATCH.currentByIp) {
|
||||||
CAPTURES_WATCH.currentByAccessKey[accessKey] += 1;
|
CAPTURES_WATCH.currentByIp[ip] += 1;
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
CAPTURES_WATCH.currentByAccessKey[accessKey] = 1;
|
CAPTURES_WATCH.currentByIp[ip] = 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
const tweets = new TwitterCapture(data.url, {runBrowserBehaviors: "unfold-thread" in data});
|
const tweets = new TwitterCapture(data.url, {runBrowserBehaviors: "unfold-thread" in data});
|
||||||
const pdf = await tweets.capture();
|
const pdf = await tweets.capture();
|
||||||
|
|
||||||
successLog.add(accessKey, pdf);
|
successLog.add(ip, why, pdf);
|
||||||
|
|
||||||
// Generate a filename for the PDF based on url.
|
// Generate a filename for the PDF based on url.
|
||||||
// Example: harvardlil-status-123456789-2022-11-25.pdf
|
// Example: harvardlil-status-123456789-2022-11-25.pdf
|
||||||
|
@ -232,8 +258,8 @@ async function capture(request, reply) {
|
||||||
finally {
|
finally {
|
||||||
CAPTURES_WATCH.currentTotal -= 1;
|
CAPTURES_WATCH.currentTotal -= 1;
|
||||||
|
|
||||||
if (accessKey && accessKey in CAPTURES_WATCH.currentByAccessKey) {
|
if (ip && ip in CAPTURES_WATCH.currentByIp) {
|
||||||
CAPTURES_WATCH.currentByAccessKey[data["access-key"]] -= 1;
|
CAPTURES_WATCH.currentByIp[ip] -= 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,7 +4,6 @@
|
||||||
* @author The Harvard Library Innovation Lab
|
* @author The Harvard Library Innovation Lab
|
||||||
* @license MIT
|
* @license MIT
|
||||||
*/
|
*/
|
||||||
import fs from "fs";
|
|
||||||
import assert from "assert";
|
import assert from "assert";
|
||||||
import crypto from "crypto";
|
import crypto from "crypto";
|
||||||
|
|
||||||
|
@ -12,32 +11,19 @@ import { test } from "tap";
|
||||||
import Fastify from "fastify";
|
import Fastify from "fastify";
|
||||||
import isHtml from "is-html";
|
import isHtml from "is-html";
|
||||||
|
|
||||||
import { AccessKeys } from "./utils/index.js";
|
|
||||||
|
|
||||||
import server, { CAPTURES_WATCH, successLog } from "./server.js";
|
import server, { CAPTURES_WATCH, successLog } from "./server.js";
|
||||||
import { DATA_PATH, CERTS_PATH } from "./const.js";
|
import { DATA_PATH, CERTS_PATH } from "./const.js";
|
||||||
|
|
||||||
/**
|
|
||||||
* Access keys fixture.
|
|
||||||
* @type {{active: string[], inactive: string[]}}
|
|
||||||
*/
|
|
||||||
const ACCESS_KEYS = (() => {
|
|
||||||
const rawAccessKeys = JSON.parse(fs.readFileSync(AccessKeys.filepath));
|
|
||||||
|
|
||||||
const out = { active: [], inactive: [] };
|
|
||||||
|
|
||||||
for (let [key, value] of Object.entries(rawAccessKeys)) {
|
|
||||||
value === true ? out.active.push(key) : out.inactive.push(key);
|
|
||||||
}
|
|
||||||
|
|
||||||
return out;
|
|
||||||
})();
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Dummy url of a thread to capture.
|
* Dummy url of a thread to capture.
|
||||||
*/
|
*/
|
||||||
const THREAD_URL = "https://twitter.com/HarvardLIL/status/1595150565428039680";
|
const THREAD_URL = "https://twitter.com/HarvardLIL/status/1595150565428039680";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Dummy reason for capture
|
||||||
|
*/
|
||||||
|
const WHY = "Testing thread-keeper";
|
||||||
|
|
||||||
test("Integration tests for server.js", async(t) => {
|
test("Integration tests for server.js", async(t) => {
|
||||||
|
|
||||||
// Do not run tests if `CERTS_PATH` and `DATA_PATH` do not point to a `fixtures` folder
|
// Do not run tests if `CERTS_PATH` and `DATA_PATH` do not point to a `fixtures` folder
|
||||||
|
@ -64,36 +50,42 @@ test("Integration tests for server.js", async(t) => {
|
||||||
t.type(isHtml(response.body), true, "Server serves HTML.");
|
t.type(isHtml(response.body), true, "Server serves HTML.");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("[POST] / returns HTTP 401 + HTML on failed access key check.", async (t) => {
|
test("[POST] / returns HTTP 401 + HTML on blocked IP check.", async (t) => {
|
||||||
const app = Fastify({logger: false});
|
const app = Fastify({logger: false});
|
||||||
await server(app, {});
|
await server(app, {});
|
||||||
|
|
||||||
const scenarios = [
|
const params = new URLSearchParams();
|
||||||
"FOO-BAR", // Invalid key
|
params.append("url", THREAD_URL);
|
||||||
ACCESS_KEYS.inactive[0], // Inactive key
|
|
||||||
null // No key
|
|
||||||
]
|
|
||||||
|
|
||||||
for (const accessKey of scenarios) {
|
const response = await app.inject({
|
||||||
const params = new URLSearchParams();
|
method: "POST",
|
||||||
|
url: "/",
|
||||||
|
remoteAddress: "1.2.3.4",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/x-www-form-urlencoded",
|
||||||
|
},
|
||||||
|
body: params.toString()
|
||||||
|
});
|
||||||
|
|
||||||
if (accessKey) {
|
t.equal(response.statusCode, 401, "Server returns HTTP 401.");
|
||||||
params.append("access-key", accessKey);
|
|
||||||
}
|
|
||||||
|
|
||||||
const response = await app.inject({
|
const body = `${response.body}`;
|
||||||
method: "POST",
|
t.type(isHtml(body), true, "Server serves HTML");
|
||||||
url: "/",
|
t.equal(body.includes(`data-reason="IP"`), true, "With error message.");
|
||||||
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
});
|
||||||
body: params.toString(),
|
|
||||||
});
|
|
||||||
|
|
||||||
t.equal(response.statusCode, 401, "Server returns HTTP 401.");
|
test("[POST] / lets IPs that are no longer blocked make requests.", async (t) => {
|
||||||
|
const app = Fastify({logger: false});
|
||||||
|
await server(app, {});
|
||||||
|
|
||||||
const body = `${response.body}`;
|
const response = await app.inject({
|
||||||
t.type(isHtml(body), true, "Server serves HTML");
|
method: "POST",
|
||||||
t.equal(body.includes(`data-reason="ACCESS-KEY"`), true, "With error message.");
|
url: "/",
|
||||||
}
|
remoteAddress: "4.3.2.1",
|
||||||
|
});
|
||||||
|
|
||||||
|
// Should fail because no URL were passed, not because IP was blocked
|
||||||
|
t.equal(response.statusCode, 400, "Server returns HTTP 400.");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("[POST] / returns HTTP 400 + HTML on failed url check.", async (t) => {
|
test("[POST] / returns HTTP 400 + HTML on failed url check.", async (t) => {
|
||||||
|
@ -110,8 +102,6 @@ test("Integration tests for server.js", async(t) => {
|
||||||
for (const url of scenarios) {
|
for (const url of scenarios) {
|
||||||
const params = new URLSearchParams();
|
const params = new URLSearchParams();
|
||||||
|
|
||||||
params.append("access-key", ACCESS_KEYS.active[0]);
|
|
||||||
|
|
||||||
if (url) {
|
if (url) {
|
||||||
params.append("url", url);
|
params.append("url", url);
|
||||||
}
|
}
|
||||||
|
@ -123,7 +113,7 @@ test("Integration tests for server.js", async(t) => {
|
||||||
body: params.toString(),
|
body: params.toString(),
|
||||||
});
|
});
|
||||||
|
|
||||||
t.equal(response.statusCode, 400, "Server returns HTTP 401.");
|
t.equal(response.statusCode, 400, "Server returns HTTP 400.");
|
||||||
|
|
||||||
const body = `${response.body}`;
|
const body = `${response.body}`;
|
||||||
t.type(isHtml(body), true, "Server serves HTML");
|
t.type(isHtml(body), true, "Server serves HTML");
|
||||||
|
@ -131,6 +121,40 @@ test("Integration tests for server.js", async(t) => {
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("[POST] / returns HTTP 400 + HTML on failed \"why\" check.", async (t) => {
|
||||||
|
const app = Fastify({logger: false});
|
||||||
|
await server(app, {});
|
||||||
|
|
||||||
|
const scenarios = [
|
||||||
|
null, // Nothing
|
||||||
|
" ", // Empty
|
||||||
|
"",
|
||||||
|
]
|
||||||
|
|
||||||
|
for (const why of scenarios) {
|
||||||
|
const params = new URLSearchParams();
|
||||||
|
|
||||||
|
params.append("url", THREAD_URL);
|
||||||
|
|
||||||
|
if (why) {
|
||||||
|
params.append("why", why);
|
||||||
|
}
|
||||||
|
|
||||||
|
const response = await app.inject({
|
||||||
|
method: "POST",
|
||||||
|
url: "/",
|
||||||
|
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
||||||
|
body: params.toString(),
|
||||||
|
});
|
||||||
|
|
||||||
|
t.equal(response.statusCode, 400, "Server returns HTTP 400.");
|
||||||
|
|
||||||
|
const body = `${response.body}`;
|
||||||
|
t.type(isHtml(body), true, "Server serves HTML");
|
||||||
|
t.equal(body.includes(`data-reason="WHY"`), true, "With error message.");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test("[POST] / returns HTTP 503 + HTML on failed server capacity check.", async (t) => {
|
test("[POST] / returns HTTP 503 + HTML on failed server capacity check.", async (t) => {
|
||||||
const app = Fastify({logger: false});
|
const app = Fastify({logger: false});
|
||||||
await server(app, {});
|
await server(app, {});
|
||||||
|
@ -138,8 +162,8 @@ test("Integration tests for server.js", async(t) => {
|
||||||
CAPTURES_WATCH.currentTotal = CAPTURES_WATCH.maxTotal; // Simulate peak
|
CAPTURES_WATCH.currentTotal = CAPTURES_WATCH.maxTotal; // Simulate peak
|
||||||
|
|
||||||
const params = new URLSearchParams();
|
const params = new URLSearchParams();
|
||||||
params.append("access-key", ACCESS_KEYS.active[0]);
|
|
||||||
params.append("url", THREAD_URL);
|
params.append("url", THREAD_URL);
|
||||||
|
params.append("why", WHY);
|
||||||
|
|
||||||
const response = await app.inject({
|
const response = await app.inject({
|
||||||
method: "POST",
|
method: "POST",
|
||||||
|
@ -161,19 +185,18 @@ test("Integration tests for server.js", async(t) => {
|
||||||
const app = Fastify({logger: false});
|
const app = Fastify({logger: false});
|
||||||
await server(app, {});
|
await server(app, {});
|
||||||
|
|
||||||
const userKey = ACCESS_KEYS.active[0];
|
const userIp = "127.0.0.1";
|
||||||
|
CAPTURES_WATCH.currentByIp[userIp] = CAPTURES_WATCH.maxPerIp;
|
||||||
CAPTURES_WATCH.currentByAccessKey[userKey] = CAPTURES_WATCH.maxPerAccessKey;
|
|
||||||
|
|
||||||
const params = new URLSearchParams();
|
const params = new URLSearchParams();
|
||||||
params.append("access-key", userKey);
|
|
||||||
params.append("url", THREAD_URL);
|
params.append("url", THREAD_URL);
|
||||||
|
params.append("why", WHY);
|
||||||
|
|
||||||
const response = await app.inject({
|
const response = await app.inject({
|
||||||
method: "POST",
|
method: "POST",
|
||||||
url: "/",
|
url: "/",
|
||||||
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
||||||
body: params.toString(),
|
body: params.toString()
|
||||||
});
|
});
|
||||||
|
|
||||||
t.equal(response.statusCode, 429, "Server returns HTTP 503.");
|
t.equal(response.statusCode, 429, "Server returns HTTP 503.");
|
||||||
|
@ -182,7 +205,7 @@ test("Integration tests for server.js", async(t) => {
|
||||||
t.type(isHtml(body), true, "Server serves HTML");
|
t.type(isHtml(body), true, "Server serves HTML");
|
||||||
t.equal(body.includes(`TOO-MANY-CAPTURES-USER`), true, "With error message.");
|
t.equal(body.includes(`TOO-MANY-CAPTURES-USER`), true, "With error message.");
|
||||||
|
|
||||||
delete CAPTURES_WATCH.currentByAccessKey[userKey];
|
delete CAPTURES_WATCH.currentByIp[userIp];
|
||||||
});
|
});
|
||||||
|
|
||||||
test("[POST] / returns HTTP 200 + PDF", async (t) => {
|
test("[POST] / returns HTTP 200 + PDF", async (t) => {
|
||||||
|
@ -190,8 +213,8 @@ test("Integration tests for server.js", async(t) => {
|
||||||
await server(app, {});
|
await server(app, {});
|
||||||
|
|
||||||
const params = new URLSearchParams();
|
const params = new URLSearchParams();
|
||||||
params.append("access-key", ACCESS_KEYS.active[0]);
|
|
||||||
params.append("url", THREAD_URL);
|
params.append("url", THREAD_URL);
|
||||||
|
params.append("why", WHY);
|
||||||
params.append("unfold-thread", "on");
|
params.append("unfold-thread", "on");
|
||||||
|
|
||||||
const response = await app.inject({
|
const response = await app.inject({
|
||||||
|
@ -246,7 +269,7 @@ test("Integration tests for server.js", async(t) => {
|
||||||
// Add entry to success logs
|
// Add entry to success logs
|
||||||
const toHash = Buffer.from(`${Date.now()}`);
|
const toHash = Buffer.from(`${Date.now()}`);
|
||||||
const hash = crypto.createHash('sha512').update(toHash).digest('base64');
|
const hash = crypto.createHash('sha512').update(toHash).digest('base64');
|
||||||
successLog.add(ACCESS_KEYS.active[0], toHash);
|
successLog.add("127.0.0.1", WHY, toHash);
|
||||||
|
|
||||||
const response = await app.inject({
|
const response = await app.inject({
|
||||||
method: "GET",
|
method: "GET",
|
||||||
|
|
|
@ -266,7 +266,7 @@ body#index > main form label {
|
||||||
padding-bottom: 0.25rem;
|
padding-bottom: 0.25rem;
|
||||||
}
|
}
|
||||||
|
|
||||||
body#index > main form input {
|
body#index > main form input, body#index > main form textarea {
|
||||||
display: block;
|
display: block;
|
||||||
width: 100%;
|
width: 100%;
|
||||||
padding: 0.5rem;
|
padding: 0.5rem;
|
||||||
|
|
|
@ -7,15 +7,15 @@ const formSubmitDialog = document.querySelector("dialog#form-submit");
|
||||||
document.querySelector("body#index form button").addEventListener("click", (e) => {
|
document.querySelector("body#index form button").addEventListener("click", (e) => {
|
||||||
e.preventDefault();
|
e.preventDefault();
|
||||||
const url = document.querySelector("body#index form input[name='url']");
|
const url = document.querySelector("body#index form input[name='url']");
|
||||||
const accessKey = document.querySelector("body#index form input[name='access-key']")
|
const why = document.querySelector("body#index form input[name='why']");
|
||||||
|
|
||||||
if (!url.checkValidity()) {
|
if (!url.checkValidity()) {
|
||||||
url.reportValidity();
|
url.reportValidity();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!accessKey.checkValidity()) {
|
if (!why.checkValidity()) {
|
||||||
accessKey.reportValidity();
|
why.reportValidity();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -34,11 +34,8 @@
|
||||||
</fieldset>
|
</fieldset>
|
||||||
|
|
||||||
<fieldset>
|
<fieldset>
|
||||||
<label for="access-key">Access key <a href="https://ocb.to/archive-social-form" title="Access key request form">(request access)</a></label>
|
<label for="why">Reason for archiving <a href="#who-can-use-it">(why this question?)</a></label>
|
||||||
<input type="password"
|
<textarea required name="why" id="why" rows="2"></textarea>
|
||||||
name="access-key"
|
|
||||||
id="access-key"
|
|
||||||
required>
|
|
||||||
</fieldset>
|
</fieldset>
|
||||||
|
|
||||||
<fieldset class="submit">
|
<fieldset class="submit">
|
||||||
|
@ -66,20 +63,24 @@
|
||||||
<dialog id="form-error" data-reason="{{ errorReason }}">
|
<dialog id="form-error" data-reason="{{ errorReason }}">
|
||||||
<h2>Something went wrong</h2>
|
<h2>Something went wrong</h2>
|
||||||
|
|
||||||
{% if errorReason and errorReason == "ACCESS-KEY" %}
|
{% if errorReason and errorReason == "IP" %}
|
||||||
<p>The access key provided is invalid or no longer active.</p>
|
<p>Your access to this service has been restricted.</p>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
{% if errorReason and errorReason == "URL" %}
|
{% if errorReason and errorReason == "URL" %}
|
||||||
<p>The url provided is not a valid twitter.com url.</p>
|
<p>The url provided is not a valid twitter.com url.</p>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
|
{% if errorReason and errorReason == "WHY" %}
|
||||||
|
<p>The url provided is not a valid twitter.com url.</p>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
{% if errorReason and errorReason == "TOO-MANY-CAPTURES-TOTAL" %}
|
{% if errorReason and errorReason == "TOO-MANY-CAPTURES-TOTAL" %}
|
||||||
<p>Too many requests. Please retry in a minute.</p>
|
<p>Too many requests. Please retry in a minute.</p>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
{% if errorReason and errorReason == "TOO-MANY-CAPTURES-USER" %}
|
{% if errorReason and errorReason == "TOO-MANY-CAPTURES-USER" %}
|
||||||
<p>Please wait until the capture requests you've started are completed before starting a new one.</p>
|
<p>Too many requests for your IP address. Please retry in a minute.</p>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
{% if errorReason and errorReason == "CAPTURE-ISSUE" %}
|
{% if errorReason and errorReason == "CAPTURE-ISSUE" %}
|
||||||
|
@ -95,9 +96,9 @@
|
||||||
|
|
||||||
<p>This site is an experiment by the <a href="https://lil.law.harvard.edu">Harvard Library Innovation Lab</a> to let you download signed PDFs of Twitter URLs. <a href="/static/example.pdf">Here's an example PDF</a> we made from <a href="https://twitter.com/doctorow/status/1591759999323492358">this tweet</a>.</p>
|
<p>This site is an experiment by the <a href="https://lil.law.harvard.edu">Harvard Library Innovation Lab</a> to let you download signed PDFs of Twitter URLs. <a href="/static/example.pdf">Here's an example PDF</a> we made from <a href="https://twitter.com/doctorow/status/1591759999323492358">this tweet</a>.</p>
|
||||||
|
|
||||||
<h2>Who can use it?</h2>
|
<h2 id="who-can-use-it">Who can use it?</h2>
|
||||||
|
|
||||||
<p>To use our website <a href="https://ocb.to/archive-social-form">you'll need to contact us</a> for an API key. We're currently only able to share a limited number with people like journalists, internet scholars, and archivists. But you can also use <a href="https://github.com/harvard-lil/thread-keeper">our open source software</a> to stand up an archive server of your own, and share it with your friends.</p>
|
<p>While anyone can use this website, it is experimental and running with limited server capacity. But you can also use <a href="https://github.com/harvard-lil/thread-keeper">our open source software</a> to stand up an archive server of your own, and share it with your friends.</p>
|
||||||
|
|
||||||
<h2>Why make a PDF archiving tool for Twitter?</h2>
|
<h2>Why make a PDF archiving tool for Twitter?</h2>
|
||||||
|
|
||||||
|
|
|
@ -1,63 +0,0 @@
|
||||||
/**
|
|
||||||
* thread-keeper
|
|
||||||
* @module utils.AccessKeys
|
|
||||||
* @author The Harvard Library Innovation Lab
|
|
||||||
* @license MIT
|
|
||||||
*/
|
|
||||||
import assert from "assert";
|
|
||||||
import fs from "fs";
|
|
||||||
|
|
||||||
import { validate as uuidValidate } from 'uuid';
|
|
||||||
|
|
||||||
import { DATA_PATH } from "../const.js";
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Utility class for handling access keys to the app.
|
|
||||||
* [!] For alpha launch only.
|
|
||||||
*/
|
|
||||||
export class AccessKeys {
|
|
||||||
/**
|
|
||||||
* Complete path to `access-keys.json`.
|
|
||||||
* @type {string}
|
|
||||||
*/
|
|
||||||
static filepath = `${DATA_PATH}access-keys.json`;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Frozen hashmap of available access keys
|
|
||||||
* (app needs to be restarted for new keys to be taken into account, for now).
|
|
||||||
* @type {object.<string,boolean>}
|
|
||||||
*/
|
|
||||||
#keys = {};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* On init:
|
|
||||||
* - Create access keys file if it doesn't exist
|
|
||||||
* - Load keys from file into `this.#keys`.
|
|
||||||
*/
|
|
||||||
constructor() {
|
|
||||||
const filepath = AccessKeys.filepath;
|
|
||||||
|
|
||||||
try {
|
|
||||||
const keys = fs.readFileSync(filepath);
|
|
||||||
this.#keys = Object.freeze(JSON.parse(keys));
|
|
||||||
}
|
|
||||||
catch (err) {
|
|
||||||
fs.writeFileSync(filepath, "{}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Checks that a given access key is valid and active.
|
|
||||||
* @param {string} accessKey
|
|
||||||
*/
|
|
||||||
check(accessKey) {
|
|
||||||
try {
|
|
||||||
assert(uuidValidate(accessKey));
|
|
||||||
assert(this.#keys[accessKey] === true);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
catch(err) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -0,0 +1,61 @@
|
||||||
|
/**
|
||||||
|
* thread-keeper
|
||||||
|
* @module utils.IPBlockList
|
||||||
|
* @author The Harvard Library Innovation Lab
|
||||||
|
* @license MIT
|
||||||
|
*/
|
||||||
|
import assert from "assert";
|
||||||
|
import fs from "fs";
|
||||||
|
|
||||||
|
import { DATA_PATH } from "../const.js";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Utility class for handling the app's IP block list.
|
||||||
|
* [!] For alpha launch only.
|
||||||
|
*/
|
||||||
|
export class IPBlockList {
|
||||||
|
/**
|
||||||
|
* Complete path to `ip-block-list.json`.
|
||||||
|
* @type {string}
|
||||||
|
*/
|
||||||
|
static filepath = `${DATA_PATH}ip-block-list.json`;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Frozen hashmap of IPs that need blocking.
|
||||||
|
* (app needs to be restarted for new list to be taken into account, for now).
|
||||||
|
* @type {object.<string,boolean>}
|
||||||
|
*/
|
||||||
|
#ips = {};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* On init:
|
||||||
|
* - Create access keys file if it doesn't exist
|
||||||
|
* - Load IPS from file into `this.#ips`.
|
||||||
|
*/
|
||||||
|
constructor() {
|
||||||
|
const filepath = IPBlockList.filepath;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const ips = fs.readFileSync(filepath);
|
||||||
|
this.#ips = Object.freeze(JSON.parse(ips));
|
||||||
|
}
|
||||||
|
catch (err) {
|
||||||
|
fs.writeFileSync(filepath, "{}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Checks that a IP is on the blocklist and (still) blocked.
|
||||||
|
* @param {string} ip
|
||||||
|
*/
|
||||||
|
check(ip) {
|
||||||
|
try {
|
||||||
|
assert(ip in this.#ips)
|
||||||
|
assert(this.#ips[ip] === true);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
catch(err) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -69,15 +69,25 @@ export class SuccessLog {
|
||||||
* - Creates a success log entry
|
* - Creates a success log entry
|
||||||
* - Updates `this.#hashes` (so it doesn't need to reload from file)
|
* - Updates `this.#hashes` (so it doesn't need to reload from file)
|
||||||
*
|
*
|
||||||
* @param {string} accessKey
|
* @param {string} ip
|
||||||
|
* @param {string} why - Reason for creating this archive
|
||||||
* @param {Buffer} pdfBytes - Used to store a SHA512 hash of the PDF that was delivered
|
* @param {Buffer} pdfBytes - Used to store a SHA512 hash of the PDF that was delivered
|
||||||
*/
|
*/
|
||||||
add(accessKey, pdfBytes) {
|
add(ip, why, pdfBytes) {
|
||||||
// Calculate SHA512 hash of the PDF
|
// Calculate SHA512 hash of the PDF
|
||||||
const hash = crypto.createHash('sha512').update(pdfBytes).digest('base64');
|
const hash = crypto.createHash('sha512').update(pdfBytes).digest('base64');
|
||||||
|
|
||||||
|
// "why" field sanitization
|
||||||
|
why = why
|
||||||
|
.replaceAll("\t", " ")
|
||||||
|
.replaceAll("\n", " ")
|
||||||
|
.replaceAll("<", "")
|
||||||
|
.replaceAll(">", "")
|
||||||
|
.replaceAll("{", "")
|
||||||
|
.replaceAll("}", "");
|
||||||
|
|
||||||
// Save entry
|
// Save entry
|
||||||
const entry = `${new Date().toISOString()}\t${accessKey}\tsha512-${hash}\n`;
|
const entry = `${new Date().toISOString()}\t${ip}\t${why}\tsha512-${hash}\n`;
|
||||||
fs.appendFileSync(SuccessLog.filepath, entry);
|
fs.appendFileSync(SuccessLog.filepath, entry);
|
||||||
this.#hashes[`sha512-${hash}`] = true;
|
this.#hashes[`sha512-${hash}`] = true;
|
||||||
}
|
}
|
||||||
|
@ -108,7 +118,7 @@ export class SuccessLog {
|
||||||
* @returns {void}
|
* @returns {void}
|
||||||
*/
|
*/
|
||||||
reset() {
|
reset() {
|
||||||
fs.writeFileSync(SuccessLog.filepath, "date-time\taccess-key\thash\n");
|
fs.writeFileSync(SuccessLog.filepath, "date-time\tip\twhy\thash\n");
|
||||||
this.#hashes = {};
|
this.#hashes = {};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,9 +4,9 @@
|
||||||
* @author The Harvard Library Innovation Lab
|
* @author The Harvard Library Innovation Lab
|
||||||
* @license MIT
|
* @license MIT
|
||||||
*/
|
*/
|
||||||
import { AccessKeys } from "./AccessKeys.js";
|
import { IPBlockList } from "./IPBlockList.js";
|
||||||
import { TwitterCapture } from "./TwitterCapture.js";
|
import { TwitterCapture } from "./TwitterCapture.js";
|
||||||
import { SuccessLog } from "./SuccessLog.js";
|
import { SuccessLog } from "./SuccessLog.js";
|
||||||
import { CertsHistory } from "./CertsHistory.js";
|
import { CertsHistory } from "./CertsHistory.js";
|
||||||
|
|
||||||
export { AccessKeys, SuccessLog, CertsHistory, TwitterCapture };
|
export { IPBlockList, SuccessLog, CertsHistory, TwitterCapture };
|
|
@ -14,7 +14,7 @@ thread-keeper
|
||||||
* [.EXECUTABLES_FOLDER](#module_const.EXECUTABLES_FOLDER)
|
* [.EXECUTABLES_FOLDER](#module_const.EXECUTABLES_FOLDER)
|
||||||
* [.STATIC_PATH](#module_const.STATIC_PATH)
|
* [.STATIC_PATH](#module_const.STATIC_PATH)
|
||||||
* [.MAX_PARALLEL_CAPTURES_TOTAL](#module_const.MAX_PARALLEL_CAPTURES_TOTAL)
|
* [.MAX_PARALLEL_CAPTURES_TOTAL](#module_const.MAX_PARALLEL_CAPTURES_TOTAL)
|
||||||
* [.MAX_PARALLEL_CAPTURES_PER_ACCESS_KEY](#module_const.MAX_PARALLEL_CAPTURES_PER_ACCESS_KEY)
|
* [.MAX_PARALLEL_CAPTURES_PER_IP](#module_const.MAX_PARALLEL_CAPTURES_PER_IP)
|
||||||
* [.APP_VERSION](#module_const.APP_VERSION)
|
* [.APP_VERSION](#module_const.APP_VERSION)
|
||||||
|
|
||||||
<a name="module_const.CERTS_PATH"></a>
|
<a name="module_const.CERTS_PATH"></a>
|
||||||
|
@ -59,10 +59,10 @@ Path to the "static" folder.
|
||||||
Maximum capture processes that can be run in parallel.
|
Maximum capture processes that can be run in parallel.
|
||||||
|
|
||||||
**Kind**: static constant of [<code>const</code>](#module_const)
|
**Kind**: static constant of [<code>const</code>](#module_const)
|
||||||
<a name="module_const.MAX_PARALLEL_CAPTURES_PER_ACCESS_KEY"></a>
|
<a name="module_const.MAX_PARALLEL_CAPTURES_PER_IP"></a>
|
||||||
|
|
||||||
### const.MAX\_PARALLEL\_CAPTURES\_PER\_ACCESS\_KEY
|
### const.MAX\_PARALLEL\_CAPTURES\_PER\_IP
|
||||||
Maximum capture processes that can be run in parallel for a given key.
|
Maximum capture processes that can be run in parallel for a IP address.
|
||||||
|
|
||||||
**Kind**: static constant of [<code>const</code>](#module_const)
|
**Kind**: static constant of [<code>const</code>](#module_const)
|
||||||
<a name="module_const.APP_VERSION"></a>
|
<a name="module_const.APP_VERSION"></a>
|
||||||
|
|
|
@ -9,9 +9,10 @@ thread-keeper
|
||||||
* [server](#module_server)
|
* [server](#module_server)
|
||||||
* _static_
|
* _static_
|
||||||
* [.successLog](#module_server.successLog) : <code>SuccessLog</code>
|
* [.successLog](#module_server.successLog) : <code>SuccessLog</code>
|
||||||
|
* [.options](#module_server.options)
|
||||||
* [.CAPTURES_WATCH](#module_server.CAPTURES_WATCH) : <code>Object</code>
|
* [.CAPTURES_WATCH](#module_server.CAPTURES_WATCH) : <code>Object</code>
|
||||||
* _inner_
|
* _inner_
|
||||||
* [~accessKeys](#module_server..accessKeys) : <code>AccessKeys</code>
|
* [~ipBlockList](#module_server..ipBlockList) : <code>IPBlockList</code>
|
||||||
* [~index(request, reply)](#module_server..index) ⇒ <code>Promise.<fastify.FastifyReply></code>
|
* [~index(request, reply)](#module_server..index) ⇒ <code>Promise.<fastify.FastifyReply></code>
|
||||||
* [~capture(request, reply)](#module_server..capture) ⇒ <code>Promise.<fastify.FastifyReply></code>
|
* [~capture(request, reply)](#module_server..capture) ⇒ <code>Promise.<fastify.FastifyReply></code>
|
||||||
* [~check(request, reply)](#module_server..check) ⇒ <code>Promise.<fastify.FastifyReply></code>
|
* [~check(request, reply)](#module_server..check) ⇒ <code>Promise.<fastify.FastifyReply></code>
|
||||||
|
@ -20,6 +21,12 @@ thread-keeper
|
||||||
<a name="module_server.successLog"></a>
|
<a name="module_server.successLog"></a>
|
||||||
|
|
||||||
### server.successLog : <code>SuccessLog</code>
|
### server.successLog : <code>SuccessLog</code>
|
||||||
|
**Kind**: static constant of [<code>server</code>](#module_server)
|
||||||
|
<a name="module_server.options"></a>
|
||||||
|
|
||||||
|
### server.options
|
||||||
|
Fastify-cli options
|
||||||
|
|
||||||
**Kind**: static constant of [<code>server</code>](#module_server)
|
**Kind**: static constant of [<code>server</code>](#module_server)
|
||||||
<a name="module_server.CAPTURES_WATCH"></a>
|
<a name="module_server.CAPTURES_WATCH"></a>
|
||||||
|
|
||||||
|
@ -30,9 +37,9 @@ May be used to redirect users if over capacity.
|
||||||
[!] Only good for early prototyping.
|
[!] Only good for early prototyping.
|
||||||
|
|
||||||
**Kind**: static constant of [<code>server</code>](#module_server)
|
**Kind**: static constant of [<code>server</code>](#module_server)
|
||||||
<a name="module_server..accessKeys"></a>
|
<a name="module_server..ipBlockList"></a>
|
||||||
|
|
||||||
### server~accessKeys : <code>AccessKeys</code>
|
### server~ipBlockList : <code>IPBlockList</code>
|
||||||
**Kind**: inner constant of [<code>server</code>](#module_server)
|
**Kind**: inner constant of [<code>server</code>](#module_server)
|
||||||
<a name="module_server..index"></a>
|
<a name="module_server..index"></a>
|
||||||
|
|
||||||
|
@ -58,7 +65,6 @@ Returns to form with specific error code, passed as `errorReason`, otherwise.
|
||||||
Subject to captures rate limiting (see `CAPTURES_WATCH`).
|
Subject to captures rate limiting (see `CAPTURES_WATCH`).
|
||||||
|
|
||||||
Body is expected as `application/x-www-form-urlencoded` with the following fields:
|
Body is expected as `application/x-www-form-urlencoded` with the following fields:
|
||||||
- access-key
|
|
||||||
- url
|
- url
|
||||||
- unfold-thread (optional)
|
- unfold-thread (optional)
|
||||||
|
|
||||||
|
|
|
@ -1,51 +0,0 @@
|
||||||
<a name="utils.module_AccessKeys"></a>
|
|
||||||
|
|
||||||
## AccessKeys
|
|
||||||
thread-keeper
|
|
||||||
|
|
||||||
**Author**: The Harvard Library Innovation Lab
|
|
||||||
**License**: MIT
|
|
||||||
|
|
||||||
* [AccessKeys](#utils.module_AccessKeys)
|
|
||||||
* [.AccessKeys](#utils.module_AccessKeys.AccessKeys)
|
|
||||||
* [new exports.AccessKeys()](#new_utils.module_AccessKeys.AccessKeys_new)
|
|
||||||
* [.filepath](#utils.module_AccessKeys.AccessKeys+filepath) : <code>string</code>
|
|
||||||
* [.check(accessKey)](#utils.module_AccessKeys.AccessKeys+check)
|
|
||||||
|
|
||||||
<a name="utils.module_AccessKeys.AccessKeys"></a>
|
|
||||||
|
|
||||||
### AccessKeys.AccessKeys
|
|
||||||
Utility class for handling access keys to the app.
|
|
||||||
[!] For alpha launch only.
|
|
||||||
|
|
||||||
**Kind**: static class of [<code>AccessKeys</code>](#utils.module_AccessKeys)
|
|
||||||
|
|
||||||
* [.AccessKeys](#utils.module_AccessKeys.AccessKeys)
|
|
||||||
* [new exports.AccessKeys()](#new_utils.module_AccessKeys.AccessKeys_new)
|
|
||||||
* [.filepath](#utils.module_AccessKeys.AccessKeys+filepath) : <code>string</code>
|
|
||||||
* [.check(accessKey)](#utils.module_AccessKeys.AccessKeys+check)
|
|
||||||
|
|
||||||
<a name="new_utils.module_AccessKeys.AccessKeys_new"></a>
|
|
||||||
|
|
||||||
#### new exports.AccessKeys()
|
|
||||||
On init:
|
|
||||||
- Create access keys file if it doesn't exist
|
|
||||||
- Load keys from file into `this.#keys`.
|
|
||||||
|
|
||||||
<a name="utils.module_AccessKeys.AccessKeys+filepath"></a>
|
|
||||||
|
|
||||||
#### accessKeys.filepath : <code>string</code>
|
|
||||||
Complete path to `access-keys.json`.
|
|
||||||
|
|
||||||
**Kind**: instance property of [<code>AccessKeys</code>](#utils.module_AccessKeys.AccessKeys)
|
|
||||||
<a name="utils.module_AccessKeys.AccessKeys+check"></a>
|
|
||||||
|
|
||||||
#### accessKeys.check(accessKey)
|
|
||||||
Checks that a given access key is valid and active.
|
|
||||||
|
|
||||||
**Kind**: instance method of [<code>AccessKeys</code>](#utils.module_AccessKeys.AccessKeys)
|
|
||||||
|
|
||||||
| Param | Type |
|
|
||||||
| --- | --- |
|
|
||||||
| accessKey | <code>string</code> |
|
|
||||||
|
|
|
@ -0,0 +1,51 @@
|
||||||
|
<a name="utils.module_IPBlockList"></a>
|
||||||
|
|
||||||
|
## IPBlockList
|
||||||
|
thread-keeper
|
||||||
|
|
||||||
|
**Author**: The Harvard Library Innovation Lab
|
||||||
|
**License**: MIT
|
||||||
|
|
||||||
|
* [IPBlockList](#utils.module_IPBlockList)
|
||||||
|
* [.IPBlockList](#utils.module_IPBlockList.IPBlockList)
|
||||||
|
* [new exports.IPBlockList()](#new_utils.module_IPBlockList.IPBlockList_new)
|
||||||
|
* [.filepath](#utils.module_IPBlockList.IPBlockList+filepath) : <code>string</code>
|
||||||
|
* [.check(ip)](#utils.module_IPBlockList.IPBlockList+check)
|
||||||
|
|
||||||
|
<a name="utils.module_IPBlockList.IPBlockList"></a>
|
||||||
|
|
||||||
|
### IPBlockList.IPBlockList
|
||||||
|
Utility class for handling the app's IP block list.
|
||||||
|
[!] For alpha launch only.
|
||||||
|
|
||||||
|
**Kind**: static class of [<code>IPBlockList</code>](#utils.module_IPBlockList)
|
||||||
|
|
||||||
|
* [.IPBlockList](#utils.module_IPBlockList.IPBlockList)
|
||||||
|
* [new exports.IPBlockList()](#new_utils.module_IPBlockList.IPBlockList_new)
|
||||||
|
* [.filepath](#utils.module_IPBlockList.IPBlockList+filepath) : <code>string</code>
|
||||||
|
* [.check(ip)](#utils.module_IPBlockList.IPBlockList+check)
|
||||||
|
|
||||||
|
<a name="new_utils.module_IPBlockList.IPBlockList_new"></a>
|
||||||
|
|
||||||
|
#### new exports.IPBlockList()
|
||||||
|
On init:
|
||||||
|
- Create access keys file if it doesn't exist
|
||||||
|
- Load IPS from file into `this.#ips`.
|
||||||
|
|
||||||
|
<a name="utils.module_IPBlockList.IPBlockList+filepath"></a>
|
||||||
|
|
||||||
|
#### ipBlockList.filepath : <code>string</code>
|
||||||
|
Complete path to `ip-block-list.json`.
|
||||||
|
|
||||||
|
**Kind**: instance property of [<code>IPBlockList</code>](#utils.module_IPBlockList.IPBlockList)
|
||||||
|
<a name="utils.module_IPBlockList.IPBlockList+check"></a>
|
||||||
|
|
||||||
|
#### ipBlockList.check(ip)
|
||||||
|
Checks that a IP is on the blocklist and (still) blocked.
|
||||||
|
|
||||||
|
**Kind**: instance method of [<code>IPBlockList</code>](#utils.module_IPBlockList.IPBlockList)
|
||||||
|
|
||||||
|
| Param | Type |
|
||||||
|
| --- | --- |
|
||||||
|
| ip | <code>string</code> |
|
||||||
|
|
|
@ -10,7 +10,7 @@ thread-keeper
|
||||||
* [.SuccessLog](#utils.module_SuccessLog.SuccessLog)
|
* [.SuccessLog](#utils.module_SuccessLog.SuccessLog)
|
||||||
* [new exports.SuccessLog()](#new_utils.module_SuccessLog.SuccessLog_new)
|
* [new exports.SuccessLog()](#new_utils.module_SuccessLog.SuccessLog_new)
|
||||||
* [.filepath](#utils.module_SuccessLog.SuccessLog+filepath) : <code>string</code>
|
* [.filepath](#utils.module_SuccessLog.SuccessLog+filepath) : <code>string</code>
|
||||||
* [.add(accessKey, pdfBytes)](#utils.module_SuccessLog.SuccessLog+add)
|
* [.add(ip, pdfBytes)](#utils.module_SuccessLog.SuccessLog+add)
|
||||||
* [.findHashInLogs(hash)](#utils.module_SuccessLog.SuccessLog+findHashInLogs) ⇒ <code>boolean</code>
|
* [.findHashInLogs(hash)](#utils.module_SuccessLog.SuccessLog+findHashInLogs) ⇒ <code>boolean</code>
|
||||||
* [.reset()](#utils.module_SuccessLog.SuccessLog+reset) ⇒ <code>void</code>
|
* [.reset()](#utils.module_SuccessLog.SuccessLog+reset) ⇒ <code>void</code>
|
||||||
|
|
||||||
|
@ -24,7 +24,7 @@ Utility class for handling success logs. Keeps trace of the hashes of the PDFs t
|
||||||
* [.SuccessLog](#utils.module_SuccessLog.SuccessLog)
|
* [.SuccessLog](#utils.module_SuccessLog.SuccessLog)
|
||||||
* [new exports.SuccessLog()](#new_utils.module_SuccessLog.SuccessLog_new)
|
* [new exports.SuccessLog()](#new_utils.module_SuccessLog.SuccessLog_new)
|
||||||
* [.filepath](#utils.module_SuccessLog.SuccessLog+filepath) : <code>string</code>
|
* [.filepath](#utils.module_SuccessLog.SuccessLog+filepath) : <code>string</code>
|
||||||
* [.add(accessKey, pdfBytes)](#utils.module_SuccessLog.SuccessLog+add)
|
* [.add(ip, pdfBytes)](#utils.module_SuccessLog.SuccessLog+add)
|
||||||
* [.findHashInLogs(hash)](#utils.module_SuccessLog.SuccessLog+findHashInLogs) ⇒ <code>boolean</code>
|
* [.findHashInLogs(hash)](#utils.module_SuccessLog.SuccessLog+findHashInLogs) ⇒ <code>boolean</code>
|
||||||
* [.reset()](#utils.module_SuccessLog.SuccessLog+reset) ⇒ <code>void</code>
|
* [.reset()](#utils.module_SuccessLog.SuccessLog+reset) ⇒ <code>void</code>
|
||||||
|
|
||||||
|
@ -43,7 +43,7 @@ Complete path to `success-log.json`.
|
||||||
**Kind**: instance property of [<code>SuccessLog</code>](#utils.module_SuccessLog.SuccessLog)
|
**Kind**: instance property of [<code>SuccessLog</code>](#utils.module_SuccessLog.SuccessLog)
|
||||||
<a name="utils.module_SuccessLog.SuccessLog+add"></a>
|
<a name="utils.module_SuccessLog.SuccessLog+add"></a>
|
||||||
|
|
||||||
#### successLog.add(accessKey, pdfBytes)
|
#### successLog.add(ip, pdfBytes)
|
||||||
Calculates hash of a PDF an:
|
Calculates hash of a PDF an:
|
||||||
- Creates a success log entry
|
- Creates a success log entry
|
||||||
- Updates `this.#hashes` (so it doesn't need to reload from file)
|
- Updates `this.#hashes` (so it doesn't need to reload from file)
|
||||||
|
@ -52,7 +52,7 @@ Calculates hash of a PDF an:
|
||||||
|
|
||||||
| Param | Type | Description |
|
| Param | Type | Description |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| accessKey | <code>string</code> | |
|
| ip | <code>string</code> | |
|
||||||
| pdfBytes | <code>Buffer</code> | Used to store a SHA512 hash of the PDF that was delivered |
|
| pdfBytes | <code>Buffer</code> | Used to store a SHA512 hash of the PDF that was delivered |
|
||||||
|
|
||||||
<a name="utils.module_SuccessLog.SuccessLog+findHashInLogs"></a>
|
<a name="utils.module_SuccessLog.SuccessLog+findHashInLogs"></a>
|
||||||
|
|
|
@ -1,4 +0,0 @@
|
||||||
{
|
|
||||||
"a9c2b7b6-0652-4207-bdce-0527ad28f3f9": true,
|
|
||||||
"60e605ec-3510-4358-a0a7-33f25b3d7b74": false
|
|
||||||
}
|
|
|
@ -0,0 +1,4 @@
|
||||||
|
{
|
||||||
|
"1.2.3.4": true,
|
||||||
|
"4.3.2.1": false
|
||||||
|
}
|
|
@ -6,7 +6,7 @@
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"start": "fastify start app/server.js -l warn",
|
"start": "fastify start app/server.js -l warn",
|
||||||
"dev": "fastify start app/server.js -l info -w",
|
"dev": "fastify start app/server.js -l info -w -P",
|
||||||
"postinstall": "cd scripts && bash download-yt-dlp.sh && bash pip-install.sh",
|
"postinstall": "cd scripts && bash download-yt-dlp.sh && bash pip-install.sh",
|
||||||
"generate-dev-cert": "cd scripts && bash generate-dev-cert.sh",
|
"generate-dev-cert": "cd scripts && bash generate-dev-cert.sh",
|
||||||
"docgen": "cd scripts && bash docgen.sh",
|
"docgen": "cd scripts && bash docgen.sh",
|
||||||
|
|
|
@ -3,7 +3,7 @@ jsdoc2md ../app/server.js > ../docs/server.md;
|
||||||
jsdoc2md ../app/const.js > ../docs/const.md;
|
jsdoc2md ../app/const.js > ../docs/const.md;
|
||||||
|
|
||||||
jsdoc2md ../app/utils/index.js > ../docs/utils/index.md;
|
jsdoc2md ../app/utils/index.js > ../docs/utils/index.md;
|
||||||
jsdoc2md ../app/utils/AccessKeys.js > ../docs/utils/AccessKeys.md;
|
jsdoc2md ../app/utils/IPBlockList.js > ../docs/utils/IPBlockList.md;
|
||||||
jsdoc2md ../app/utils/SuccessLog.js > ../docs/utils/SuccessLog.md;
|
jsdoc2md ../app/utils/SuccessLog.js > ../docs/utils/SuccessLog.md;
|
||||||
jsdoc2md ../app/utils/TwitterCapture.js > ../docs/utils/TwitterCapture.md;
|
jsdoc2md ../app/utils/TwitterCapture.js > ../docs/utils/TwitterCapture.md;
|
||||||
jsdoc2md ../app/utils/CertsHistory.js > ../docs/utils/CertsHistory.md;
|
jsdoc2md ../app/utils/CertsHistory.js > ../docs/utils/CertsHistory.md;
|
||||||
|
|
Ładowanie…
Reference in New Issue