2022-11-24 07:20:30 +00:00
|
|
|
/**
|
|
|
|
* thread-keeper
|
|
|
|
* @module server.test
|
|
|
|
* @author The Harvard Library Innovation Lab
|
|
|
|
* @license MIT
|
|
|
|
*/
|
2022-11-29 20:39:08 +00:00
|
|
|
import fs from "fs";
|
2022-11-24 07:20:30 +00:00
|
|
|
import assert from "assert";
|
|
|
|
import crypto from "crypto";
|
|
|
|
|
|
|
|
import { test } from "tap";
|
|
|
|
import Fastify from "fastify";
|
|
|
|
import isHtml from "is-html";
|
|
|
|
|
2022-11-29 20:39:08 +00:00
|
|
|
import { AccessKeys } from "./utils/index.js";
|
|
|
|
|
2022-11-24 07:20:30 +00:00
|
|
|
import server, { CAPTURES_WATCH, successLog } from "./server.js";
|
|
|
|
import { DATA_PATH, CERTS_PATH } from "./const.js";
|
|
|
|
|
|
|
|
/**
|
2022-11-29 20:39:08 +00:00
|
|
|
* Sample url of a thread to capture.
|
2022-11-24 07:20:30 +00:00
|
|
|
*/
|
2022-11-29 19:31:06 +00:00
|
|
|
const THREAD_URL = "https://twitter.com/HarvardLIL/status/1595150565428039680";
|
2022-11-24 07:20:30 +00:00
|
|
|
|
|
|
|
/**
|
2022-11-29 20:39:08 +00:00
|
|
|
* Sample reason for capture
|
2022-11-24 07:20:30 +00:00
|
|
|
*/
|
2022-11-29 20:39:08 +00:00
|
|
|
const WHY = "Testing thread-keeper.";
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Access keys fixture.
|
|
|
|
* @type {{active: string[], inactive: string[]}}
|
|
|
|
*/
|
|
|
|
const ACCESS_KEYS = (() => {
|
|
|
|
const rawAccessKeys = JSON.parse(fs.readFileSync(AccessKeys.filepath));
|
|
|
|
|
|
|
|
const out = { active: [], inactive: [] };
|
|
|
|
|
|
|
|
for (let [key, value] of Object.entries(rawAccessKeys)) {
|
|
|
|
value === true ? out.active.push(key) : out.inactive.push(key);
|
|
|
|
}
|
|
|
|
|
|
|
|
return out;
|
|
|
|
})();
|
2022-11-24 07:20:30 +00:00
|
|
|
|
|
|
|
test("Integration tests for server.js", async(t) => {
|
|
|
|
|
|
|
|
// Do not run tests if `CERTS_PATH` and `DATA_PATH` do not point to a `fixtures` folder
|
|
|
|
t.before((t) => {
|
|
|
|
try {
|
|
|
|
assert(DATA_PATH.includes("/fixtures/"));
|
|
|
|
assert(CERTS_PATH.includes("/fixtures/"));
|
|
|
|
}
|
|
|
|
catch(err) {
|
|
|
|
throw new Error("Test must be run against fixtures. Set CERTS_PATH and DATA_PATH env vars accordingly.");
|
|
|
|
}
|
|
|
|
});
|
|
|
|
|
|
|
|
test("[GET] / returns HTTP 200 + HTML", async (t) => {
|
|
|
|
const app = Fastify({logger: false});
|
|
|
|
await server(app, {});
|
|
|
|
|
|
|
|
const response = await app.inject({
|
|
|
|
method: "GET",
|
|
|
|
url: "/",
|
|
|
|
});
|
|
|
|
|
|
|
|
t.equal(response.statusCode, 200, "Server returns HTTP 200.");
|
|
|
|
t.type(isHtml(response.body), true, "Server serves HTML.");
|
|
|
|
});
|
|
|
|
|
2022-11-29 20:39:08 +00:00
|
|
|
// This assumes tests are run with the REQUIRE_ACCESS_KEY env var set to `1`.
|
|
|
|
test("[POST] / returns HTTP 401 + HTML on failed access key check.", async (t) => {
|
|
|
|
const app = Fastify({logger: false});
|
|
|
|
await server(app, {});
|
|
|
|
|
|
|
|
const scenarios = [
|
|
|
|
"FOO-BAR", // Invalid key
|
|
|
|
ACCESS_KEYS.inactive[0], // Inactive key
|
|
|
|
null // No key
|
|
|
|
]
|
|
|
|
|
|
|
|
for (const accessKey of scenarios) {
|
|
|
|
const params = new URLSearchParams();
|
|
|
|
params.append("url", THREAD_URL);
|
|
|
|
params.append("why", WHY);
|
|
|
|
|
|
|
|
if (accessKey) {
|
|
|
|
params.append("access-key", accessKey);
|
|
|
|
}
|
|
|
|
|
|
|
|
const response = await app.inject({
|
|
|
|
method: "POST",
|
|
|
|
url: "/",
|
|
|
|
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
|
|
|
body: params.toString(),
|
|
|
|
});
|
|
|
|
|
|
|
|
t.equal(response.statusCode, 401, "Server returns HTTP 401.");
|
|
|
|
|
|
|
|
const body = `${response.body}`;
|
|
|
|
t.type(isHtml(body), true, "Server serves HTML");
|
|
|
|
t.equal(body.includes(`data-reason="ACCESS-KEY"`), true, "With error message.");
|
|
|
|
}
|
|
|
|
});
|
|
|
|
|
2022-11-29 19:31:06 +00:00
|
|
|
test("[POST] / returns HTTP 401 + HTML on blocked IP check.", async (t) => {
|
|
|
|
const app = Fastify({logger: false});
|
|
|
|
await server(app, {});
|
|
|
|
|
|
|
|
const params = new URLSearchParams();
|
|
|
|
params.append("url", THREAD_URL);
|
|
|
|
|
|
|
|
const response = await app.inject({
|
|
|
|
method: "POST",
|
|
|
|
url: "/",
|
|
|
|
remoteAddress: "1.2.3.4",
|
|
|
|
headers: {
|
|
|
|
"Content-Type": "application/x-www-form-urlencoded",
|
|
|
|
},
|
|
|
|
body: params.toString()
|
|
|
|
});
|
|
|
|
|
|
|
|
t.equal(response.statusCode, 401, "Server returns HTTP 401.");
|
|
|
|
|
|
|
|
const body = `${response.body}`;
|
|
|
|
t.type(isHtml(body), true, "Server serves HTML");
|
|
|
|
t.equal(body.includes(`data-reason="IP"`), true, "With error message.");
|
|
|
|
});
|
|
|
|
|
|
|
|
test("[POST] / lets IPs that are no longer blocked make requests.", async (t) => {
|
|
|
|
const app = Fastify({logger: false});
|
|
|
|
await server(app, {});
|
|
|
|
|
2022-11-29 20:39:08 +00:00
|
|
|
const params = new URLSearchParams();
|
|
|
|
params.append("why", WHY);
|
|
|
|
params.append("access-key", ACCESS_KEYS.active[0]);
|
|
|
|
|
2022-11-29 19:31:06 +00:00
|
|
|
const response = await app.inject({
|
|
|
|
method: "POST",
|
|
|
|
url: "/",
|
|
|
|
remoteAddress: "4.3.2.1",
|
2022-11-29 20:39:08 +00:00
|
|
|
headers: {
|
|
|
|
"Content-Type": "application/x-www-form-urlencoded"
|
|
|
|
},
|
|
|
|
body: params.toString()
|
2022-11-29 19:31:06 +00:00
|
|
|
});
|
|
|
|
|
|
|
|
// Should fail because no URL were passed, not because IP was blocked
|
|
|
|
t.equal(response.statusCode, 400, "Server returns HTTP 400.");
|
|
|
|
});
|
|
|
|
|
|
|
|
test("[POST] / returns HTTP 400 + HTML on failed url check.", async (t) => {
|
2022-11-24 07:20:30 +00:00
|
|
|
const app = Fastify({logger: false});
|
|
|
|
await server(app, {});
|
|
|
|
|
|
|
|
const scenarios = [
|
2022-11-29 19:31:06 +00:00
|
|
|
"https://lil.law.harvard.edu", // Non-twitter
|
|
|
|
"twitter.com/harvardlil", // Non-url
|
|
|
|
12,
|
|
|
|
null // Nothing
|
2022-11-24 07:20:30 +00:00
|
|
|
]
|
|
|
|
|
2022-11-29 19:31:06 +00:00
|
|
|
for (const url of scenarios) {
|
2022-11-24 07:20:30 +00:00
|
|
|
const params = new URLSearchParams();
|
2022-11-29 20:39:08 +00:00
|
|
|
params.append("why", WHY);
|
|
|
|
params.append("access-key", ACCESS_KEYS.active[0]);
|
2022-11-24 07:20:30 +00:00
|
|
|
|
2022-11-29 19:31:06 +00:00
|
|
|
if (url) {
|
|
|
|
params.append("url", url);
|
2022-11-24 07:20:30 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
const response = await app.inject({
|
|
|
|
method: "POST",
|
|
|
|
url: "/",
|
|
|
|
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
|
|
|
body: params.toString(),
|
|
|
|
});
|
2022-11-29 19:31:06 +00:00
|
|
|
|
|
|
|
t.equal(response.statusCode, 400, "Server returns HTTP 400.");
|
|
|
|
|
2022-11-24 07:20:30 +00:00
|
|
|
const body = `${response.body}`;
|
|
|
|
t.type(isHtml(body), true, "Server serves HTML");
|
2022-11-29 19:31:06 +00:00
|
|
|
t.equal(body.includes(`data-reason="URL"`), true, "With error message.");
|
2022-11-24 07:20:30 +00:00
|
|
|
}
|
|
|
|
});
|
|
|
|
|
2022-11-29 19:31:06 +00:00
|
|
|
test("[POST] / returns HTTP 400 + HTML on failed \"why\" check.", async (t) => {
|
2022-11-24 07:20:30 +00:00
|
|
|
const app = Fastify({logger: false});
|
|
|
|
await server(app, {});
|
|
|
|
|
|
|
|
const scenarios = [
|
2022-11-29 19:31:06 +00:00
|
|
|
null, // Nothing
|
|
|
|
" ", // Empty
|
|
|
|
"",
|
2022-11-24 07:20:30 +00:00
|
|
|
]
|
|
|
|
|
2022-11-29 19:31:06 +00:00
|
|
|
for (const why of scenarios) {
|
2022-11-24 07:20:30 +00:00
|
|
|
const params = new URLSearchParams();
|
2022-11-29 19:31:06 +00:00
|
|
|
params.append("url", THREAD_URL);
|
2022-11-29 20:39:08 +00:00
|
|
|
params.append("access-key", ACCESS_KEYS.active[0]);
|
2022-11-24 07:20:30 +00:00
|
|
|
|
2022-11-29 19:31:06 +00:00
|
|
|
if (why) {
|
|
|
|
params.append("why", why);
|
2022-11-24 07:20:30 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
const response = await app.inject({
|
|
|
|
method: "POST",
|
|
|
|
url: "/",
|
|
|
|
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
|
|
|
body: params.toString(),
|
|
|
|
});
|
|
|
|
|
2022-11-29 19:31:06 +00:00
|
|
|
t.equal(response.statusCode, 400, "Server returns HTTP 400.");
|
2022-11-24 07:20:30 +00:00
|
|
|
|
|
|
|
const body = `${response.body}`;
|
|
|
|
t.type(isHtml(body), true, "Server serves HTML");
|
2022-11-29 19:31:06 +00:00
|
|
|
t.equal(body.includes(`data-reason="WHY"`), true, "With error message.");
|
2022-11-24 07:20:30 +00:00
|
|
|
}
|
|
|
|
});
|
|
|
|
|
|
|
|
test("[POST] / returns HTTP 503 + HTML on failed server capacity check.", async (t) => {
|
|
|
|
const app = Fastify({logger: false});
|
|
|
|
await server(app, {});
|
|
|
|
|
|
|
|
CAPTURES_WATCH.currentTotal = CAPTURES_WATCH.maxTotal; // Simulate peak
|
|
|
|
|
|
|
|
const params = new URLSearchParams();
|
|
|
|
params.append("url", THREAD_URL);
|
2022-11-29 19:31:06 +00:00
|
|
|
params.append("why", WHY);
|
2022-11-29 20:39:08 +00:00
|
|
|
params.append("access-key", ACCESS_KEYS.active[0]);
|
2022-11-24 07:20:30 +00:00
|
|
|
|
|
|
|
const response = await app.inject({
|
|
|
|
method: "POST",
|
|
|
|
url: "/",
|
|
|
|
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
|
|
|
body: params.toString(),
|
|
|
|
});
|
|
|
|
|
|
|
|
t.equal(response.statusCode, 503, "Server returns HTTP 503");
|
|
|
|
|
|
|
|
const body = `${response.body}`;
|
|
|
|
t.type(isHtml(body), true, "Server serves HTML");
|
|
|
|
t.equal(body.includes(`TOO-MANY-CAPTURES-TOTAL`), true, "With error message.");
|
|
|
|
|
|
|
|
CAPTURES_WATCH.currentTotal = 0;
|
|
|
|
});
|
|
|
|
|
|
|
|
test("[POST] / returns HTTP 429 + HTML on user over parallel capture allowance.", async (t) => {
|
|
|
|
const app = Fastify({logger: false});
|
|
|
|
await server(app, {});
|
|
|
|
|
2022-11-29 19:31:06 +00:00
|
|
|
const userIp = "127.0.0.1";
|
|
|
|
CAPTURES_WATCH.currentByIp[userIp] = CAPTURES_WATCH.maxPerIp;
|
2022-11-24 07:20:30 +00:00
|
|
|
|
|
|
|
const params = new URLSearchParams();
|
|
|
|
params.append("url", THREAD_URL);
|
2022-11-29 19:31:06 +00:00
|
|
|
params.append("why", WHY);
|
2022-11-29 20:39:08 +00:00
|
|
|
params.append("access-key", ACCESS_KEYS.active[0]);
|
2022-11-24 07:20:30 +00:00
|
|
|
|
|
|
|
const response = await app.inject({
|
|
|
|
method: "POST",
|
|
|
|
url: "/",
|
|
|
|
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
2022-11-29 19:31:06 +00:00
|
|
|
body: params.toString()
|
2022-11-24 07:20:30 +00:00
|
|
|
});
|
|
|
|
|
|
|
|
t.equal(response.statusCode, 429, "Server returns HTTP 503.");
|
|
|
|
|
|
|
|
const body = `${response.body}`;
|
|
|
|
t.type(isHtml(body), true, "Server serves HTML");
|
|
|
|
t.equal(body.includes(`TOO-MANY-CAPTURES-USER`), true, "With error message.");
|
|
|
|
|
2022-11-29 19:31:06 +00:00
|
|
|
delete CAPTURES_WATCH.currentByIp[userIp];
|
2022-11-24 07:20:30 +00:00
|
|
|
});
|
|
|
|
|
|
|
|
test("[POST] / returns HTTP 200 + PDF", async (t) => {
|
|
|
|
const app = Fastify({logger: false});
|
|
|
|
await server(app, {});
|
|
|
|
|
|
|
|
const params = new URLSearchParams();
|
|
|
|
params.append("url", THREAD_URL);
|
2022-11-29 19:31:06 +00:00
|
|
|
params.append("why", WHY);
|
2022-11-29 20:39:08 +00:00
|
|
|
params.append("access-key", ACCESS_KEYS.active[0]);
|
2022-11-24 07:20:30 +00:00
|
|
|
params.append("unfold-thread", "on");
|
|
|
|
|
|
|
|
const response = await app.inject({
|
|
|
|
method: "POST",
|
|
|
|
url: "/",
|
|
|
|
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
|
|
|
body: params.toString(),
|
|
|
|
});
|
|
|
|
|
|
|
|
t.equal(response.statusCode, 200, "Server returns HTTP 200.");
|
|
|
|
t.equal(response.headers["content-type"], "application/pdf");
|
|
|
|
t.equal(response.headers["content-disposition"].startsWith("attachment"), true);
|
|
|
|
t.equal(response.body.substring(0, 8), "%PDF-1.7", "Server returns a PDF as attachment.");
|
|
|
|
|
|
|
|
// Check filename processing
|
|
|
|
const contentDisposition = response.headers["content-disposition"];
|
|
|
|
const filename = contentDisposition.substring(22, contentDisposition.length - 1);
|
|
|
|
t.match(filename, /^twitter-com-[a-z0-9-]+-[0-9]{4}-[0-9]{2}-[0-9]{2}\.pdf$/);
|
|
|
|
});
|
|
|
|
|
|
|
|
test("[GET] /check returns HTTP 200 + HTML", async (t) => {
|
|
|
|
const app = Fastify({logger: false});
|
|
|
|
await server(app, {});
|
|
|
|
|
|
|
|
const response = await app.inject({
|
|
|
|
method: "GET",
|
|
|
|
url: "/check",
|
|
|
|
});
|
|
|
|
|
|
|
|
t.equal(response.statusCode, 200, "Server returns HTTP 200.");
|
|
|
|
t.type(isHtml(response.body), true, "Server serves HTML.");
|
|
|
|
});
|
|
|
|
|
|
|
|
test("[GET] /api/v1/hashes/check/<sha512-hash> returns HTTP 404 on failed hash check.", async (t) => {
|
|
|
|
const app = Fastify({logger: false});
|
|
|
|
await server(app, {});
|
|
|
|
|
|
|
|
const randomHash = crypto.createHash('sha512').update(Buffer.from("HELLO WORLD")).digest('base64');
|
|
|
|
|
|
|
|
const response = await app.inject({
|
|
|
|
method: "GET",
|
|
|
|
url: `/api/v1/hashes/check/${encodeURIComponent(randomHash)}`,
|
|
|
|
});
|
|
|
|
|
|
|
|
t.equal(response.statusCode, 404, "Server returns HTTP 404.");
|
|
|
|
});
|
|
|
|
|
|
|
|
test("[GET] /api/v1/hashes/check/<sha512-hash> returns HTTP 200 on successful hash check.", async (t) => {
|
|
|
|
const app = Fastify({logger: false});
|
|
|
|
await server(app, {});
|
|
|
|
|
|
|
|
// Add entry to success logs
|
|
|
|
const toHash = Buffer.from(`${Date.now()}`);
|
|
|
|
const hash = crypto.createHash('sha512').update(toHash).digest('base64');
|
2022-11-29 19:31:06 +00:00
|
|
|
successLog.add("127.0.0.1", WHY, toHash);
|
2022-11-24 07:20:30 +00:00
|
|
|
|
|
|
|
const response = await app.inject({
|
|
|
|
method: "GET",
|
|
|
|
url: `/api/v1/hashes/check/${encodeURIComponent(hash)}`,
|
|
|
|
});
|
|
|
|
|
|
|
|
t.equal(response.statusCode, 200, "Server returns HTTP 200.");
|
|
|
|
});
|
|
|
|
|
|
|
|
});
|