Add scribe.privacyredirect.com

Unknown if this will actually stop them, but at least I can show my intent. User agents sourced from https://darkvisitors.com/

CHANGELOG

@@ -1,3 +1,21 @@
+Unreleased
+
+* Add a bunch of well-known, LLM scrapers to robots.txt
+* Add command to tag releases
+* Modernize nix config
+* Added scribe.manasiwibi.com instance
+
+2023-12-18
+
+* Added release script
+* Update License to include https and package.json
+* Add Nix package and NixOS moodule
+* Remove outdated postgres check and dep from shell.nix
+* Fixed Dockerfile
+* Update Dockerfile to use multi-arch base image
+* Add Docker.arm64 file
+* Remove instances that promote hate
+
 2023-05-21
 
 * Remove unused carbon shard which should fix build failures

Dockerfile

@@ -3,13 +3,13 @@ WORKDIR /tmp_build
 
 COPY package.json .
 COPY yarn.lock .
-RUN yarn install --no-progress --frozen-lockfile
+RUN yarn install --network-timeout 120000 --no-progress --frozen-lockfile
 
 COPY webpack.mix.js .
 COPY src ./src
 RUN yarn prod
 
-FROM crystallang/crystal:1.5-alpine as lucky_build
+FROM 84codes/crystal:1.8.1-alpine as lucky_build
 ENV SKIP_LUCKY_TASK_PRECOMPILATION="1"
 RUN apk add yaml-static
 WORKDIR /tmp_build

LICENSE

@@ -1,7 +1,7 @@
                     GNU AFFERO GENERAL PUBLIC LICENSE
                        Version 3, 19 November 2007
 
- Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 
@@ -643,7 +643,7 @@ the "copyright" line and a pointer to where the full notice is found.
     GNU Affero General Public License for more details.
 
     You should have received a copy of the GNU Affero General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 Also add information on how to contact you by electronic and paper mail.
 
@@ -658,4 +658,4 @@ specific requirements.
   You should also get your employer (if you work as a programmer) or school,
 if any, to sign a "copyright disclaimer" for the program, if necessary.
 For more information on this, and how to apply and follow the GNU AGPL, see
-<http://www.gnu.org/licenses/>.
+<https://www.gnu.org/licenses/>.

default.nix

@@ -0,0 +1,58 @@
+{ crystal
+, mkYarnPackage
+, fetchYarnDeps
+}:
+
+let
+  version = "1.0.0";
+
+  ui = mkYarnPackage {
+    pname = "scribe-ui";
+    inherit version;
+    src = ./.;
+    packageJSON = ./package.json;
+
+    offlineCache = fetchYarnDeps {
+      yarnLock = ./yarn.lock;
+      sha256 = "sha256-ixnGRTTKq20tnjOnHeibu12a+n3edV1eM5Om2iNO9fo=";
+    };
+
+    configurePhase = ''
+      runHook preConfigure
+      cp -r $node_modules node_modules
+      chmod +w node_modules
+      runHook postConfigure
+    '';
+
+    buildPhase = ''
+      runHook preBuild
+      export HOME=$(mktemp -d)
+      OUTPUT_DIR=$out yarn --offline prod
+      runHook postBuild
+    '';
+
+    installPhase = ''
+      mkdir -p "$out"
+      mv public "$out/public"
+    '';
+    distPhase = "true";
+  };
+in
+crystal.buildCrystalPackage rec {
+  pname = "scribe";
+  inherit version;
+
+  src = ./.;
+  shardsFile = ./shards.nix;
+
+  preBuild = ''
+    cp -a ${ui}/public/mix-manifest.json public/mix-manifest.json
+  '';
+
+  doCheck = false;
+  doInstallCheck = false;
+  format = "shards";
+  postInstall = ''
+    cp -r ${ui}/public "$out/public"
+  '';
+}

docs/instances.json

@@ -4,9 +4,11 @@
   "https://scribe.citizen4.eu",
   "https://scribe.bus-hit.me",
   "https://scribe.froth.zone",
-  "https://scribe.esmailelbob.xyz",
   "https://scribe.privacydev.net",
   "https://scribe.rawbit.ninja",
-  "https://sc.vern.cc"
-  "https://m.opnxng.com"
+  "https://sc.vern.cc",
+  "https://m.opnxng.com",
+  "https://scribe.manasiwibi.com",
+  "https://scribe.r4fo.com",
+  "https://scribe.privacyredirect.com"
 ]

docs/instances.md

@@ -5,13 +5,16 @@
 * <https://scribe.citizen4.eu>
 * <https://scribe.bus-hit.me>
 * <https://scribe.froth.zone>
-* <https://scribe.esmailelbob.xyz>
 * <https://scribe.privacydev.net>
 * <https://scribe.rawbit.ninja>
 * <https://m.opnxng.com>
-* <http://scribe.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd.onion> (Tor)
+* <https://scribe.manasiwibi.com>
+* <https://scribe.r4fo.com>
+* <https://scribe.privacyredirect.com>
 * <http://w7uhv5lxhgck72hhimdglmusc54t4m6bionlmd5mvyddq3bs53mohqid.onion> (Tor)
 * <http://scribe.g4c3eya4clenolymqbpgwz3q3tawoxw56yhzk4vugqrl6dtu3ejvhjid.onion> (Tor)
+* <http://umxccfmp4gyfllsdlzkrnhpd3lxlf4necjolrz22yzcrgwflbrzgtiad.onion> (Tor)
+* <http://scribe.r4focoma7gu2zdwwcjjad47ysxt634lg73sxmdbkdozanwqslho5ohyd.onion> (Tor)
 * [sc.vern.i2p](http://vern3whzyfmjclq6snhlupma6nrmojghwp37tydfgqotj7sc6izq.b32.i2p) (I2P)
 
 ## How do I get my instance on this list?

flake.lock

@@ -17,15 +17,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1634282420,
-        "narHash": "sha256-YOI78SSF4Q/ZFoEgfO8Xy3EnjCW/F9VgB2Qz9YljzhI=",
+        "lastModified": 1701253981,
+        "narHash": "sha256-ztaDIyZ7HrTAfEEUt9AtTDNoCYxUdSd6NrRHaYOIxtk=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "0a68ef410b40f49de76aecb5c8b5cc5111bac91d",
+        "rev": "e92039b55bcd58469325ded85d4f58dd5a4eaf58",
         "type": "github"
       },
       "original": {
         "id": "nixpkgs",
+        "ref": "nixos-unstable",
         "type": "indirect"
       }
     },

flake.nix

@@ -1,8 +1,21 @@
 {
-  inputs = { flake-utils.url = "github:numtide/flake-utils"; };
+  description = "Scribe";
+
+  inputs = {
+    nixpkgs.url = "nixpkgs/nixos-unstable";
+    flake-utils.url = "github:numtide/flake-utils";
+  };
 
   outputs = { self, nixpkgs, flake-utils }:
-    flake-utils.lib.eachDefaultSystem (system:
-      let pkgs = nixpkgs.legacyPackages.${system};
-      in { devShell = import ./shell.nix { inherit pkgs; }; });
+    flake-utils.lib.eachDefaultSystem
+      (system:
+        let pkgs = nixpkgs.legacyPackages.${system};
+        in
+        {
+          devShell = import ./shell.nix { inherit pkgs; };
+          packages.default = pkgs.callPackage ./default.nix { };
+        })
+    // {
+      nixosModules.default = import ./module.nix self;
+    };
 }

module.nix

@@ -0,0 +1,111 @@
+self: { config, lib, pkgs, ... }:
+let
+  cfg = config.services.scribe;
+in
+{
+  options.services.scribe = {
+    enable = lib.mkEnableOption (lib.mdDoc "Enable or disable the Scribe service");
+
+    package = lib.mkOption {
+      type = lib.types.package;
+      default = self.packages."${pkgs.system}".default;
+      description = lib.mdDoc "Overridable attribute of the scribe package to use.";
+    };
+
+    user = lib.mkOption {
+      type = lib.types.str;
+      default = "scribe";
+      description = lib.mdDoc "User to run the Scribe service as.";
+    };
+
+    group = lib.mkOption {
+      type = lib.types.str;
+      default = "scribe";
+      description = lib.mdDoc "Group to run the Scribe service as.";
+    };
+
+    appDomain = lib.mkOption {
+      type = lib.types.str;
+      description = lib.mdDoc ''
+        The domain that Scribe is being run on. This will appear on the Scribe homepage.
+      '';
+    };
+
+    port = lib.mkOption {
+      type = lib.types.port;
+      description = lib.mdDoc "Port for the Scribe service to use.";
+    };
+
+    environmentFile = lib.mkOption {
+      type = lib.types.str;
+      description = lib.mdDoc ''
+        The path to a file containing environment varible to be set in Scribes environment.
+        This should be user to set SECRET_KEY_BASE, GITHUB_USERNAME, and GITHUB_PERSONAL_ACCESS_TOKEN. 
+        Descriptions of these settings can be found
+        [in the official docs](https://sr.ht/~edwardloveall/Scribe/#configuration).
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.services.scribe = {
+      description = "Scribe";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      environment = {
+        LUCKY_ENV = "production";
+        APP_DOMAIN = cfg.appDomain;
+        PORT = (toString cfg.port);
+      };
+      serviceConfig = {
+        ExecStart = "${cfg.package}/bin/scribe";
+        EnvironmentFile = cfg.environmentFile;
+        Restart = "on-failure";
+        User = cfg.user;
+        Group = cfg.group;
+        UMask = "0007";
+        ProtectSystem = "strict";
+        ProtectClock = true;
+        ProtectKernelLogs = true;
+        SystemCallArchitectures = "native";
+        ProtectHome = true;
+        ProtectProc = "noaccess";
+        MemoryDenyWriteExecute = true;
+        NoNewPrivileges = true;
+        PrivateDevices = true;
+        PrivateMounts = true;
+        PrivateTmp = true;
+        ProtectControlGroups = true;
+        ProtectHostname = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        RestrictNamespaces = true;
+        RestrictRealtime = true;
+        RestrictSUIDSGID = true;
+        CapabilityBoundingSet = [
+          "~CAP_SYS_PTRACE"
+          "~CAP_SYS_ADMIN"
+          "~CAP_SETGID"
+          "~CAP_SETUID"
+          "~CAP_SETPCAP"
+          "~CAP_SYS_TIME"
+          "~CAP_KILL"
+          "~CAP_SYS_PACCT"
+          "~CAP_SYS_TTY_CONFIG "
+          "~CAP_SYS_CHROOT"
+          "~CAP_SYS_BOOT"
+          "~CAP_NET_ADMIN"
+        ];
+      };
+    };
+    users.users = lib.optionalAttrs (cfg.user == "scribe") {
+      "scribe" = {
+        group = "scribe";
+        isSystemUser = true;
+      };
+    };
+    users.groups = lib.optionalAttrs (cfg.group == "scribe") {
+      "scribe" = { };
+    };
+  };
+}

package.json

@@ -1,10 +1,11 @@
 {
-  "license": "UNLICENSED",
+  "name": "scribe-ui",
+  "license": "AGPL-3.0-or-later",
   "private": true,
   "dependencies": {
     "@rails/ujs": "^6.0.0",
     "compression-webpack-plugin": "^8.0.1",
-    "laravel-mix": "^6.0.28",
+    "laravel-mix": "^6.0.49",
     "modern-normalize": "^1.1.0",
     "postcss": "^8.3.6",
     "tufte-css": "^1.8.0",
@@ -22,5 +23,6 @@
     "resolve-url-loader": "^3.1.1",
     "sass": "^1.26.10",
     "sass-loader": "^10.0.2"
-  }
+  },
+  "version": "0.0.0"
 }

public/robots.txt

@@ -1,4 +1,55 @@
-# Learn more about robots.txt: https://www.robotstxt.org/robotstxt.html
-User-agent: *
-# 'Disallow' with an empty value allows all paths to be crawled
-Disallow:
+# ChatGPT-User
+User-agent: ChatGPT-User
+Disallow: /
+
+# cohere-ai
+User-agent: cohere-ai
+Disallow: /
+
+# anthropic-ai
+User-agent: anthropic-ai
+Disallow: /
+
+# Bytespider
+User-agent: Bytespider
+Disallow: /
+
+# CCBot
+User-agent: CCBot
+Disallow: /
+
+# FacebookBot
+User-agent: FacebookBot
+Disallow: /
+
+# Google-Extended
+User-agent: Google-Extended
+Disallow: /
+
+# GPTBot
+User-agent: GPTBot
+Disallow: /
+
+# omgili
+User-agent: omgili
+Disallow: /
+
+# Amazonbot
+User-agent: Amazonbot
+Disallow: /
+
+# Applebot
+User-agent: Applebot
+Disallow: /
+
+# PerplexityBot
+User-agent: PerplexityBot
+Disallow: /
+
+# PerplexityBot
+User-agent: PerplexityBot
+Disallow: /
+
+# YouBot
+User-agent: YouBot
+Disallow: /

script/build_static

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+# Exit if any subcommand fails
+set -e
+set -o pipefail
+
+todays_date=$(date "+%Y-%m-%d")
+if ! git show HEAD:src/version.cr | rg -q $todays_date; then
+  echo "Date in committed src/version.cr is not today's date ($todays_date)"
+  echo "Make sure that the file is both up to date and commited to git."
+  echo
+  echo "## src/version.cr"
+  git show HEAD:src/version.cr
+  # exit 1
+fi
+
+# Via: https://crystal-lang.org/reference/1.10/guides/static_linking.html#linux
+
+~/.docker/bin/docker run --rm -it -v $(pwd):/workspace -w /workspace crystallang/crystal:latest-alpine \
+  crystal build src/start_server.cr -o ubuntu_server --static --release

script/release

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+set -e
+
+todays_date=$(date "+%Y-%m-%d")
+sed -i '' -E "s/[0-9]{4}-[0-9]{2}-[0-9]{2}/$todays_date/" src/version.cr
+
+# Delete the tag if it exists. This will show an error but it's fine
+git tag -d "$todays_date" || true
+
+echo
+echo "Bumped version to $todays_date"
+echo "Here are the commits since the last tag"
+echo "Update the Changelog"
+echo
+
+git log $(git describe --tags --abbrev=0)~..HEAD --oneline
+git tag "$todays_date"

script/system_check

@@ -17,17 +17,6 @@ if command_not_found "yarn"; then
   print_error "Yarn is not installed\n  See https://yarnpkg.com/lang/en/docs/install/ for install instructions."
 fi
 
-if command_not_found "createdb"; then
-  MSG="Please install the postgres CLI tools, then try again."
-  if is_mac; then
-    MSG="$MSG\nIf you're using Postgres.app, see https://postgresapp.com/documentation/cli-tools.html."
-  fi
-  MSG="$MSG\nSee https://www.postgresql.org/docs/current/tutorial-install.html for install instructions."
-
-  print_error "$MSG"
-fi
-
-
 ## CUSTOM PRE-BOOT CHECKS ##
 # example:
 # if command_not_running "redis-cli ping"; then

shards.nix

@@ -0,0 +1,176 @@
+{
+  authentic = {
+    owner = "luckyframework";
+    repo = "authentic";
+    rev = "v1.0.0";
+    sha256 = "0mc7xqh0zm4jg8vc1awlzr249fviiy1y40w4fvyvq959hlpd6zx4";
+  };
+  avram = {
+    owner = "luckyframework";
+    repo = "avram";
+    rev = "v1.0.0";
+    sha256 = "18w90m5iq0jy026zma05swh2am936j132fs3j730lq7x5yr8289c";
+  };
+  backtracer = {
+    owner = "sija";
+    repo = "backtracer.cr";
+    rev = "v1.2.2";
+    sha256 = "1rknyylsi14m7i77x7c3138wdw27i4f6sd78m3srw851p47bwr20";
+  };
+  cadmium_transliterator = {
+    owner = "cadmiumcr";
+    repo = "transliterator";
+    rev = "46c4c14594057dbcfaf27e7e7c8c164d3f0ce3f1";
+    sha256 = "15x9xbgybqrmqb7s5cpx3fgwysp5ld97vlvz8b196lqmyqnnp3d3";
+  };
+  cry = {
+    owner = "luckyframework";
+    repo = "cry";
+    rev = "v0.4.3";
+    sha256 = "0bcvpbi418855cq1jq911dv6r9wmg81rcvcirqrbw8fv2a093ss5";
+  };
+  crystar = {
+    owner = "naqvis";
+    repo = "crystar";
+    rev = "56db8bb9dfbd5ed6d7908353405a5fae632a6561";
+    sha256 = "0bzq7im3z3asr22wzwyj1z0m3m5aq5hh1kscp5gd8vjw192w2z2a";
+  };
+  db = {
+    owner = "crystal-lang";
+    repo = "crystal-db";
+    rev = "v0.11.0";
+    sha256 = "1ylfhpn64p72ywi39niqb179f61z08q4qd4hhjza05z18mdaghl3";
+  };
+  dexter = {
+    owner = "luckyframework";
+    repo = "dexter";
+    rev = "v0.3.4";
+    sha256 = "08fv3ns0wxkyr2rcifj3ihyaf7g4lsmfamfhdxbkdkmxa9l1z6cj";
+  };
+  exception_page = {
+    owner = "crystal-loot";
+    repo = "exception_page";
+    rev = "v0.3.0";
+    sha256 = "1w82283mgaaw1hy5xk997a1av4sxaa01ydipbxm5nb9nq7fgfydk";
+  };
+  fnv = {
+    owner = "naqvis";
+    repo = "crystal-fnv";
+    rev = "v0.1.3";
+    sha256 = "1vhy3j0ifc0rlrx5b6wbpcvjzw15k303jrz3bzvnxqvi600fvv2b";
+  };
+  habitat = {
+    owner = "luckyframework";
+    repo = "habitat";
+    rev = "v0.4.7";
+    sha256 = "0d183pnswgjwqg388zmnx7s41ai88ca96nl5cybi0z6icr5npw64";
+  };
+  html5 = {
+    owner = "naqvis";
+    repo = "crystal-html5";
+    rev = "v0.4.0";
+    sha256 = "0mr4vd4bl3a22jl8h698zrh8rz6m5lm2lcyx11055gn6fw0yq57k";
+  };
+  lucky = {
+    owner = "luckyframework";
+    repo = "lucky";
+    rev = "v1.0.0";
+    sha256 = "13by6bbgpbbbdncgj87cqy5y6z7s9zb3nr88dh3fwl5mfgygk66z";
+  };
+  lucky_cache = {
+    owner = "luckyframework";
+    repo = "lucky_cache";
+    rev = "v0.1.1";
+    sha256 = "1ic9nfmiv89q5v82ybshd9xqnwv62bv8a5n8rhmsm9cwvdhgc92x";
+  };
+  lucky_env = {
+    owner = "luckyframework";
+    repo = "lucky_env";
+    rev = "v0.1.4";
+    sha256 = "0rcz0kh9rkypgm34r7maqqmgirxblhwzycwxpp0y9ai68lq71qxk";
+  };
+  lucky_flow = {
+    owner = "luckyframework";
+    repo = "lucky_flow";
+    rev = "v0.9.0";
+    sha256 = "1gyxba7lbjhzbd7a5hcswr3i04mz6rqypihhpgx213aa2685c0mw";
+  };
+  lucky_router = {
+    owner = "luckyframework";
+    repo = "lucky_router";
+    rev = "v0.5.2";
+    sha256 = "1gl93rijnbaqybpry19rn951kbx1q1bb5w0npdp4fm0r212b3yh8";
+  };
+  lucky_task = {
+    owner = "luckyframework";
+    repo = "lucky_task";
+    rev = "v0.1.1";
+    sha256 = "0w0rnf22pvj3lp5z8c4sshzwhqgwpbjpm7nry9mf0iz3fa0v48f7";
+  };
+  monads = {
+    owner = "alex-lairan";
+    repo = "monads";
+    rev = "v1.0.0";
+    sha256 = "0wwhsmnzsmw03dn2j4n75sprp4baxg24i1hn1xhfzz9b33rmlxxf";
+  };
+  pg = {
+    owner = "will";
+    repo = "crystal-pg";
+    rev = "v0.26.0";
+    sha256 = "04fwbgrlf2nzma0p2c8ki7p8sk113jhziq2al3ivif2lpmhr39fy";
+  };
+  pulsar = {
+    owner = "luckyframework";
+    repo = "pulsar";
+    rev = "v0.2.3";
+    sha256 = "03pp0r1klqk49fkzjwg9mnxqplv6pdfjn6a1p59f2w1ha5piyy90";
+  };
+  selenium = {
+    owner = "matthewmcgarvey";
+    repo = "selenium.cr";
+    rev = "v0.10.0";
+    sha256 = "062baqafz2rn9czaj8wl2b1l7ngxdph2j8xcr088f2kd8bb0hj7v";
+  };
+  shell-table = {
+    owner = "luckyframework";
+    repo = "shell-table.cr";
+    rev = "v0.9.3";
+    sha256 = "046vymm2r37c6j5bqyjzxdgg5h62slsannzvfhbckkv2r9chwd3w";
+  };
+  splay_tree_map = {
+    owner = "wyhaines";
+    repo = "splay_tree_map.cr";
+    rev = "v0.2.2";
+    sha256 = "0196zpg0v190dv23mwnbia35znxz2j2g8dqynd2b8827qiwmz1vn";
+  };
+  teeplate = {
+    owner = "luckyframework";
+    repo = "teeplate";
+    rev = "v0.8.5";
+    sha256 = "1kr05qrp674rph1324wry57gzvgvcvlz0w27brlvdgd3gi4s8sdj";
+  };
+  webdrivers = {
+    owner = "matthewmcgarvey";
+    repo = "webdrivers.cr";
+    rev = "v0.4.1";
+    sha256 = "05q6z1rv29hrwq77wpas2ki4alwhx4fpallb94q4m9g5h5vfn6ag";
+  };
+  webless = {
+    owner = "matthewmcgarvey";
+    repo = "webless";
+    rev = "v0.1.0";
+    sha256 = "0fg79wy3fq0af77jm121pqfm43dzb7l5rlx13vrl74pgqagms0ih";
+  };
+  wordsmith = {
+    owner = "luckyframework";
+    repo = "wordsmith";
+    rev = "v0.4.0";
+    sha256 = "13fsmwdh431smbmsv869pa8p34g1hqd84za33xsymsycq5459xq2";
+  };
+  xpath2 = {
+    owner = "naqvis";
+    repo = "crystal-xpath2";
+    rev = "v0.1.3";
+    sha256 = "17jl0br2fibc22sz9qdpsqd17rsmnar0jwh4iq25y8rg64pgb1h0";
+  };
+}

shell.nix

@@ -1,17 +1,16 @@
 { pkgs ? import <nixpkgs> { } }:
 
 pkgs.mkShell {
-  shellHook = ''
-    export PKG_CONFIG_PATH=${pkgs.openssl.dev}/lib/pkgconfig
-  '';
   buildInputs = with pkgs; [
     crystal
     lucky-cli
     overmind
     nodejs
-    openssl.dev
-    postgresql
+    openssl
+    pkg-config
     shards
     yarn
+    crystal2nix
+    pcre
   ];
 }

src/version.cr

@@ -1,3 +1,3 @@
 module Scribe
-  VERSION = "2023-05-21"
+  VERSION = "2023-12-18"
 end