.tool-versions

@@ -1,3 +1,2 @@
 nodejs 16.18.0
 crystal 1.5.0
-yarn 1.22.19

CHANGELOG

@@ -1,21 +1,3 @@
-Unreleased
-
-* Add a bunch of well-known, LLM scrapers to robots.txt
-* Add command to tag releases
-* Modernize nix config
-* Added scribe.manasiwibi.com instance
-
-2023-12-18
-
-* Added release script
-* Update License to include https and package.json
-* Add Nix package and NixOS moodule
-* Remove outdated postgres check and dep from shell.nix
-* Fixed Dockerfile
-* Update Dockerfile to use multi-arch base image
-* Add Docker.arm64 file
-* Remove instances that promote hate
-
 2023-05-21
 
 * Remove unused carbon shard which should fix build failures

Dockerfile

@@ -3,13 +3,13 @@ WORKDIR /tmp_build
 
 COPY package.json .
 COPY yarn.lock .
-RUN yarn install --network-timeout 120000 --no-progress --frozen-lockfile
+RUN yarn install --no-progress --frozen-lockfile
 
 COPY webpack.mix.js .
 COPY src ./src
 RUN yarn prod
 
-FROM 84codes/crystal:1.8.1-alpine as lucky_build
+FROM crystallang/crystal:1.5-alpine as lucky_build
 ENV SKIP_LUCKY_TASK_PRECOMPILATION="1"
 RUN apk add yaml-static
 WORKDIR /tmp_build

LICENSE

@@ -1,7 +1,7 @@
                     GNU AFFERO GENERAL PUBLIC LICENSE
                        Version 3, 19 November 2007
 
- Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 
@@ -643,7 +643,7 @@ the "copyright" line and a pointer to where the full notice is found.
     GNU Affero General Public License for more details.
 
     You should have received a copy of the GNU Affero General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 Also add information on how to contact you by electronic and paper mail.
 
@@ -658,4 +658,4 @@ specific requirements.
   You should also get your employer (if you work as a programmer) or school,
 if any, to sign a "copyright disclaimer" for the program, if necessary.
 For more information on this, and how to apply and follow the GNU AGPL, see
-<https://www.gnu.org/licenses/>.
+<http://www.gnu.org/licenses/>.

default.nix

@@ -1,58 +0,0 @@
-{ crystal
-, mkYarnPackage
-, fetchYarnDeps
-}:
-
-let
-  version = "1.0.0";
-
-  ui = mkYarnPackage {
-    pname = "scribe-ui";
-    inherit version;
-    src = ./.;
-    packageJSON = ./package.json;
-
-    offlineCache = fetchYarnDeps {
-      yarnLock = ./yarn.lock;
-      sha256 = "sha256-ixnGRTTKq20tnjOnHeibu12a+n3edV1eM5Om2iNO9fo=";
-    };
-
-    configurePhase = ''
-      runHook preConfigure
-      cp -r $node_modules node_modules
-      chmod +w node_modules
-      runHook postConfigure
-    '';
-
-    buildPhase = ''
-      runHook preBuild
-      export HOME=$(mktemp -d)
-      OUTPUT_DIR=$out yarn --offline prod
-      runHook postBuild
-    '';
-
-    installPhase = ''
-      mkdir -p "$out"
-      mv public "$out/public"
-    '';
-    distPhase = "true";
-  };
-in
-crystal.buildCrystalPackage rec {
-  pname = "scribe";
-  inherit version;
-
-  src = ./.;
-  shardsFile = ./shards.nix;
-
-  preBuild = ''
-    cp -a ${ui}/public/mix-manifest.json public/mix-manifest.json
-  '';
-
-  doCheck = false;
-  doInstallCheck = false;
-  format = "shards";
-  postInstall = ''
-    cp -r ${ui}/public "$out/public"
-  '';
-}

docs/instances.json

@@ -4,11 +4,9 @@
   "https://scribe.citizen4.eu",
   "https://scribe.bus-hit.me",
   "https://scribe.froth.zone",
+  "https://scribe.esmailelbob.xyz",
   "https://scribe.privacydev.net",
   "https://scribe.rawbit.ninja",
-  "https://sc.vern.cc",
+  "https://sc.vern.cc"
-  "https://m.opnxng.com",
+  "https://m.opnxng.com"
-  "https://scribe.manasiwibi.com",
-  "https://scribe.r4fo.com",
-  "https://scribe.privacyredirect.com"
 ]

docs/instances.md

@@ -5,16 +5,13 @@
 * <https://scribe.citizen4.eu>
 * <https://scribe.bus-hit.me>
 * <https://scribe.froth.zone>
+* <https://scribe.esmailelbob.xyz>
 * <https://scribe.privacydev.net>
 * <https://scribe.rawbit.ninja>
 * <https://m.opnxng.com>
-* <https://scribe.manasiwibi.com>
+* <http://scribe.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd.onion> (Tor)
-* <https://scribe.r4fo.com>
-* <https://scribe.privacyredirect.com>
 * <http://w7uhv5lxhgck72hhimdglmusc54t4m6bionlmd5mvyddq3bs53mohqid.onion> (Tor)
 * <http://scribe.g4c3eya4clenolymqbpgwz3q3tawoxw56yhzk4vugqrl6dtu3ejvhjid.onion> (Tor)
-* <http://umxccfmp4gyfllsdlzkrnhpd3lxlf4necjolrz22yzcrgwflbrzgtiad.onion> (Tor)
-* <http://scribe.r4focoma7gu2zdwwcjjad47ysxt634lg73sxmdbkdozanwqslho5ohyd.onion> (Tor)
 * [sc.vern.i2p](http://vern3whzyfmjclq6snhlupma6nrmojghwp37tydfgqotj7sc6izq.b32.i2p) (I2P)
 
 ## How do I get my instance on this list?

flake.lock

@@ -17,16 +17,15 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1701253981,
+        "lastModified": 1634282420,
-        "narHash": "sha256-ztaDIyZ7HrTAfEEUt9AtTDNoCYxUdSd6NrRHaYOIxtk=",
+        "narHash": "sha256-YOI78SSF4Q/ZFoEgfO8Xy3EnjCW/F9VgB2Qz9YljzhI=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "e92039b55bcd58469325ded85d4f58dd5a4eaf58",
+        "rev": "0a68ef410b40f49de76aecb5c8b5cc5111bac91d",
         "type": "github"
       },
       "original": {
         "id": "nixpkgs",
-        "ref": "nixos-unstable",
         "type": "indirect"
       }
     },

flake.nix

@@ -1,21 +1,8 @@
 {
-  description = "Scribe";
+  inputs = { flake-utils.url = "github:numtide/flake-utils"; };
-
-  inputs = {
-    nixpkgs.url = "nixpkgs/nixos-unstable";
-    flake-utils.url = "github:numtide/flake-utils";
-  };
 
   outputs = { self, nixpkgs, flake-utils }:
-    flake-utils.lib.eachDefaultSystem
+    flake-utils.lib.eachDefaultSystem (system:
-      (system:
+      let pkgs = nixpkgs.legacyPackages.${system};
-        let pkgs = nixpkgs.legacyPackages.${system};
+      in { devShell = import ./shell.nix { inherit pkgs; }; });
-        in
-        {
-          devShell = import ./shell.nix { inherit pkgs; };
-          packages.default = pkgs.callPackage ./default.nix { };
-        })
-    // {
-      nixosModules.default = import ./module.nix self;
-    };
 }

module.nix

@@ -1,111 +0,0 @@
-self: { config, lib, pkgs, ... }:
-let
-  cfg = config.services.scribe;
-in
-{
-  options.services.scribe = {
-    enable = lib.mkEnableOption (lib.mdDoc "Enable or disable the Scribe service");
-
-    package = lib.mkOption {
-      type = lib.types.package;
-      default = self.packages."${pkgs.system}".default;
-      description = lib.mdDoc "Overridable attribute of the scribe package to use.";
-    };
-
-    user = lib.mkOption {
-      type = lib.types.str;
-      default = "scribe";
-      description = lib.mdDoc "User to run the Scribe service as.";
-    };
-
-    group = lib.mkOption {
-      type = lib.types.str;
-      default = "scribe";
-      description = lib.mdDoc "Group to run the Scribe service as.";
-    };
-
-    appDomain = lib.mkOption {
-      type = lib.types.str;
-      description = lib.mdDoc ''
-        The domain that Scribe is being run on. This will appear on the Scribe homepage.
-      '';
-    };
-
-    port = lib.mkOption {
-      type = lib.types.port;
-      description = lib.mdDoc "Port for the Scribe service to use.";
-    };
-
-    environmentFile = lib.mkOption {
-      type = lib.types.str;
-      description = lib.mdDoc ''
-        The path to a file containing environment varible to be set in Scribes environment.
-        This should be user to set SECRET_KEY_BASE, GITHUB_USERNAME, and GITHUB_PERSONAL_ACCESS_TOKEN. 
-        Descriptions of these settings can be found
-        [in the official docs](https://sr.ht/~edwardloveall/Scribe/#configuration).
-      '';
-    };
-  };
-
-  config = lib.mkIf cfg.enable {
-    systemd.services.scribe = {
-      description = "Scribe";
-      after = [ "network.target" ];
-      wantedBy = [ "multi-user.target" ];
-      environment = {
-        LUCKY_ENV = "production";
-        APP_DOMAIN = cfg.appDomain;
-        PORT = (toString cfg.port);
-      };
-      serviceConfig = {
-        ExecStart = "${cfg.package}/bin/scribe";
-        EnvironmentFile = cfg.environmentFile;
-        Restart = "on-failure";
-        User = cfg.user;
-        Group = cfg.group;
-        UMask = "0007";
-        ProtectSystem = "strict";
-        ProtectClock = true;
-        ProtectKernelLogs = true;
-        SystemCallArchitectures = "native";
-        ProtectHome = true;
-        ProtectProc = "noaccess";
-        MemoryDenyWriteExecute = true;
-        NoNewPrivileges = true;
-        PrivateDevices = true;
-        PrivateMounts = true;
-        PrivateTmp = true;
-        ProtectControlGroups = true;
-        ProtectHostname = true;
-        ProtectKernelModules = true;
-        ProtectKernelTunables = true;
-        RestrictNamespaces = true;
-        RestrictRealtime = true;
-        RestrictSUIDSGID = true;
-        CapabilityBoundingSet = [
-          "~CAP_SYS_PTRACE"
-          "~CAP_SYS_ADMIN"
-          "~CAP_SETGID"
-          "~CAP_SETUID"
-          "~CAP_SETPCAP"
-          "~CAP_SYS_TIME"
-          "~CAP_KILL"
-          "~CAP_SYS_PACCT"
-          "~CAP_SYS_TTY_CONFIG "
-          "~CAP_SYS_CHROOT"
-          "~CAP_SYS_BOOT"
-          "~CAP_NET_ADMIN"
-        ];
-      };
-    };
-    users.users = lib.optionalAttrs (cfg.user == "scribe") {
-      "scribe" = {
-        group = "scribe";
-        isSystemUser = true;
-      };
-    };
-    users.groups = lib.optionalAttrs (cfg.group == "scribe") {
-      "scribe" = { };
-    };
-  };
-}

package.json

@@ -1,11 +1,10 @@
 {
-  "name": "scribe-ui",
+  "license": "UNLICENSED",
-  "license": "AGPL-3.0-or-later",
   "private": true,
   "dependencies": {
     "@rails/ujs": "^6.0.0",
     "compression-webpack-plugin": "^8.0.1",
-    "laravel-mix": "^6.0.49",
+    "laravel-mix": "^6.0.28",
     "modern-normalize": "^1.1.0",
     "postcss": "^8.3.6",
     "tufte-css": "^1.8.0",
@@ -23,6 +22,5 @@
     "resolve-url-loader": "^3.1.1",
     "sass": "^1.26.10",
     "sass-loader": "^10.0.2"
-  },
+  }
-  "version": "0.0.0"
 }

public/robots.txt

@@ -1,55 +1,4 @@
-# ChatGPT-User
+# Learn more about robots.txt: https://www.robotstxt.org/robotstxt.html
-User-agent: ChatGPT-User
+User-agent: *
-Disallow: /
+# 'Disallow' with an empty value allows all paths to be crawled
-
+Disallow:
-# cohere-ai
-User-agent: cohere-ai
-Disallow: /
-
-# anthropic-ai
-User-agent: anthropic-ai
-Disallow: /
-
-# Bytespider
-User-agent: Bytespider
-Disallow: /
-
-# CCBot
-User-agent: CCBot
-Disallow: /
-
-# FacebookBot
-User-agent: FacebookBot
-Disallow: /
-
-# Google-Extended
-User-agent: Google-Extended
-Disallow: /
-
-# GPTBot
-User-agent: GPTBot
-Disallow: /
-
-# omgili
-User-agent: omgili
-Disallow: /
-
-# Amazonbot
-User-agent: Amazonbot
-Disallow: /
-
-# Applebot
-User-agent: Applebot
-Disallow: /
-
-# PerplexityBot
-User-agent: PerplexityBot
-Disallow: /
-
-# PerplexityBot
-User-agent: PerplexityBot
-Disallow: /
-
-# YouBot
-User-agent: YouBot
-Disallow: /

script/build_static

@@ -1,20 +0,0 @@
-#!/usr/bin/env bash
-
-# Exit if any subcommand fails
-set -e
-set -o pipefail
-
-todays_date=$(date "+%Y-%m-%d")
-if ! git show HEAD:src/version.cr | rg -q $todays_date; then
-  echo "Date in committed src/version.cr is not today's date ($todays_date)"
-  echo "Make sure that the file is both up to date and commited to git."
-  echo
-  echo "## src/version.cr"
-  git show HEAD:src/version.cr
-  # exit 1
-fi
-
-# Via: https://crystal-lang.org/reference/1.10/guides/static_linking.html#linux
-
-~/.docker/bin/docker run --rm -it -v $(pwd):/workspace -w /workspace crystallang/crystal:latest-alpine \
-  crystal build src/start_server.cr -o ubuntu_server --static --release

script/release

@@ -1,18 +0,0 @@
-#!/bin/sh
-
-set -e
-
-todays_date=$(date "+%Y-%m-%d")
-sed -i '' -E "s/[0-9]{4}-[0-9]{2}-[0-9]{2}/$todays_date/" src/version.cr
-
-# Delete the tag if it exists. This will show an error but it's fine
-git tag -d "$todays_date" || true
-
-echo
-echo "Bumped version to $todays_date"
-echo "Here are the commits since the last tag"
-echo "Update the Changelog"
-echo
-
-git log $(git describe --tags --abbrev=0)~..HEAD --oneline
-git tag "$todays_date"

script/system_check

@@ -17,6 +17,17 @@ if command_not_found "yarn"; then
   print_error "Yarn is not installed\n  See https://yarnpkg.com/lang/en/docs/install/ for install instructions."
 fi
 
+if command_not_found "createdb"; then
+  MSG="Please install the postgres CLI tools, then try again."
+  if is_mac; then
+    MSG="$MSG\nIf you're using Postgres.app, see https://postgresapp.com/documentation/cli-tools.html."
+  fi
+  MSG="$MSG\nSee https://www.postgresql.org/docs/current/tutorial-install.html for install instructions."
+
+  print_error "$MSG"
+fi
+
+
 ## CUSTOM PRE-BOOT CHECKS ##
 # example:
 # if command_not_running "redis-cli ping"; then

shards.nix

@@ -1,176 +0,0 @@
-{
-  authentic = {
-    owner = "luckyframework";
-    repo = "authentic";
-    rev = "v1.0.0";
-    sha256 = "0mc7xqh0zm4jg8vc1awlzr249fviiy1y40w4fvyvq959hlpd6zx4";
-  };
-  avram = {
-    owner = "luckyframework";
-    repo = "avram";
-    rev = "v1.0.0";
-    sha256 = "18w90m5iq0jy026zma05swh2am936j132fs3j730lq7x5yr8289c";
-  };
-  backtracer = {
-    owner = "sija";
-    repo = "backtracer.cr";
-    rev = "v1.2.2";
-    sha256 = "1rknyylsi14m7i77x7c3138wdw27i4f6sd78m3srw851p47bwr20";
-  };
-  cadmium_transliterator = {
-    owner = "cadmiumcr";
-    repo = "transliterator";
-    rev = "46c4c14594057dbcfaf27e7e7c8c164d3f0ce3f1";
-    sha256 = "15x9xbgybqrmqb7s5cpx3fgwysp5ld97vlvz8b196lqmyqnnp3d3";
-  };
-  cry = {
-    owner = "luckyframework";
-    repo = "cry";
-    rev = "v0.4.3";
-    sha256 = "0bcvpbi418855cq1jq911dv6r9wmg81rcvcirqrbw8fv2a093ss5";
-  };
-  crystar = {
-    owner = "naqvis";
-    repo = "crystar";
-    rev = "56db8bb9dfbd5ed6d7908353405a5fae632a6561";
-    sha256 = "0bzq7im3z3asr22wzwyj1z0m3m5aq5hh1kscp5gd8vjw192w2z2a";
-  };
-  db = {
-    owner = "crystal-lang";
-    repo = "crystal-db";
-    rev = "v0.11.0";
-    sha256 = "1ylfhpn64p72ywi39niqb179f61z08q4qd4hhjza05z18mdaghl3";
-  };
-  dexter = {
-    owner = "luckyframework";
-    repo = "dexter";
-    rev = "v0.3.4";
-    sha256 = "08fv3ns0wxkyr2rcifj3ihyaf7g4lsmfamfhdxbkdkmxa9l1z6cj";
-  };
-  exception_page = {
-    owner = "crystal-loot";
-    repo = "exception_page";
-    rev = "v0.3.0";
-    sha256 = "1w82283mgaaw1hy5xk997a1av4sxaa01ydipbxm5nb9nq7fgfydk";
-  };
-  fnv = {
-    owner = "naqvis";
-    repo = "crystal-fnv";
-    rev = "v0.1.3";
-    sha256 = "1vhy3j0ifc0rlrx5b6wbpcvjzw15k303jrz3bzvnxqvi600fvv2b";
-  };
-  habitat = {
-    owner = "luckyframework";
-    repo = "habitat";
-    rev = "v0.4.7";
-    sha256 = "0d183pnswgjwqg388zmnx7s41ai88ca96nl5cybi0z6icr5npw64";
-  };
-  html5 = {
-    owner = "naqvis";
-    repo = "crystal-html5";
-    rev = "v0.4.0";
-    sha256 = "0mr4vd4bl3a22jl8h698zrh8rz6m5lm2lcyx11055gn6fw0yq57k";
-  };
-  lucky = {
-    owner = "luckyframework";
-    repo = "lucky";
-    rev = "v1.0.0";
-    sha256 = "13by6bbgpbbbdncgj87cqy5y6z7s9zb3nr88dh3fwl5mfgygk66z";
-  };
-  lucky_cache = {
-    owner = "luckyframework";
-    repo = "lucky_cache";
-    rev = "v0.1.1";
-    sha256 = "1ic9nfmiv89q5v82ybshd9xqnwv62bv8a5n8rhmsm9cwvdhgc92x";
-  };
-  lucky_env = {
-    owner = "luckyframework";
-    repo = "lucky_env";
-    rev = "v0.1.4";
-    sha256 = "0rcz0kh9rkypgm34r7maqqmgirxblhwzycwxpp0y9ai68lq71qxk";
-  };
-  lucky_flow = {
-    owner = "luckyframework";
-    repo = "lucky_flow";
-    rev = "v0.9.0";
-    sha256 = "1gyxba7lbjhzbd7a5hcswr3i04mz6rqypihhpgx213aa2685c0mw";
-  };
-  lucky_router = {
-    owner = "luckyframework";
-    repo = "lucky_router";
-    rev = "v0.5.2";
-    sha256 = "1gl93rijnbaqybpry19rn951kbx1q1bb5w0npdp4fm0r212b3yh8";
-  };
-  lucky_task = {
-    owner = "luckyframework";
-    repo = "lucky_task";
-    rev = "v0.1.1";
-    sha256 = "0w0rnf22pvj3lp5z8c4sshzwhqgwpbjpm7nry9mf0iz3fa0v48f7";
-  };
-  monads = {
-    owner = "alex-lairan";
-    repo = "monads";
-    rev = "v1.0.0";
-    sha256 = "0wwhsmnzsmw03dn2j4n75sprp4baxg24i1hn1xhfzz9b33rmlxxf";
-  };
-  pg = {
-    owner = "will";
-    repo = "crystal-pg";
-    rev = "v0.26.0";
-    sha256 = "04fwbgrlf2nzma0p2c8ki7p8sk113jhziq2al3ivif2lpmhr39fy";
-  };
-  pulsar = {
-    owner = "luckyframework";
-    repo = "pulsar";
-    rev = "v0.2.3";
-    sha256 = "03pp0r1klqk49fkzjwg9mnxqplv6pdfjn6a1p59f2w1ha5piyy90";
-  };
-  selenium = {
-    owner = "matthewmcgarvey";
-    repo = "selenium.cr";
-    rev = "v0.10.0";
-    sha256 = "062baqafz2rn9czaj8wl2b1l7ngxdph2j8xcr088f2kd8bb0hj7v";
-  };
-  shell-table = {
-    owner = "luckyframework";
-    repo = "shell-table.cr";
-    rev = "v0.9.3";
-    sha256 = "046vymm2r37c6j5bqyjzxdgg5h62slsannzvfhbckkv2r9chwd3w";
-  };
-  splay_tree_map = {
-    owner = "wyhaines";
-    repo = "splay_tree_map.cr";
-    rev = "v0.2.2";
-    sha256 = "0196zpg0v190dv23mwnbia35znxz2j2g8dqynd2b8827qiwmz1vn";
-  };
-  teeplate = {
-    owner = "luckyframework";
-    repo = "teeplate";
-    rev = "v0.8.5";
-    sha256 = "1kr05qrp674rph1324wry57gzvgvcvlz0w27brlvdgd3gi4s8sdj";
-  };
-  webdrivers = {
-    owner = "matthewmcgarvey";
-    repo = "webdrivers.cr";
-    rev = "v0.4.1";
-    sha256 = "05q6z1rv29hrwq77wpas2ki4alwhx4fpallb94q4m9g5h5vfn6ag";
-  };
-  webless = {
-    owner = "matthewmcgarvey";
-    repo = "webless";
-    rev = "v0.1.0";
-    sha256 = "0fg79wy3fq0af77jm121pqfm43dzb7l5rlx13vrl74pgqagms0ih";
-  };
-  wordsmith = {
-    owner = "luckyframework";
-    repo = "wordsmith";
-    rev = "v0.4.0";
-    sha256 = "13fsmwdh431smbmsv869pa8p34g1hqd84za33xsymsycq5459xq2";
-  };
-  xpath2 = {
-    owner = "naqvis";
-    repo = "crystal-xpath2";
-    rev = "v0.1.3";
-    sha256 = "17jl0br2fibc22sz9qdpsqd17rsmnar0jwh4iq25y8rg64pgb1h0";
-  };
-}

shell.nix

@@ -1,16 +1,17 @@
 { pkgs ? import <nixpkgs> { } }:
 
 pkgs.mkShell {
+  shellHook = ''
+    export PKG_CONFIG_PATH=${pkgs.openssl.dev}/lib/pkgconfig
+  '';
   buildInputs = with pkgs; [
     crystal
     lucky-cli
     overmind
     nodejs
-    openssl
+    openssl.dev
-    pkg-config
+    postgresql
     shards
     yarn
-    crystal2nix
-    pcre
   ];
 }

src/version.cr

@@ -1,3 +1,3 @@
 module Scribe
-  VERSION = "2023-12-18"
+  VERSION = "2023-05-21"
 end