update glitch

nixos/pkgs/glitch-soc/README.md

@@ -2,16 +2,10 @@
 
 <https://github.com/glitch-soc/mastodon>
 
-Mostly copied from [nixpkgs upstream](https://github.com/NixOS/nixpkgs/tree/master/pkgs/servers/mastodon).
+Based on [nixpkgs upstream](https://github.com/NixOS/nixpkgs/tree/master/pkgs/servers/mastodon).
 
-Modifications for the new yarn berry lockfiles in `default.nix`, `yarn.nix` and `yarn-typescript.patch` stolen (with permissions) from [catgirl.cloud](https://git.catgirl.cloud/999eagle/dotfiles-nix/-/tree/main/overlay/mastodon/glitch).
+Modifications for the new yarn berry lockfiles and some other improvements stolen and adjusted (with permissions) from [catgirl.cloud](https://git.catgirl.cloud/999eagle/dotfiles-nix/-/tree/main/overlay/mastodon/glitch) (see also https://github.com/NixOS/nixpkgs/issues/277697).
 
-See also: https://github.com/NixOS/nixpkgs/issues/277697
+I've also made some further modifications myself to try and simplify the package and better understand it.
 
-Update:
+The package can be updated to the latest glitch-soc commit with `./update.sh`. The yarn hash in `version_data.nix` has to be adjusted manually afterwards from the error message though.
-
-```
-./update.sh --owner glitch-soc --repo mastodon --patches "./yarn-typescript.patch" --rev $COMMIT
-```
-
-The yarn hash isn't updated automatically due to the lockfile thing, run a build and copy the hash from the error message to source.nix by hand.

nixos/pkgs/glitch-soc/default.nix

@@ -1,63 +1,81 @@
-/*
+{ lib, stdenv, stdenvNoCC, nodejs-slim, bundlerEnv
-Copied from nixpkgs upstream:
-https://github.com/NixOS/nixpkgs/blob/nixos-23.11/pkgs/servers/mastodon/default.nix
-
-Modifications for new yarn lockfiles stolen (with permission) from:
-https://git.catgirl.cloud/999eagle/dotfiles-nix/-/blob/main/overlay/mastodon/glitch/yarn.nix
-*/
-
-{ lib, stdenv, nodejs-slim, bundlerEnv, nixosTests
 , yarn-berry, callPackage, imagemagick, ffmpeg, file, ruby, writeShellScript
-, brotli
+, brotli, cacert
-
-  # Allow building a fork or custom version of Mastodon:
-, pname ? "mastodon"
-, version ? srcOverride.version
-, patches ? []
-  # src is a package
-, srcOverride ? callPackage ./source.nix { inherit patches; }
-, gemset ? ./. + "/gemset.nix"
-, yarnHash ? srcOverride.yarnHash
 }:
 
-stdenv.mkDerivation rec {
+let
-  inherit pname version;
 
-  src = srcOverride;
+  # optimally, updates only need to touch `version_data.nix`, and nothing else should be in there
+  versionData = import ./version_data.nix;
 
+  # use the first 7 characters of the glitch-soc commit hash as version string
+  version = builtins.substring 0 7 versionData.rev;
+
+  # the patched glitch-soc source
+  src = callPackage ./source.nix { };
+
+  # ruby gems, built from `gemset.nix`, which is generated by bundix in `update.sh` from the source Gemfile
   mastodonGems = bundlerEnv {
-    name = "${pname}-gems-${version}";
+    name = "glitch-soc-gems-${version}";  # bundlerEnv breaks when pname is set instead
-    inherit version gemset ruby;
+    inherit version ruby;
+    gemset = ./gemset.nix;
     gemdir = src;
-    # This fix (copied from https://github.com/NixOS/nixpkgs/pull/76765) replaces the gem
-    # symlinks with directories, resolving this error when running rake:
-    #   /nix/store/451rhxkggw53h7253izpbq55nrhs7iv0-mastodon-gems-3.0.1/lib/ruby/gems/2.6.0/gems/bundler-1.17.3/lib/bundler/settings.rb:6:in `<module:Bundler>': uninitialized constant Bundler::Settings (NameError)
-    postBuild = ''
-      for gem in "$out"/lib/ruby/gems/*/gems/*; do
-        cp -a "$gem/" "$gem.new"
-        rm "$gem"
-        # needed on macOS, otherwise the mv yields permission denied
-        chmod +w "$gem.new"
-        mv "$gem.new" "$gem"
-      done
-    '';
   };
 
+  # fetches JS dependencies via yarn based on the lockfile in the source
+  mastodonYarnDeps = stdenvNoCC.mkDerivation {
+    pname = "glitch-soc-yarn-deps";
+    inherit version src;
+
+    nativeBuildInputs = [ yarn-berry cacert ];
+
+    dontInstall = true;
+
+    NODE_EXTRA_CA_CERTS = "${cacert}/etc/ssl/certs/ca-bundle.crt";
+
+    buildPhase = ''
+      mkdir -p $out
+
+      export HOME=$(mktemp -d)
+      echo $HOME
+
+      export YARN_ENABLE_TELEMETRY=0
+      export YARN_COMPRESSION_LEVEL=0
+
+      cache="$(yarn config get cacheFolder)"
+      yarn install --immutable --mode skip-build
+
+      cp -r $cache/* $out/
+    '';
+
+    outputHashAlgo = "sha256";
+    outputHash = versionData.yarnHash;
+    outputHashMode = "recursive";
+  };
+
+  # builds the node modules for mastodon using the previously fetched yarn deps
   mastodonModules = stdenv.mkDerivation {
-    pname = "${pname}-modules";
+    pname = "glitch-soc-modules";
-    inherit src version;
+    inherit version src;
 
-    # use the fixed yarn berry offline cache thingy
+    yarnOfflineCache = mastodonYarnDeps;
-    yarnOfflineCache = callPackage ./yarn.nix {
-      inherit src;
-      hash = yarnHash;
-    };
 
-    nativeBuildInputs = [ nodejs-slim yarn-berry mastodonGems mastodonGems.wrappedRuby brotli ];
+    nativeBuildInputs = [ nodejs-slim yarn-berry brotli mastodonGems mastodonGems.wrappedRuby ];
 
     RAILS_ENV = "production";
     NODE_ENV = "production";
 
+    /*
+    So it seems that somehow a change in Linux 6.9 changed something that broke libuv, an IO lib
+    used by Node. This undocumented env var disables the broken IO feature in libuv and it works
+    again.
+
+    - https://lore.kernel.org/lkml/d7003b6e-b8e3-41c4-9e6e-2b9abd0c5572@gmail.com/t/
+    - https://github.com/nodejs/node/issues/53051#issuecomment-2124940205
+    - https://github.com/nodejs/docker-node/issues/1912#issuecomment-1594233686
+    */
+    UV_USE_IO_URING = "0";
+
     buildPhase = ''
       runHook preBuild
 
@@ -70,7 +88,8 @@ stdenv.mkDerivation rec {
       mkdir -p ~/.yarn/berry
       ln -sf $yarnOfflineCache ~/.yarn/berry/cache
 
-      yarn install --immutable --immutable-cache
+      # --inline-builds prints build logs inline so they can be inspected with nix log
+      yarn install --immutable --immutable-cache --inline-builds
 
       patchShebangs ~/bin
       patchShebangs ~/node_modules
@@ -78,8 +97,13 @@ stdenv.mkDerivation rec {
       # skip running yarn install
       rm -rf ~/bin/yarn
 
-      OTP_SECRET=precompile_placeholder SECRET_KEY_BASE=precompile_placeholder \
+      OTP_SECRET=precompile_placeholder \
+      SECRET_KEY_BASE=precompile_placeholder \
+      ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY=precompile_placeholder \
+      ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT=precompile_placeholder \
+      ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY=precompile_placeholder \
         rails assets:precompile
+
       yarn cache clean
       rm -rf ~/node_modules/.cache
 
@@ -107,7 +131,15 @@ stdenv.mkDerivation rec {
     '';
   };
 
-  propagatedBuildInputs = [ imagemagick ffmpeg file mastodonGems.wrappedRuby ];
+# the actual main glitch-soc package
+
+in stdenv.mkDerivation {
+
+  pname = "glitch-soc";
+  inherit version src mastodonGems mastodonModules;
+
+  propagatedBuildInputs = [ mastodonGems.wrappedRuby ];
+  nativeBuildInputs = [ brotli ];
   buildInputs = [ mastodonGems nodejs-slim ];
 
   buildPhase = ''
@@ -164,17 +196,4 @@ stdenv.mkDerivation rec {
     runHook postInstall
   '';
 
-  passthru = {
-    tests.mastodon = nixosTests.mastodon;
-    # run with: nix-shell ./maintainers/scripts/update.nix --argstr package mastodon
-    updateScript = ./update.sh;
-  };
-
-  meta = with lib; {
-    description = "Self-hosted, globally interconnected microblogging software based on ActivityPub";
-    homepage = "https://joinmastodon.org";
-    license = licenses.agpl3Plus;
-    platforms = [ "x86_64-linux" "i686-linux" "aarch64-linux" ];
-    maintainers = with maintainers; [ happy-river erictapen izorkin ghuntley ];
-  };
 }

nixos/pkgs/glitch-soc/source.nix

@@ -1,19 +1,22 @@
-# This file was generated by pkgs.mastodon.updateScript.
+/*
-{ fetchFromGitHub, applyPatches, patches ? [] }:
+This fetches the glitch-soc source from GitHub and patches it.
+
+This needs to be a separately buildable package so that update.sh can build it during upgrading,
+because it needs it for generating `gemset.nix` from the Gemfile in the source.
+*/
+
+{
+  applyPatches,
+  fetchFromGitHub,
+}:
+
 let
-  version = "d7d4770";
+  versionData = import ./version_data.nix;
-  revision = "d7d477047eba7cb88df54dd78f42095ed0fbea76";
+in applyPatches {
-in
+  src = fetchFromGitHub {
-(
+    owner = "glitch-soc";
-  applyPatches {
+    repo = "mastodon";
-    src = fetchFromGitHub {
+    inherit (versionData) rev hash;
-      owner = "glitch-soc";
+  };
-      repo = "mastodon";
+  patches = [];
-      rev = "${revision}";
-      hash = "sha256-x1fqDtCOiNS61EhnpObUuxrdTd5n2mhjoGbIYGivbDg=";
-    };
-    patches = patches ++ [./yarn-typescript.patch];
-  }) // {
-  inherit version;
-  yarnHash = "sha256-CIIz5wwWzvDKc/VbSIT7Z5D9kwOLoErXoO0WQWfV/g4=";
 }

nixos/pkgs/glitch-soc/update.sh

@@ -1,108 +1,42 @@
-#!/usr/bin/env -S nix shell nixpkgs#bundix nixpkgs#coreutils nixpkgs#diffutils nixpkgs#nix-prefetch-git nixpkgs#nix-prefetch-github nixpkgs#gnused nixpkgs#jq nixpkgs#prefetch-yarn-deps -c bash
+#!/usr/bin/env -S nix shell nixpkgs#coreutils nixpkgs#bundix nixpkgs#nix-prefetch-github nixpkgs#jq -c bash
+
 set -e
 
-OWNER=mastodon
+cd "$(dirname "$0")"  # cd to the script's directory
-REPO=mastodon
 
-POSITIONAL=()
+echo "Retrieving latest glitch-soc commit..."
-while [[ $# -gt 0 ]]; do
+commit="$(curl -SsL 'https://api.github.com/repos/glitch-soc/mastodon/branches/main')"
-    key="$1"
+rev="$(jq -r '.commit.sha' <<<"$commit")"
+echo "Latest commit is $rev."
 
-    case $key in
+echo
-        --owner)
+echo "Prefetching source..."
-            OWNER="$2"
+hash="$(nix-prefetch-github glitch-soc mastodon --rev "$rev" | jq -r '.hash')"
-            shift # past argument
-            shift # past value
-            ;;
-        --repo)
-            REPO="$2"
-            shift # past argument
-            shift # past value
-            ;;
-        --ver)
-            VERSION="$2"
-            shift # past argument
-            shift # past value
-            ;;
-        --rev)
-            REVISION="$2"
-            shift # past argument
-            shift # past value
-            ;;
-        --patches)
-            PATCHES="$2"
-            shift # past argument
-            shift # past value
-            ;;
-        *)  # unknown option
-            POSITIONAL+=("$1")
-            shift # past argument
-            ;;
-    esac
-done
 
-if [[ -n "$POSITIONAL" ]]; then
+echo
-    echo "Usage: update.sh [--owner OWNER] [--repo REPO] [--ver VERSION] [--rev REVISION] [--patches PATCHES]"
+echo "Generating version_data.nix..."
-    echo "OWNER and REPO must be paths on github."
+cat > version_data.nix << EOF
-    echo "If REVISION is not provided, the latest tag from github.com/mastodon/mastodon is fetched and VERSION is calculated from it."
+# This file was generated with update.sh.
-    echo "If OWNER and REPO are not provided, it defaults they default to mastodon and mastodon."
+{
-    echo "PATCHES, if provided, should be one or more Nix expressions separated by spaces."
+  rev = "$rev";
-    exit 1
+  hash = "$hash";
-fi
+  yarnHash = "";
-
-VERSION="${REVISION:0:7}"
-
-rm -f gemset.nix source.nix
-cd "$(dirname "${BASH_SOURCE[0]}")" || exit 1
-
-WORK_DIR=$(mktemp -d)
-
-# Check that working directory was created.
-if [[ -z "$WORK_DIR" || ! -d "$WORK_DIR" ]]; then
-    echo "Could not create temporary directory"
-    exit 1
-fi
-
-# Delete the working directory on exit.
-function cleanup {
-    # Report errors, if any, from nix-prefetch-git
-    grep "fatal" $WORK_DIR/nix-prefetch-git.out >/dev/stderr || true
-    rm -rf "$WORK_DIR"
-}
-trap cleanup EXIT
-
-echo "Fetching source code $REVISION"
-JSON=$(nix-prefetch-github "$OWNER" "$REPO" --rev "$REVISION" 2> $WORK_DIR/nix-prefetch-git.out)
-HASH=$(echo "$JSON" | jq -r .hash)
-
-cat > source.nix << EOF
-# This file was generated by pkgs.mastodon.updateScript.
-{ fetchFromGitHub, applyPatches, patches ? [] }:
-let
-  version = "$VERSION";
-  revision = "$REVISION";
-in
-(
-  applyPatches {
-    src = fetchFromGitHub {
-      owner = "$OWNER";
-      repo = "$REPO";
-      rev = "\${revision}";
-      hash = "$HASH";
-    };
-    patches = patches ++ [$PATCHES];
-  }) // {
-  inherit version;
-  yarnHash = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=";
 }
 EOF
-SOURCE_DIR="$(nix-build --no-out-link -E '(import <nixpkgs> {}).callPackage ./source.nix {}')"
 
-echo "Creating gemset.nix"
+echo
-bundix --lockfile="$SOURCE_DIR/Gemfile.lock" --gemfile="$SOURCE_DIR/Gemfile"
+echo "Building source derivation..."
-echo "" >> gemset.nix  # Create trailing newline to please EditorConfig checks
+srcdir="$(nix build --no-link --print-out-paths --no-warn-dirty ../..#glitch-soc-source)"
+echo "Source derivation is $srcdir."
 
-# echo "Creating yarn-hash.nix"
+echo
-# YARN_HASH="$(prefetch-yarn-deps "$SOURCE_DIR/yarn.lock")"
+echo "Generating gemset.nix using built source derivation..."
-# YARN_HASH="$(nix hash to-sri --type sha256 "$YARN_HASH")"
+rm -f gemset.nix
-# sed -i "s/sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=/$YARN_HASH/g" source.nix
+bundix --quiet --lockfile $srcdir/Gemfile.lock --gemfile $srcdir/Gemfile
+echo "" >> gemset.nix
+
+echo
+echo "Done."
+
+echo
+echo "You'll have to manually enter the commit hash for the yarn deps from the error message when first trying to build the package."

nixos/pkgs/glitch-soc/version_data.nix

@@ -0,0 +1,6 @@
+# This file was generated with update.sh.
+{
+  rev = "60c2310fd879885755c620b060828e3d6a560e0b";
+  hash = "sha256-BiRtSYaBQQK6jgKkJqCaKtf2SaRDN1Sa5H5o0XKj4eQ=";
+  yarnHash = "sha256-haLT8KnJr1r4VPjeXfR5nm0yUbAbeB+D9reOXrdfwCY=";
+}

nixos/pkgs/glitch-soc/yarn-typescript.patch

@@ -1,15 +0,0 @@
---- a/yarn.lock
-+++ b/yarn.lock
-@@ -16483,11 +16483,11 @@
-
- "typescript@patch:typescript@npm%3A5#optional!builtin<compat/typescript>, typescript@patch:typescript@npm%3A^5.0.4#optional!builtin<compat/typescript>":
-   version: 5.3.3
--  resolution: "typescript@patch:typescript@npm%3A5.3.3#optional!builtin<compat/typescript>::version=5.3.3&hash=e012d7"
-+  resolution: "typescript@patch:typescript@npm%3A5.3.3#optional!builtin<compat/typescript>::version=5.3.3&hash=29ae49"
-   bin:
-     tsc: bin/tsc
-     tsserver: bin/tsserver
--  checksum: 1d0a5f4ce496c42caa9a30e659c467c5686eae15d54b027ee7866744952547f1be1262f2d40de911618c242b510029d51d43ff605dba8fb740ec85ca2d3f9500
-+  checksum: e22df47df9b2b2f2617b8bf511a29aea3d177f9f7a0756818230a76b01cbd7da988bf55f9463aaa1a4c1ff90b80f8dc5676460d4e9dfc010572cbba59b822b0c
-   languageName: node
-   linkType: hard

nixos/pkgs/glitch-soc/yarn.nix

@@ -1,40 +0,0 @@
-/*
-Stolen (with permission) from:
-https://git.catgirl.cloud/999eagle/dotfiles-nix/-/blob/main/overlay/mastodon/glitch/yarn.nix
-*/
-
-{
-  stdenvNoCC,
-  yarn-berry,
-  cacert,
-  src,
-  hash,
-}:
-stdenvNoCC.mkDerivation {
-  name = "yarn-deps";
-  nativeBuildInputs = [yarn-berry cacert];
-  inherit src;
-
-  dontInstall = true;
-
-  NODE_EXTRA_CA_CERTS = "${cacert}/etc/ssl/certs/ca-bundle.crt";
-
-  buildPhase = ''
-    mkdir -p $out
-
-    export HOME=$(mktemp -d)
-    echo $HOME
-
-    export YARN_ENABLE_TELEMETRY=0
-    export YARN_COMPRESSION_LEVEL=0
-
-    cache="$(yarn config get cacheFolder)"
-    yarn install --immutable --mode skip-build
-
-    cp -r $cache/* $out/
-  '';
-
-  outputHashAlgo = "sha256";
-  outputHash = hash;
-  outputHashMode = "recursive";
-}