update mastodon

nixos/pkgs/glitch-soc/default.nix

@@ -1,39 +1,27 @@
-{ lib
+{ lib, stdenv, nodejs-slim, bundlerEnv, nixosTests
-, stdenv
+, yarn, callPackage, imagemagick, ffmpeg, file, ruby, writeShellScript
-, nodejs-slim
+, fetchYarnDeps, prefetch-yarn-deps
-, bundlerEnv
+, brotli
-, nixosTests
-, yarn
-, callPackage
-, imagemagick
-, ffmpeg
-, file
-, ruby_3_0
-, writeShellScript
-, fetchYarnDeps
-, fixup_yarn_lock
 
   # Allow building a fork or custom version of Mastodon:
 , pname ? "mastodon"
-, version ? import ./version.nix
+, version ? srcOverride.version
-, srcOverride ? null
+, patches ? []
-, dependenciesDir ? ./. # Should contain gemset.nix, yarn.nix and package.json.
+  # src is a package
+, srcOverride ? callPackage ./source.nix { inherit patches; }
+, gemset ? ./. + "/gemset.nix"
+, yarnHash ? srcOverride.yarnHash
 }:
 
 stdenv.mkDerivation rec {
   inherit pname version;
 
-  # Using overrideAttrs on src does not build the gems and modules with the overridden src.
+  src = srcOverride;
-  # Putting the callPackage up in the arguments list also does not work.
-  src =
-    if srcOverride != null then srcOverride else callPackage ./source.nix { };
 
-  mastodon-gems = bundlerEnv {
+  mastodonGems = bundlerEnv {
     name = "${pname}-gems-${version}";
-    inherit version;
+    inherit version gemset ruby;
-    ruby = ruby_3_0;
     gemdir = src;
-    gemset = dependenciesDir + "/gemset.nix";
     # This fix (copied from https://github.com/NixOS/nixpkgs/pull/76765) replaces the gem
     # symlinks with directories, resolving this error when running rake:
     #   /nix/store/451rhxkggw53h7253izpbq55nrhs7iv0-mastodon-gems-3.0.1/lib/ruby/gems/2.6.0/gems/bundler-1.17.3/lib/bundler/settings.rb:6:in `<module:Bundler>': uninitialized constant Bundler::Settings (NameError)
@@ -48,33 +36,28 @@ stdenv.mkDerivation rec {
     '';
   };
 
-  mastodon-modules = stdenv.mkDerivation {
+  mastodonModules = stdenv.mkDerivation {
     pname = "${pname}-modules";
     inherit src version;
 
     yarnOfflineCache = fetchYarnDeps {
       yarnLock = "${src}/yarn.lock";
-      sha256 = "sha256-iSqMe4B9ZEFPAw7uXgnbTWBnXg89WM/3gnDbQAUJv2E=";
+      hash = yarnHash;
-      # sha256 = lib.fakeSha256;
     };
 
-    nativeBuildInputs = [
+    nativeBuildInputs = [ prefetch-yarn-deps nodejs-slim yarn mastodonGems mastodonGems.wrappedRuby brotli ];
-      fixup_yarn_lock
-      nodejs-slim
-      yarn
-      mastodon-gems
-      mastodon-gems.wrappedRuby
-    ];
 
     RAILS_ENV = "production";
     NODE_ENV = "production";
 
     buildPhase = ''
+      runHook preBuild
+
       export HOME=$PWD
       # This option is needed for openssl-3 compatibility
       # Otherwise we encounter this upstream issue: https://github.com/mastodon/mastodon/issues/17924
       export NODE_OPTIONS=--openssl-legacy-provider
-      fixup_yarn_lock ~/yarn.lock
+      fixup-yarn-lock ~/yarn.lock
       yarn config --offline set yarn-offline-mirror $yarnOfflineCache
       yarn install --offline --frozen-lockfile --ignore-engines --ignore-scripts --no-progress
 
@@ -88,61 +71,99 @@ stdenv.mkDerivation rec {
         rails assets:precompile
       yarn cache clean --offline
       rm -rf ~/node_modules/.cache
+
+      # Create missing static gzip and brotli files
+      gzip --best --keep ~/public/assets/500.html
+      gzip --best --keep ~/public/packs/report.html
+      find ~/public/assets -maxdepth 1 -type f -name '.*.json' \
+        -exec gzip --best --keep --force {} ';'
+      brotli --best --keep ~/public/packs/report.html
+      find ~/public/assets -type f -regextype posix-extended -iregex '.*\.(css|js|json|html)' \
+        -exec brotli --best --keep {} ';'
+
+      runHook postBuild
     '';
 
     installPhase = ''
+      runHook preInstall
+
       mkdir -p $out/public
       cp -r node_modules $out/node_modules
       cp -r public/assets $out/public
       cp -r public/packs $out/public
+
+      runHook postInstall
     '';
   };
 
-  propagatedBuildInputs = [ imagemagick ffmpeg file mastodon-gems.wrappedRuby ];
+  propagatedBuildInputs = [ imagemagick ffmpeg file mastodonGems.wrappedRuby ];
-  buildInputs = [ mastodon-gems nodejs-slim ];
+  buildInputs = [ mastodonGems nodejs-slim ];
 
   buildPhase = ''
-    ln -s ${mastodon-modules}/node_modules node_modules
+    runHook preBuild
-    ln -s ${mastodon-modules}/public/assets public/assets
+
-    ln -s ${mastodon-modules}/public/packs public/packs
+    ln -s $mastodonModules/node_modules node_modules
+    ln -s $mastodonModules/public/assets public/assets
+    ln -s $mastodonModules/public/packs public/packs
 
     patchShebangs bin/
-    for b in $(ls ${mastodon-gems}/bin/)
+    for b in $(ls $mastodonGems/bin/)
     do
       if [ ! -f bin/$b ]; then
-        ln -s ${mastodon-gems}/bin/$b bin/$b
+        ln -s $mastodonGems/bin/$b bin/$b
       fi
     done
 
+    # Remove execute permissions
+    chmod 0444 public/emoji/*.svg
+
+    # Create missing static gzip and brotli files
+    find public -maxdepth 1 -type f -regextype posix-extended -iregex '.*\.(css|js|svg|txt|xml)' \
+      -exec gzip --best --keep --force {} ';' \
+      -exec brotli --best --keep {} ';'
+    find public/emoji -type f -name '.*.svg' \
+      -exec gzip --best --keep --force {} ';' \
+      -exec brotli --best --keep {} ';'
+    ln -s assets/500.html.gz public/500.html.gz
+    ln -s assets/500.html.br public/500.html.br
+    ln -s packs/sw.js.gz public/sw.js.gz
+    ln -s packs/sw.js.br public/sw.js.br
+    ln -s packs/sw.js.map.gz public/sw.js.map.gz
+    ln -s packs/sw.js.map.br public/sw.js.map.br
+
     rm -rf log
     ln -s /var/log/mastodon log
     ln -s /tmp tmp
+
+    runHook postBuild
   '';
 
-  installPhase =
+  installPhase = let
-    let
+    run-streaming = writeShellScript "run-streaming.sh" ''
-      run-streaming = writeShellScript "run-streaming.sh" ''
+      # NixOS helper script to consistently use the same NodeJS version the package was built with.
-        # NixOS helper script to consistently use the same NodeJS version the package was built with.
+      ${nodejs-slim}/bin/node ./streaming
-        ${nodejs-slim}/bin/node ./streaming
-      '';
-    in
-    ''
-      mkdir -p $out
-      cp -r * $out/
-      ln -s ${run-streaming} $out/run-streaming.sh
     '';
+  in ''
+    runHook preInstall
+
+    mkdir -p $out
+    cp -r * $out/
+    ln -s ${run-streaming} $out/run-streaming.sh
+
+    runHook postInstall
+  '';
 
   passthru = {
     tests.mastodon = nixosTests.mastodon;
-    updateScript = callPackage ./update.nix { };
+    # run with: nix-shell ./maintainers/scripts/update.nix --argstr package mastodon
+    updateScript = ./update.sh;
   };
 
   meta = with lib; {
-    description =
+    description = "Self-hosted, globally interconnected microblogging software based on ActivityPub";
-      "Self-hosted, globally interconnected microblogging software based on ActivityPub";
     homepage = "https://joinmastodon.org";
     license = licenses.agpl3Plus;
     platforms = [ "x86_64-linux" "i686-linux" "aarch64-linux" ];
-    maintainers = with maintainers; [ happy-river erictapen izorkin ];
+    maintainers = with maintainers; [ happy-river erictapen izorkin ghuntley ];
   };
 }

nixos/pkgs/glitch-soc/source.nix

@@ -1,13 +1,18 @@
 # This file was generated by pkgs.mastodon.updateScript.
-{ fetchgit, applyPatches }:
+{ fetchFromGitHub, applyPatches, patches ? [] }:
 let
-  src = fetchgit {
+  version = "0e562916cce3241d98bd10f04a6aa7419700605";
-    url = "https://github.com/glitch-soc/mastodon.git";
-    rev = "a1df9fdb06854bd55f018918236132ccfa7d9d84";
-    sha256 = "0amqiigq3qgag6qm119aaysmd2k93vwgr1aynxjxmbpn35ykcay3";
-  };
 in
-applyPatches {
+(
-  inherit src;
+  applyPatches {
-  patches = [ ];
+    src = fetchFromGitHub {
+      owner = "glitch-soc";
+      repo = "mastodon";
+      rev = "v${version}";
+      hash = "sha256-fZH3zPEU5jnYFhLx8OKDNrvsSVT46Peu92L84Fg5YpQ=";
+    };
+    patches = patches ++ [];
+  }) // {
+  inherit version;
+  yarnHash = "sha256-P7KswzsCusyiS4MxUFnC1HYMTQ6fLpIwd97AglCukIk=";
 }

nixos/pkgs/glitch-soc/update.nix

@@ -1,38 +0,0 @@
-{ runCommand
-, lib
-, makeWrapper
-, yarn2nix
-, bundix
-, coreutils
-, diffutils
-, nix-prefetch-git
-, gnused
-, jq
-}:
-let
-  binPath = lib.makeBinPath [
-    yarn2nix
-    bundix
-    coreutils
-    diffutils
-    nix-prefetch-git
-    gnused
-    jq
-  ];
-in
-runCommand "mastodon-update-script"
-{
-  nativeBuildInputs = [ makeWrapper ];
-
-  meta = {
-    maintainers = with lib.maintainers; [ happy-river ];
-    description =
-      "Utility to generate Nix expressions for Mastodon's dependencies";
-    platforms = lib.platforms.unix;
-  };
-} ''
-  mkdir -p $out/bin
-  cp ${./update.sh} $out/bin/update.sh
-  patchShebangs $out/bin/update.sh
-  wrapProgram $out/bin/update.sh --prefix PATH : ${binPath}
-''

nixos/pkgs/glitch-soc/update.sh

@@ -1,15 +1,22 @@
-#!/usr/bin/env bash
+#!/usr/bin/env nix-shell
+#! nix-shell -i bash -p bundix coreutils diffutils nix-prefetch-github gnused jq prefetch-yarn-deps
 set -e
 
-URL=https://github.com/mastodon/mastodon.git
+OWNER=mastodon
+REPO=mastodon
 
 POSITIONAL=()
 while [[ $# -gt 0 ]]; do
     key="$1"
 
     case $key in
-        --url)
+        --owner)
-            URL="$2"
+            OWNER="$2"
+            shift # past argument
+            shift # past value
+            ;;
+        --repo)
+            REPO="$2"
             shift # past argument
             shift # past value
             ;;
@@ -35,29 +42,28 @@ while [[ $# -gt 0 ]]; do
     esac
 done
 
-if [[ -z "$VERSION" || -n "$POSITIONAL" ]]; then
+if [[ -n "$POSITIONAL" ]]; then
-    echo "Usage: update.sh [--url URL] --ver VERSION [--rev REVISION] [--patches PATCHES]"
+    echo "Usage: update.sh [--owner OWNER] [--repo REPO] [--ver VERSION] [--rev REVISION] [--patches PATCHES]"
-    echo "URL may be any path acceptable to 'git clone' and VERSION the"
+    echo "OWNER and REPO must be paths on github."
-    echo "semantic version number.  If VERSION is not a revision acceptable to"
+    echo "If REVISION is not provided, the latest tag from github.com/mastodon/mastodon is fetched and VERSION is calculated from it."
-    echo "'git checkout', you must provide one in REVISION.  If URL is not"
+    echo "If OWNER and REPO are not provided, it defaults they default to mastodon and mastodon."
-    echo "provided, it defaults to https://github.com/mastodon/mastodon.git."
+    echo "PATCHES, if provided, should be one or more Nix expressions separated by spaces."
-    echo "PATCHES, if provided, should be one or more Nix expressions"
-    echo "separated by spaces."
     exit 1
 fi
 
 if [[ -z "$REVISION" ]]; then
-    REVISION="$VERSION"
+    REVISION="$(curl ${GITHUB_TOKEN:+" -u \":$GITHUB_TOKEN\""} -s "https://api.github.com/repos/$OWNER/$REPO/releases" | jq -r  'map(select(.prerelease == false)) | .[0].tag_name')"
 fi
 
-rm -f gemset.nix version.nix source.nix
+VERSION="$(echo "$REVISION" | cut -c2-)"
-TARGET_DIR="$PWD"
 
+rm -f gemset.nix source.nix
+cd "$(dirname "${BASH_SOURCE[0]}")" || exit 1
 
 WORK_DIR=$(mktemp -d)
 
 # Check that working directory was created.
-if [[ ! "$WORK_DIR" || ! -d "$WORK_DIR" ]]; then
+if [[ -z "$WORK_DIR" || ! -d "$WORK_DIR" ]]; then
     echo "Could not create temporary directory"
     exit 1
 fi
@@ -70,29 +76,37 @@ function cleanup {
 }
 trap cleanup EXIT
 
-echo "Fetching source code $REVISION from $URL"
+echo "Fetching source code $REVISION"
-JSON=$(nix-prefetch-git --url "$URL" --rev "$REVISION"  2> $WORK_DIR/nix-prefetch-git.out)
+JSON=$(nix-prefetch-github "$OWNER" "$REPO" --rev "$REVISION" 2> $WORK_DIR/nix-prefetch-git.out)
-SHA=$(echo $JSON | jq -r .sha256)
+HASH=$(echo "$JSON" | jq -r .hash)
-FETCHED_SOURCE_DIR=$(grep '^path is' $WORK_DIR/nix-prefetch-git.out | sed 's/^path is //')
-
-echo "Creating version.nix"
-echo \"$VERSION\" | sed 's/^"v/"/' > version.nix
 
 cat > source.nix << EOF
 # This file was generated by pkgs.mastodon.updateScript.
-{ fetchgit, applyPatches }: let
+{ fetchFromGitHub, applyPatches, patches ? [] }:
-  src = fetchgit {
+let
-    url = "$URL";
+  version = "$VERSION";
-    rev = "$REVISION";
+in
-    sha256 = "$SHA";
+(
-  };
+  applyPatches {
-in applyPatches {
+    src = fetchFromGitHub {
-  inherit src;
+      owner = "$OWNER";
-  patches = [$PATCHES];
+      repo = "$REPO";
+      rev = "v\${version}";
+      hash = "$HASH";
+    };
+    patches = patches ++ [$PATCHES];
+  }) // {
+  inherit version;
+  yarnHash = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=";
 }
 EOF
 SOURCE_DIR="$(nix-build --no-out-link -E '(import <nixpkgs> {}).callPackage ./source.nix {}')"
 
 echo "Creating gemset.nix"
 bundix --lockfile="$SOURCE_DIR/Gemfile.lock" --gemfile="$SOURCE_DIR/Gemfile"
-echo "" >> $TARGET_DIR/gemset.nix  # Create trailing newline to please EditorConfig checks
+echo "" >> gemset.nix  # Create trailing newline to please EditorConfig checks
+
+echo "Creating yarn-hash.nix"
+YARN_HASH="$(prefetch-yarn-deps "$SOURCE_DIR/yarn.lock")"
+YARN_HASH="$(nix hash to-sri --type sha256 "$YARN_HASH")"
+sed -i "s/sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=/$YARN_HASH/g" source.nix

nixos/pkgs/glitch-soc/version.nix

@@ -1 +0,0 @@
-"4.1.7"