nixos/modules/services/backup/syncoid.nix

   1 { config, lib, pkgs, ... }:
   2
   3 with lib;
   4
   5 let
   6   cfg = config.services.syncoid;
   7   inherit (config.networking) nftables;
   8
   9   # Extract local dataset names (so no datasets containing "@")
  10   localDatasetName = d: optionals (d != null) (
  11     let m = builtins.match "([^/@]+[^@]*)" d; in
  12     optionals (m != null) m
  13   );
  14
  15   # Escape as required by: https://www.freedesktop.org/software/systemd/man/systemd.unit.html
  16   escapeUnitName = name:
  17     concatMapStrings (s: if isList s then "-" else s)
  18       (builtins.split "[^a-zA-Z0-9_.\\-]+" name);
  19 in
  20 {
  21
  22   # Interface
  23
  24   options.services.syncoid = {
  25     enable = mkEnableOption (lib.mdDoc "Syncoid ZFS synchronization service");
  26
  27     package = lib.mkPackageOptionMD pkgs "sanoid" { };
  28
  29     nftables.enable = mkEnableOption (lib.mdDoc ''
  30       maintaining an nftables set of the active syncoid UIDs.
  31
  32       This can be used like so (assuming `output-net`
  33       is being called by the output chain):
  34       ```
  35       networking.nftables.ruleset = "table inet filter { chain output-net { skuid @nixos-syncoid-uids meta l4proto tcp accept } }";
  36       ```
  37     '');
  38
  39     interval = mkOption {
  40       type = types.str;
  41       default = "hourly";
  42       example = "*-*-* *:15:00";
  43       description = lib.mdDoc ''
  44         Run syncoid at this interval. The default is to run hourly.
  45
  46         The format is described in
  47         {manpage}`systemd.time(7)`.
  48       '';
  49     };
  50
  51     sshKey = mkOption {
  52       type = types.nullOr types.str;
  53       default = null;
  54       description = lib.mdDoc ''
  55         SSH private key file to use to login to the remote system.
  56         It can be overridden in individual commands.
  57         It is loaded using `LoadCredentialEncrypted=`
  58         when its path is prefixed by a credential name and colon,
  59         otherwise `LoadCredential=` is used.
  60         For more SSH tuning, you may use syncoid's `--sshoption`
  61         in {option}`services.syncoid.commonArgs`
  62         and/or in the `extraArgs` of a specific command.
  63       '';
  64     };
  65
  66     localSourceAllow = mkOption {
  67       type = types.listOf types.str;
  68       # Permissions snapshot and destroy are in case --no-sync-snap is not used
  69       default = [ "bookmark" "hold" "send" "snapshot" "destroy" ];
  70       description = lib.mdDoc ''
  71         Permissions granted for the syncoid user for local source datasets.
  72         See <https://openzfs.github.io/openzfs-docs/man/8/zfs-allow.8.html>
  73         for available permissions.
  74       '';
  75     };
  76
  77     localTargetAllow = mkOption {
  78       type = types.listOf types.str;
  79       default = [ "change-key" "compression" "create" "destroy" "mount" "mountpoint" "receive" "rollback" ];
  80       example = [ "create" "mount" "receive" "rollback" ];
  81       description = lib.mdDoc ''
  82         Permissions granted for the syncoid user for local target datasets.
  83         See <https://openzfs.github.io/openzfs-docs/man/8/zfs-allow.8.html>
  84         for available permissions.
  85         Make sure to include the `change-key` permission if you send raw encrypted datasets,
  86         the `compression` permission if you send raw compressed datasets, and so on.
  87         For remote target datasets you'll have to set your remote user permissions by yourself.
  88       '';
  89     };
  90
  91     commonArgs = mkOption {
  92       type = types.listOf types.str;
  93       default = [ ];
  94       example = [ "--no-sync-snap" ];
  95       description = lib.mdDoc ''
  96         Arguments to add to every syncoid command, unless disabled for that
  97         command. See
  98         <https://github.com/jimsalterjrs/sanoid/#syncoid-command-line-options>
  99         for available options.
 100       '';
 101     };
 102
 103     service = mkOption {
 104       type = types.attrs;
 105       default = { };
 106       description = lib.mdDoc ''
 107         Systemd configuration common to all syncoid services.
 108       '';
 109     };
 110
 111     commands = mkOption {
 112       type = types.attrsOf (types.submodule ({ name, ... }: {
 113         options = {
 114           source = mkOption {
 115             type = types.str;
 116             example = "pool/dataset";
 117             description = lib.mdDoc ''
 118               Source ZFS dataset. Can be either local or remote. Defaults to
 119               the attribute name.
 120             '';
 121           };
 122
 123           target = mkOption {
 124             type = types.str;
 125             example = "user@server:pool/dataset";
 126             description = lib.mdDoc ''
 127               Target ZFS dataset. Can be either local
 128               («pool/dataset») or remote
 129               («user@server:pool/dataset»).
 130             '';
 131           };
 132
 133           recursive = mkEnableOption (lib.mdDoc ''the transfer of child datasets'');
 134
 135           sshKey = mkOption {
 136             type = types.nullOr types.str;
 137             description = lib.mdDoc ''
 138               SSH private key file to use to login to the remote system.
 139               Defaults to {option}`services.syncoid.sshKey` option.
 140             '';
 141           };
 142
 143           localSourceAllow = mkOption {
 144             type = types.listOf types.str;
 145             description = lib.mdDoc ''
 146               Permissions granted for the {option}`services.syncoid.user` user
 147               for local source datasets. See
 148               <https://openzfs.github.io/openzfs-docs/man/8/zfs-allow.8.html>
 149               for available permissions.
 150               Defaults to {option}`services.syncoid.localSourceAllow` option.
 151             '';
 152           };
 153
 154           localTargetAllow = mkOption {
 155             type = types.listOf types.str;
 156             description = lib.mdDoc ''
 157               Permissions granted for the {option}`services.syncoid.user` user
 158               for local target datasets. See
 159               <https://openzfs.github.io/openzfs-docs/man/8/zfs-allow.8.html>
 160               for available permissions.
 161               Make sure to include the `change-key` permission if you send raw encrypted datasets,
 162               the `compression` permission if you send raw compressed datasets, and so on.
 163               For remote target datasets you'll have to set your remote user permissions by yourself.
 164             '';
 165           };
 166
 167           sendOptions = mkOption {
 168             type = types.separatedString " ";
 169             default = "";
 170             example = "Lc e";
 171             description = lib.mdDoc ''
 172               Advanced options to pass to zfs send. Options are specified
 173               without their leading dashes and separated by spaces.
 174             '';
 175           };
 176
 177           recvOptions = mkOption {
 178             type = types.separatedString " ";
 179             default = "";
 180             example = "ux recordsize o compression=lz4";
 181             description = lib.mdDoc ''
 182               Advanced options to pass to zfs recv. Options are specified
 183               without their leading dashes and separated by spaces.
 184             '';
 185           };
 186
 187           useCommonArgs = mkEnableOption
 188             (lib.mdDoc ''
 189               configured common arguments to this command
 190             '') // { default = true; };
 191
 192           service = mkOption {
 193             type = types.attrs;
 194             default = { };
 195             description = lib.mdDoc ''
 196               Systemd configuration specific to this syncoid service.
 197             '';
 198           };
 199
 200           extraArgs = mkOption {
 201             type = types.listOf types.str;
 202             default = [ ];
 203             example = [ "--sshport 2222" ];
 204             description = lib.mdDoc "Extra syncoid arguments for this command.";
 205           };
 206         };
 207         config = {
 208           source = mkDefault name;
 209           sshKey = mkDefault cfg.sshKey;
 210           localSourceAllow = mkDefault cfg.localSourceAllow;
 211           localTargetAllow = mkDefault cfg.localTargetAllow;
 212         };
 213       }));
 214       default = { };
 215       example = literalExpression ''
 216         {
 217           "pool/test".target = "root@target:pool/test";
 218         }
 219       '';
 220       description = lib.mdDoc "Syncoid commands to run.";
 221     };
 222   };
 223
 224   # Implementation
 225
 226   config = mkIf cfg.enable {
 227     assertions = [
 228       {
 229         assertion = cfg.nftables.enable -> config.networking.nftables.enable;
 230         message = "config.networking.nftables.enable must be set when config.services.syncoid.nftables.enable is set";
 231       }
 232     ];
 233
 234     systemd.services = mapAttrs'
 235       (name: c:
 236         let
 237           sshKeyCred = builtins.split ":" c.sshKey;
 238         in
 239         nameValuePair "syncoid-${escapeUnitName name}" (mkMerge [
 240           {
 241             description = "Syncoid ZFS synchronization from ${c.source} to ${c.target}";
 242             after = [ "zfs.target" ];
 243             startAt = cfg.interval;
 244             # Here we explicitly use the booted system to guarantee the stable API needed by ZFS.
 245             # Moreover syncoid may need zpool to get feature@extensible_dataset.
 246             path = [ "/run/booted-system/sw" ];
 247             # Prevents missing snapshots during DST changes
 248             environment.TZ = "UTC";
 249             # A custom LD_LIBRARY_PATH is needed to access in `getent passwd`
 250             # the systemd's entry about the DynamicUser=,
 251             # so that ssh won't fail with: "No user exists for uid $UID".
 252             environment.LD_LIBRARY_PATH = config.system.nssModules.path;
 253             serviceConfig = {
 254               ExecStartPre =
 255                 # Recursively remove any residual permissions
 256                 # given on local+descendant datasets (source, target or target's parent)
 257                 # to any currently unknown (hence unused) systemd dynamic users (UID/GID range 61184…65519),
 258                 # which happens when a crash has occurred
 259                 # during any previous run of a syncoid-*.service (not only this one).
 260                 map
 261                   (dataset:
 262                     "+" + pkgs.writeShellScript "zfs-unallow-unused-dynamic-users" ''
 263                       set -eu
 264                       zfs allow "$1" |
 265                       sed -ne 's/^\t\(user\|group\) (unknown: \([0-9]\+\)).*/\1 \2/p' |
 266                       {
 267                         declare -a uids
 268                         while read -r role id; do
 269                           if [ "$id" -ge 61184 ] && [ "$id" -le 65519 ]; then
 270                             case "$role" in
 271                               (user) uids+=("$id");;
 272                             esac
 273                           fi
 274                         done
 275                         zfs unallow -r -u "$(printf %s, "''${uids[@]}")" "$1"
 276                       }
 277                     '' + " " + escapeShellArg dataset
 278                   )
 279                   (localDatasetName c.source ++ localDatasetName c.target ++ map builtins.dirOf (localDatasetName c.target)) ++
 280                 # For a local source, allow the localSourceAllow ZFS permissions.
 281                 map
 282                   (dataset:
 283                     "+/run/booted-system/sw/bin/zfs allow $USER " +
 284                       escapeShellArgs [ (concatStringsSep "," c.localSourceAllow) dataset ]
 285                   )
 286                   (localDatasetName c.source) ++
 287                 # For a local target, check if the dataset exists before delegating permissions,
 288                 # and if it doesn't exist, delegate it to the parent dataset.
 289                 # This should solve the case of provisioning new datasets.
 290                 map
 291                   (dataset:
 292                     "+" + pkgs.writeShellScript "zfs-allow-target" ''
 293                       dataset="$1"
 294                       # Run a ZFS list on the dataset to check if it exists
 295                       zfs list "$dataset" >/dev/null 2>/dev/null ||
 296                         dataset="$(dirname "$dataset")"
 297                       zfs allow "$USER" ${escapeShellArg (concatStringsSep "," c.localTargetAllow)} "$dataset"
 298                     '' + " " + escapeShellArg dataset
 299                   )
 300                   (localDatasetName c.target) ++
 301                 # Adding a user to an nftables set will not persist across a reboot,
 302                 # hence there is no need to cleanup residual dynamic users remaining in it after a crash.
 303                 optional cfg.nftables.enable
 304                   "+${pkgs.nftables}/bin/nft add element inet filter nixos-syncoid-uids { $USER }";
 305               ExecStopPost =
 306                 let
 307                   zfsUnallow = dataset: "+/run/booted-system/sw/bin/zfs unallow $USER " + escapeShellArg dataset;
 308                 in
 309                 map zfsUnallow (localDatasetName c.source) ++
 310                   # For a local target, unallow both the dataset and its parent,
 311                   # because at this stage we have no way of knowing if the allow command
 312                   # did execute on the parent dataset or not in the ExecStartPre=.
 313                   # We can't run the same if-then-else in the post hook
 314                   # since the dataset should have been created at this point.
 315                   concatMap
 316                     (dataset: [ (zfsUnallow dataset) (zfsUnallow (builtins.dirOf dataset)) ])
 317                     (localDatasetName c.target) ++
 318                   optional cfg.nftables.enable
 319                     "+${pkgs.nftables}/bin/nft delete element inet filter nixos-syncoid-uids { $USER }";
 320               ExecStart = lib.escapeShellArgs ([ "${cfg.package}/bin/syncoid" ]
 321                 ++ optionals c.useCommonArgs cfg.commonArgs
 322                 ++ optional c.recursive "--recursive"
 323                 ++ optionals (c.sshKey != null) [ "--sshkey" "\${CREDENTIALS_DIRECTORY}/${if length sshKeyCred > 1 then head sshKeyCred else "sshKey"}" ]
 324                 ++ c.extraArgs
 325                 ++ [
 326                 "--sendoptions"
 327                 c.sendOptions
 328                 "--recvoptions"
 329                 c.recvOptions
 330                 "--no-privilege-elevation"
 331                 c.source
 332                 c.target
 333               ]);
 334               DynamicUser = true;
 335               # Prevent SSH control sockets of different syncoid services from interfering
 336               PrivateTmp = true;
 337               # Permissive access to /proc because syncoid
 338               # calls ps(1) to detect ongoing `zfs receive`.
 339               ProcSubset = "all";
 340               ProtectProc = "default";
 341
 342               # The following options are only for optimizing:
 343               # systemd-analyze security | grep syncoid-'*'
 344               AmbientCapabilities = "";
 345               CapabilityBoundingSet = "";
 346               DeviceAllow = [ "/dev/zfs" ];
 347               LockPersonality = true;
 348               MemoryDenyWriteExecute = true;
 349               NoNewPrivileges = true;
 350               PrivateDevices = true;
 351               PrivateMounts = true;
 352               PrivateNetwork = mkDefault false;
 353               PrivateUsers = true;
 354               ProtectClock = true;
 355               ProtectControlGroups = true;
 356               ProtectHome = true;
 357               ProtectHostname = true;
 358               ProtectKernelLogs = true;
 359               ProtectKernelModules = true;
 360               ProtectKernelTunables = true;
 361               ProtectSystem = "strict";
 362               RemoveIPC = true;
 363               RestrictAddressFamilies = [ "AF_UNIX" "AF_INET" "AF_INET6" ];
 364               RestrictNamespaces = true;
 365               RestrictRealtime = true;
 366               RestrictSUIDSGID = true;
 367               RootDirectory = "/run/syncoid/${escapeUnitName name}";
 368               RootDirectoryStartOnly = true;
 369               BindPaths = [ "/dev/zfs" ];
 370               BindReadOnlyPaths = [ builtins.storeDir "/etc" "/run" "/bin/sh" ];
 371               # Avoid useless mounting of RootDirectory= in the own RootDirectory= of ExecStart='s mount namespace.
 372               InaccessiblePaths = [ "-+/run/syncoid/${escapeUnitName name}" ];
 373               MountAPIVFS = true;
 374               # Create RootDirectory= in the host's mount namespace.
 375               RuntimeDirectory = [ "syncoid/${escapeUnitName name}" ];
 376               RuntimeDirectoryMode = "700";
 377               SystemCallFilter = [
 378                 "@system-service"
 379                 # Groups in @system-service which do not contain a syscall listed by:
 380                 # perf stat -x, 2>perf.log -e 'syscalls:sys_enter_*' syncoid …
 381                 # awk >perf.syscalls -F "," '$1 > 0 {sub("syscalls:sys_enter_","",$3); print $3}' perf.log
 382                 # systemd-analyze syscall-filter | grep -v -e '#' | sed -e ':loop; /^[^ ]/N; s/\n //; t loop' | grep $(printf ' -e \\<%s\\>' $(cat perf.syscalls)) | cut -f 1 -d ' '
 383                 "~@aio"
 384                 "~@chown"
 385                 "~@keyring"
 386                 "~@memlock"
 387                 "~@privileged"
 388                 "~@resources"
 389                 "~@setuid"
 390               ];
 391               SystemCallArchitectures = "native";
 392               # This is for BindPaths= and BindReadOnlyPaths=
 393               # to allow traversal of directories they create in RootDirectory=.
 394               UMask = "0066";
 395             } //
 396             (
 397               if length sshKeyCred > 1
 398               then { LoadCredentialEncrypted = [ c.sshKey ]; }
 399               else { LoadCredential = [ "sshKey:${c.sshKey}" ]; }
 400             );
 401           }
 402           cfg.service
 403           c.service
 404         ]))
 405       cfg.commands;
 406
 407     networking.nftables.ruleset = optionalString cfg.nftables.enable (mkBefore ''
 408       table inet filter {
 409         # A set containing the dynamic UIDs of the syncoid services currently active
 410         set nixos-syncoid-uids { type uid; }
 411       }
 412     '');
 413   };
 414
 415   meta.maintainers = with maintainers; [ julm lopsided98 ];
 416 }