nixos/modules/services/backup/syncoid.nix

   1 { config, lib, pkgs, ... }:
   2
   3 with lib;
   4
   5 let
   6   cfg = config.services.syncoid;
   7   inherit (config.networking) nftables;
   8
   9   # Extract local dasaset names (so no datasets containing "@")
  10   localDatasetName = d: optionals (d != null) (
  11     let m = builtins.match "([^/@]+[^@]*)" d; in
  12     optionals (m != null) m
  13   );
  14
  15   # Escape as required by: https://www.freedesktop.org/software/systemd/man/systemd.unit.html
  16   escapeUnitName = name:
  17     lib.concatMapStrings (s: if lib.isList s then "-" else s)
  18       (builtins.split "[^a-zA-Z0-9_.\\-]+" name);
  19
  20   # Function to build "zfs allow" commands for the filesystems we've
  21   # delegated permissions to. It also checks if the target dataset
  22   # exists before delegating permissions, if it doesn't exist we
  23   # delegate it to the parent dataset. This should solve the case of
  24   # provisoning new datasets.
  25   buildAllowCommand = permissions: dataset: (
  26     "-+${pkgs.writeShellScript "zfs-allow-${dataset}" ''
  27       set -eux
  28       # Run a ZFS list on the dataset to check if it exists
  29       if zfs list ${lib.escapeShellArg dataset} >/dev/null 2>/dev/null; then
  30         zfs allow "$USER" ${lib.escapeShellArgs [
  31           (concatStringsSep "," permissions)
  32           dataset
  33         ]}
  34       else
  35         zfs allow "$USER" ${lib.escapeShellArgs [
  36           (concatStringsSep "," permissions)
  37           # Remove the last part of the path
  38           (builtins.dirOf dataset)
  39         ]}
  40       fi
  41     ''}"
  42   );
  43
  44   # Function to build "zfs unallow" commands for the filesystems we've
  45   # delegated permissions to. Here we unallow both the target and
  46   # the parent dataset because at this stage we have no way of
  47   # knowing if the allow command did execute on the parent dataset or
  48   # not in the pre-hook. We can't run the same if-then-else in the post hook
  49   # since the dataset should have been created at this point.
  50   buildUnallowCommand = dataset: (
  51     "-+${pkgs.writeShellScript "zfs-unallow-${dataset}" ''
  52       set -eux
  53       zfs unallow "$USER" ${lib.escapeShellArg dataset}
  54       zfs unallow "$USER" ${lib.escapeShellArg (builtins.dirOf dataset)}
  55     ''}"
  56   );
  57 in
  58 {
  59
  60   # Interface
  61
  62   options.services.syncoid = {
  63     enable = mkEnableOption "Syncoid ZFS synchronization service";
  64
  65     interval = mkOption {
  66       type = types.str;
  67       default = "hourly";
  68       example = "*-*-* *:15:00";
  69       description = ''
  70         Run syncoid at this interval. The default is to run hourly.
  71
  72         The format is described in
  73         <citerefentry><refentrytitle>systemd.time</refentrytitle>
  74         <manvolnum>7</manvolnum></citerefentry>.
  75       '';
  76     };
  77
  78     sshKey = mkOption {
  79       type = types.nullOr types.path;
  80       # Prevent key from being copied to store
  81       apply = mapNullable toString;
  82       default = null;
  83       description = ''
  84         SSH private key file to use to login to the remote system. Can be
  85         overridden in individual commands.
  86         For more SSH tuning, you may use syncoid's <literal>--sshoption</literal>
  87         in <link linkend="opt-services.syncoid.commonArgs">commonArgs</link>
  88         and/or in the <literal>extraArgs<literal> of a specific command.
  89       '';
  90     };
  91
  92     localSourceAllow = mkOption {
  93       type = types.listOf types.str;
  94       # Permissions snapshot and destroy are in case --no-sync-snap is not used
  95       default = [ "bookmark" "hold" "send" "snapshot" "destroy" ];
  96       description = ''
  97         Permissions granted for the syncoid user for local source datasets.
  98         See <link xlink:href="https://openzfs.github.io/openzfs-docs/man/8/zfs-allow.8.html"/>
  99         for available permissions.
 100       '';
 101     };
 102
 103     localTargetAllow = mkOption {
 104       type = types.listOf types.str;
 105       # destroy is needed :
 106       # syncoid[3282027]: Resuming interrupted zfs send/receive from rpool/var/mail to losurdo/backup/mermet/var/mail (~ UNKNOWN remaining):
 107       # syncoid[3282885]: cannot resume send: 'rpool/var/mail@autosnap_2021-10-17_15:00:19_hourly' used in the initial send no longer exists
 108       # syncoid[3282897]: cannot receive: failed to read from stream
 109       # syncoid[3282027]: WARN: resetting partially receive state because the snapshot source no longer exists
 110       # syncoid[3283326]: cannot destroy 'losurdo/backup/mermet/var/mail/%recv': permission denied
 111       default = [ "change-key" "compression" "create" "destroy" "mount" "mountpoint" "receive" "rollback" ];
 112       example = [ "create" "mount" "receive" "rollback" ];
 113       description = ''
 114         Permissions granted for the syncoid user for local target datasets.
 115         See <link xlink:href="https://openzfs.github.io/openzfs-docs/man/8/zfs-allow.8.html"/>
 116         for available permissions.
 117         Make sure to include the <literal>change-key</literal> permission if you send raw encrypted datasets,
 118         the <literal>compression</literal> permission if you send raw compressed datasets, and so on.
 119         For remote target datasets you'll have to set your remote user permissions by yourself.
 120       '';
 121     };
 122
 123     commonArgs = mkOption {
 124       type = types.listOf types.str;
 125       default = [ ];
 126       example = [ "--no-sync-snap" ];
 127       description = ''
 128         Arguments to add to every syncoid command, unless disabled for that
 129         command. See
 130         <link xlink:href="https://github.com/jimsalterjrs/sanoid/#syncoid-command-line-options"/>
 131         for available options.
 132       '';
 133     };
 134
 135     service = mkOption {
 136       type = types.attrs;
 137       default = { };
 138       description = ''
 139         Systemd configuration common to all syncoid services.
 140       '';
 141     };
 142
 143     commands = mkOption {
 144       type = types.attrsOf (types.submodule ({ name, ... }: {
 145         options = {
 146           source = mkOption {
 147             type = types.str;
 148             example = "pool/dataset";
 149             description = ''
 150               Source ZFS dataset. Can be either local or remote. Defaults to
 151               the attribute name.
 152             '';
 153           };
 154
 155           target = mkOption {
 156             type = types.str;
 157             example = "user@server:pool/dataset";
 158             description = ''
 159               Target ZFS dataset. Can be either local
 160               (<replaceable>pool/dataset</replaceable>) or remote
 161               (<replaceable>user@server:pool/dataset</replaceable>).
 162             '';
 163           };
 164
 165           recursive = mkEnableOption ''the transfer of child datasets'';
 166
 167           sshKey = mkOption {
 168             type = types.nullOr types.path;
 169             # Prevent key from being copied to store
 170             apply = mapNullable toString;
 171             description = ''
 172               SSH private key file to use to login to the remote system.
 173               Defaults to <option>services.syncoid.sshKey</option> option.
 174             '';
 175           };
 176
 177           localSourceAllow = mkOption {
 178             type = types.listOf types.str;
 179             description = ''
 180               Permissions granted for the <option>services.syncoid.user</option> user
 181               for local source datasets. See
 182               <link xlink:href="https://openzfs.github.io/openzfs-docs/man/8/zfs-allow.8.html"/>
 183               for available permissions.
 184               Defaults to <option>services.syncoid.localSourceAllow</option> option.
 185             '';
 186           };
 187
 188           localTargetAllow = mkOption {
 189             type = types.listOf types.str;
 190             description = ''
 191               Permissions granted for the <option>services.syncoid.user</option> user
 192               for local target datasets. See
 193               <link xlink:href="https://openzfs.github.io/openzfs-docs/man/8/zfs-allow.8.html"/>
 194               for available permissions.
 195               Make sure to include the <literal>change-key</literal> permission if you send raw encrypted datasets,
 196               the <literal>compression</literal> permission if you send raw compressed datasets, and so on.
 197               For remote target datasets you'll have to set your remote user permissions by yourself.
 198             '';
 199           };
 200
 201           sendOptions = mkOption {
 202             type = types.separatedString " ";
 203             default = "";
 204             example = "Lc e";
 205             description = ''
 206               Advanced options to pass to zfs send. Options are specified
 207               without their leading dashes and separated by spaces.
 208             '';
 209           };
 210
 211           recvOptions = mkOption {
 212             type = types.separatedString " ";
 213             default = "";
 214             example = "ux recordsize o compression=lz4";
 215             description = ''
 216               Advanced options to pass to zfs recv. Options are specified
 217               without their leading dashes and separated by spaces.
 218             '';
 219           };
 220
 221           useCommonArgs = mkEnableOption ''
 222             configured common arguments to this command
 223           '' // { default = true; };
 224
 225           service = mkOption {
 226             type = types.attrs;
 227             default = { };
 228             description = ''
 229               Systemd configuration specific to this syncoid service.
 230             '';
 231           };
 232
 233           extraArgs = mkOption {
 234             type = types.listOf types.str;
 235             default = [ ];
 236             example = [ "--sshport 2222" ];
 237             description = "Extra syncoid arguments for this command.";
 238           };
 239         };
 240         config = {
 241           source = mkDefault name;
 242           sshKey = mkDefault cfg.sshKey;
 243           localSourceAllow = mkDefault cfg.localSourceAllow;
 244           localTargetAllow = mkDefault cfg.localTargetAllow;
 245         };
 246       }));
 247       default = { };
 248       example = literalExample ''
 249         {
 250           "pool/test".target = "root@target:pool/test";
 251         }
 252       '';
 253       description = "Syncoid commands to run.";
 254     };
 255   };
 256
 257   # Implementation
 258
 259   config = mkIf cfg.enable {
 260     systemd.services = mapAttrs'
 261       (name: c:
 262         nameValuePair "syncoid-${escapeUnitName name}" (mkMerge [
 263           {
 264             description = "Syncoid ZFS synchronization from ${c.source} to ${c.target}";
 265             after = [ "zfs.target" ];
 266             startAt = cfg.interval;
 267             # Here we explicitly use the booted system to guarantee the stable API needed by ZFS.
 268             # Moreover syncoid may need zpool to get feature@extensible_dataset.
 269             path = [ "/run/booted-system/sw" ];
 270             serviceConfig = {
 271               ExecStartPre =
 272                 (map (buildAllowCommand c.localSourceAllow) (localDatasetName c.source)) ++
 273                 (map (buildAllowCommand c.localTargetAllow) (localDatasetName c.target)) ++
 274                 optional nftables.enable "+${pkgs.nftables}/bin/nft add element inet filter nixos-syncoid-uids { $USER }";
 275               ExecStopPost =
 276                 (map buildUnallowCommand (localDatasetName c.source)) ++
 277                 (map buildUnallowCommand (localDatasetName c.target)) ++
 278                 optional nftables.enable "+${pkgs.nftables}/bin/nft delete element inet filter nixos-syncoid-uids { $USER }";
 279               ExecStart = lib.escapeShellArgs ([ "${pkgs.sanoid}/bin/syncoid" ]
 280                 ++ optionals c.useCommonArgs cfg.commonArgs
 281                 ++ optional c.recursive "--recursive"
 282                 ++ optionals (c.sshKey != null) [ "--sshkey" "\${CREDENTIALS_DIRECTORY}/ssh-key" ]
 283                 ++ c.extraArgs
 284                 ++ [
 285                 "--sendoptions"
 286                 c.sendOptions
 287                 "--recvoptions"
 288                 c.recvOptions
 289                 "--no-privilege-elevation"
 290                 c.source
 291                 c.target
 292               ]);
 293               DynamicUser = true;
 294               LoadCredential = [ "ssh-key:${c.sshKey}" ];
 295               # Prevent SSH control sockets of different syncoid services from interfering
 296               PrivateTmp = true;
 297               # Permissive access to /proc because syncoid
 298               # calls ps(1) to detect ongoing `zfs receive`.
 299               ProcSubset = "all";
 300               ProtectProc = "default";
 301
 302               # The following options are only for optimizing:
 303               # systemd-analyze security | grep syncoid-'*'
 304               AmbientCapabilities = "";
 305               CapabilityBoundingSet = "";
 306               DeviceAllow = [ "/dev/zfs" ];
 307               LockPersonality = true;
 308               MemoryDenyWriteExecute = true;
 309               NoNewPrivileges = true;
 310               PrivateDevices = true;
 311               PrivateMounts = true;
 312               PrivateNetwork = mkDefault false;
 313               PrivateUsers = true;
 314               ProtectClock = true;
 315               ProtectControlGroups = true;
 316               ProtectHome = true;
 317               ProtectHostname = true;
 318               ProtectKernelLogs = true;
 319               ProtectKernelModules = true;
 320               ProtectKernelTunables = true;
 321               ProtectSystem = "strict";
 322               RemoveIPC = true;
 323               RestrictAddressFamilies = [ "AF_UNIX" "AF_INET" "AF_INET6" ];
 324               RestrictNamespaces = true;
 325               RestrictRealtime = true;
 326               RestrictSUIDSGID = true;
 327               RootDirectory = "/run/syncoid/${escapeUnitName name}";
 328               RootDirectoryStartOnly = true;
 329               BindPaths = [ "/dev/zfs" ];
 330               BindReadOnlyPaths = [ builtins.storeDir "/etc" "/run" "/bin/sh"
 331                 # A custom LD_LIBRARY_PATH is needed to access in `getent passwd`
 332                 # the systemd's entry about the DynamicUser=,
 333                 # so that ssh won't fail with: "No user exists for uid $UID".
 334                 # Unfortunately, Bash is incompatible with libnss_systemd.so:
 335                 # https://www.mail-archive.com/bug-bash@gnu.org/msg24306.html
 336                 # Hence the wrapping of ssh is done here as a mounted path,
 337                 # because Nixpkgs' wrapping of syncoid enforces the use
 338                 # of the ${pkgs.openssh}/bin/ssh path.
 339                 # This problem does not arise on NixOS systems where stdenv.hostPlatform.libc == "musl",
 340                 # because then Bash is built with --without-bash-malloc
 341                 ("${pkgs.writeShellScript "ssh-with-support-for-DynamicUser" ''
 342                   export LD_LIBRARY_PATH="${config.system.nssModules.path}"
 343                   exec -a ${pkgs.openssh}/bin/ssh /bin/ssh "$@"
 344                 ''}:${pkgs.openssh}/bin/ssh")
 345                 "${pkgs.openssh}/bin/ssh:/bin/ssh"
 346               ];
 347               # Avoid useless mounting of RootDirectory= in the own RootDirectory= of ExecStart='s mount namespace.
 348               InaccessiblePaths = [ "-+/run/syncoid/${escapeUnitName name}" ];
 349               MountAPIVFS = true;
 350               # Create RootDirectory= in the host's mount namespace.
 351               RuntimeDirectory = [ "syncoid/${escapeUnitName name}" ];
 352               RuntimeDirectoryMode = "700";
 353               SystemCallFilter = [
 354                 "@system-service"
 355                 # Groups in @system-service which do not contain a syscall listed by:
 356                 # perf stat -x, 2>perf.log -e 'syscalls:sys_enter_*' syncoid …
 357                 # awk >perf.syscalls -F "," '$1 > 0 {sub("syscalls:sys_enter_","",$3); print $3}' perf.log
 358                 # systemd-analyze syscall-filter | grep -v -e '#' | sed -e ':loop; /^[^ ]/N; s/\n //; t loop' | grep $(printf ' -e \\<%s\\>' $(cat perf.syscalls)) | cut -f 1 -d ' '
 359                 "~@aio"
 360                 "~@chown"
 361                 "~@keyring"
 362                 "~@memlock"
 363                 "~@privileged"
 364                 "~@resources"
 365                 "~@setuid"
 366                 "~@timer"
 367               ];
 368               SystemCallArchitectures = "native";
 369               # This is for BindPaths= and BindReadOnlyPaths=
 370               # to allow traversal of directories they create in RootDirectory=.
 371               UMask = "0066";
 372             };
 373           }
 374           cfg.service
 375           c.service
 376         ]))
 377       cfg.commands;
 378     networking.nftables.ruleset = ''
 379       # A set containing the dynamic UIDs of the syncoid services currently active
 380       add set inet filter nixos-syncoid-uids { type uid; }
 381       # Example of use (assuming fw2net is being called by the output chain):
 382       #add rule inet filter fw2net meta skuid @nixos-syncoid-uids meta l4proto tcp accept
 383     '';
 384   };
 385
 386   meta.maintainers = with maintainers; [ julm lopsided98 ];
 387 }