diff options
author | Igor Pashev <pashev.igor@gmail.com> | 2016-09-29 13:51:44 +0300 |
---|---|---|
committer | Igor Pashev <pashev.igor@gmail.com> | 2016-09-29 13:51:44 +0300 |
commit | 62f28d30a069135f9c48678507203958adfc334f (patch) | |
tree | 7f38af0c8d3f445ee8cc50906a639baec7011127 /modules/apps/filebackup.nix | |
parent | 1af9e6589bdd18e6ba7eeabf073aa7d710020cdd (diff) | |
download | nixsap-62f28d30a069135f9c48678507203958adfc334f.tar.gz |
Moved everything into ./modules
Diffstat (limited to 'modules/apps/filebackup.nix')
-rw-r--r-- | modules/apps/filebackup.nix | 282 |
1 files changed, 282 insertions, 0 deletions
diff --git a/modules/apps/filebackup.nix b/modules/apps/filebackup.nix new file mode 100644 index 0000000..4aee0a9 --- /dev/null +++ b/modules/apps/filebackup.nix @@ -0,0 +1,282 @@ +{ config, pkgs, lib, ... }: +let + + inherit (builtins) + isBool isList isString toString ; + inherit (lib) + concatMapStringsSep concatStringsSep filter filterAttrs + flatten hasPrefix mapAttrsToList mkIf + mkOption optionalString removeSuffix ; + inherit (lib.types) + attrsOf bool either enum int listOf nullOr path str submodule ; + + cfg = config.nixsap.apps.filebackup; + privateDir = "/run/filebackup"; + + s3cmd = "${pkgs.s3cmd}/bin/s3cmd ${optionalString (cfg.s3cfg != null) "-c '${cfg.s3cfg}'"}"; + + gpgPubKeys = flatten [ cfg.encrypt ]; + gpg = "${pkgs.gpg}/bin/gpg2"; + pubring = pkgs.runCommand "pubring.gpg" {} '' + ${gpg} --homedir . --import ${toString gpgPubKeys} + cp pubring.gpg $out + ''; + + default = d: t: mkOption { type = t; default = d; }; + optional = type: mkOption { type = nullOr type; default = null; }; + sub = options: submodule { inherit options; } ; + mandatory = type: mkOption { inherit type; }; + concatMapAttrsSep = s: f: attrs: concatStringsSep s (mapAttrsToList f attrs); + + command = sub + { + absolute-names = optional bool; + exclude = optional (either str (listOf str)); + exclude-from = optional path; + exclude-vcs = optional bool; + exclude-vcs-ignores = optional bool; + group = optional str; + ignore-case = optional bool; + mode = optional str; + owner = optional str; + path = mandatory (either path (listOf path)); + }; + + job = name: o: + let + args = filterAttrs (k: v: + v != null && k != "_module" + && ( k != "path" ) + ) o; + + mkArg = k: v: + if isBool v then (optionalString v "--${k}") + else if isList v then concatMapStringsSep " " (i: "--${k}='${i}'") v + else if isString v then "--${k}='${v}'" + else "--${k}=${toString v}" ; + + tar = pkgs.writeBashScript "tar-${name}" '' + exec ${pkgs.gnutar}/bin/tar -c -f- \ + ${concatMapAttrsSep " " mkArg args} \ + "$@" + ''; + + in pkgs.writeBashScript "tar-${name}-job" '' + set -euo pipefail + cd "${cfg.tarballDir}/$DATE" + host=$(${pkgs.nettools}/bin/hostname -f) + + tarball="${name}@$host,$DATE.tar.xz" + ${ + if (gpgPubKeys != []) then + ''aim="$tarball.gpg"'' + else + ''aim="$tarball"'' + } + + if ! [ -r "$aim" ]; then + ${tar} ${concatMapStringsSep " " (p: "'${p}'") (flatten [o.path])} \ + | ${pkgs.pxz}/bin/pxz -2 -T2 > "$tarball.tmp" + mv "$tarball".tmp "$tarball" + + ${optionalString (gpgPubKeys != []) '' + recipient=( $(${gpg} --homedir '${privateDir}/gnupg' -k --with-colons --fast-list-mode | \ + ${pkgs.gawk}/bin/awk -F: '/^pub/{print $5}') ) + r=( "''${recipient[@]/#/-r}" ) + ${gpg} --homedir '${privateDir}/gnupg' --batch --no-tty --yes \ + "''${r[@]}" --trust-model always \ + --compress-algo none \ + -v -e "$tarball" + rm -f "$tarball" + ''} + else + echo "$aim exists. Not creating." >&2 + fi + ${optionalString (cfg.s3uri != null) '' + remote="${removeSuffix "/" cfg.s3uri}/$DATE/$aim" + if ! ${s3cmd} ls "$remote" | ${pkgs.gnugrep}/bin/grep -qF "/$aim"; then + ${s3cmd} put "$aim" "$remote" + else + echo "$remote exists. Not uploading." >&2 + fi + ''} + ''; + + preStart = '' + mkdir --mode=0750 -p '${cfg.tarballDir}' + chown -R ${cfg.user}:${cfg.user} '${cfg.tarballDir}' + chmod -R u=rwX,g=rX,o= ${cfg.tarballDir} + + rm -rf '${privateDir}' + mkdir --mode=0700 -p '${privateDir}' + chown ${cfg.user}:${cfg.user} '${privateDir}' + ''; + + main = pkgs.writeBashScriptBin "filebackup" '' + set -euo pipefail + umask 0027 + DATE=$(date --iso-8601) + HOME='${privateDir}' + PARALLEL_SHELL=${pkgs.bash}/bin/bash + export DATE + export HOME + export PARALLEL_SHELL + + clean() { + ${pkgs.findutils}/bin/find '${cfg.tarballDir}' \ + -name '*.tmp' -exec rm -rf {} + || true + } + + listSets() { + ${pkgs.findutils}/bin/find '${cfg.tarballDir}' \ + -maxdepth 1 -mindepth 1 -type d -name '????-??-??' \ + | sort -V + } + + enoughStorage() { + local n + local used + local total + local avg + local p + n=$(listSets | wc -l) + used=$(du -x -s --block-size=1M '${cfg.tarballDir}' | cut -f1) + total=$(df --output=size --block-size=1M '${cfg.tarballDir}' | tail -n 1) + if [ "$n" -eq 0 ]; then + echo "no sets" >&2 + return 0 + fi + + avg=$(( used / n )) + p=$(( 100 * avg * (n + 1) / total )) + printf "estimated storage: %d of %d MiB (%d%%, max ${toString cfg.storage}%%)\n" \ + "$((used + avg))" "$total" "$p" >&2 + if [ "$p" -le ${toString cfg.storage} ]; then + return 0 + else + return 1 + fi + } + + clean + + listSets | head -n -${toString (cfg.slots - 1)} \ + | ${pkgs.findutils}/bin/xargs --no-run-if-empty rm -rfv \ + || true + + while ! enoughStorage; do + listSets | head -n 1 \ + | ${pkgs.findutils}/bin/xargs --no-run-if-empty rm -rfv \ + || true + done + + mkdir -p "${cfg.tarballDir}/$DATE" + + ${optionalString (gpgPubKeys != []) '' + # shellcheck disable=SC2174 + mkdir --mode=0700 -p '${privateDir}/gnupg' + ln -sf ${pubring} '${privateDir}/gnupg/pubring.gpg' + ''} + + failed=0 + log="${cfg.tarballDir}/$DATE/joblog.txt" + + # shellcheck disable=SC2016 + ${pkgs.parallel}/bin/parallel \ + --halt-on-error 0 \ + --joblog "$log" \ + --jobs 50% \ + --line-buffer \ + --no-notice \ + --no-run-if-empty \ + --retries 2 \ + --rpl '{nixbase} s:^/nix/store/[^-]+-tar-(.+)-job$:$1:' \ + --tagstr '* {nixbase}:' \ + --timeout ${toString (6 * 60 * 60)} ::: \ + ${concatMapAttrsSep " " job cfg.files} \ + || failed=$? + + cat "$log" + clean + + du -sh "${cfg.tarballDir}/$DATE" || true + exit "$failed" + ''; + + keys = filter (f: f != null && hasPrefix "/run/keys/" f) ( [cfg.s3cfg] ); + +in { + options.nixsap.apps.filebackup = { + user = mkOption { + description = "User to run as"; + default = "filebackup"; + type = str; + }; + + tarballDir = mkOption { + description = "Directory to save tarballs in"; + default = "/filebackup"; + type = path; + }; + + slots = mkOption { + description = '' + How many backup sets should be kept locally. + However, old sets will be removed anyway if storage + constraints apply. + ''; + default = 60; + type = int; + }; + + storage = mkOption { + description = '' + Percent of storage backups can occupy. + ''; + default = 75; + type = int; + }; + + encrypt = mkOption { + description = "Public GPG key(s) for encrypting the dumps"; + default = [ ]; + type = either path (listOf path); + }; + + s3cfg = mkOption { + description = "s3cmd config file (secret)"; + type = nullOr path; + default = null; + }; + + s3uri = mkOption { + description = "S3 bucket URI with prefix in s3cmd format"; + type = nullOr str; + default = null; + example = "s3://backups/nightly"; + }; + + files = mkOption { + description = "tar commands"; + default = {}; + type = attrsOf command; + }; + }; + + config = mkIf (cfg.files != {}) { + nixsap.system.users.daemons = [ cfg.user ]; + nixsap.deployment.keyrings.${cfg.user} = keys; + systemd.services.filebackup = { + description = "Directory backup with tar"; + after = [ "local-fs.target" "keys.target" ]; + wants = [ "keys.target" ]; + startAt = "02:00"; + inherit preStart; + serviceConfig = { + ExecStart = "${main}/bin/filebackup"; + User = cfg.user; + PermissionsStartOnly = true; + }; + }; + }; +} |