Add restic commands to the restic-guix package

  • Open
  • quality assurance status badge
Details
2 participants
  • paul
  • Fabio Natali
Owner
unassigned
Submitted by
paul
Severity
normal
P
(address . guix-patches@gnu.org)
db336bf4-14d8-e969-b998-dd5f98108066@autistici.org
Dear all,

I'm sending a patch adding some more restic commands to the restic-guix
package provided by the restic-backup-service-type. It allows for
commands like the following:

restic-guix restore remote-ftp -t `pwd`/restored -i .config/guix/channels.scm latest


Thank you for your work,

giacomo
Attachment: file
G
G
Giacomo Leidi wrote on 25 Aug 15:56 +0200
[PATCH] services: restic-backup: Add more restic commands to the restic-guix package.
(address . 72803@debbugs.gnu.org)(name . Giacomo Leidi)(address . goodoldpaul@autistici.org)
1084765da10bf285803cbb7457997f73f785983d.1724594201.git.goodoldpaul@autistici.org
This patch refactors the way restic commands can be added to the
restic-guix package with a more general approach. This way new
subcommands for restic-guix can be added more easily.

* gnu/services/backup.scm (restic-backup-job-program): Generalize to
restic-action-program;
(restic-guix): allow for multiple actions.

* doc/guix.texi: Document it.

Change-Id: Ib2b5d74bebc51e35f1ae6e1aa32cedee0da59697
---
doc/guix.texi | 10 +++-
gnu/services/backup.scm | 116 +++++++++++++++++++++++++++++-----------
2 files changed, 93 insertions(+), 33 deletions(-)

Toggle diff (201 lines)
diff --git a/doc/guix.texi b/doc/guix.texi
index fcaf6b3fbb..9bbc2694ec 100644
--- a/doc/guix.texi
+++ b/doc/guix.texi
@@ -41589,7 +41589,15 @@ Miscellaneous Services
configuration, without waiting for the scheduled job:
@example
-restic-guix backup remote-ftp
+restic-guix run remote-ftp
+@end example
+
+All arguments passed after the job name will be passed to the underlying
+@code{restic} command, together with the @code{extra-flags} field from the
+@code{restic-backup-job} record:
+
+@example
+restic-guix restore remote-ftp -t `pwd`/restored -i .config/guix/channels.scm latest
@end example
@c %start of fragment
diff --git a/gnu/services/backup.scm b/gnu/services/backup.scm
index 555e9fc959..f304361263 100644
--- a/gnu/services/backup.scm
+++ b/gnu/services/backup.scm
@@ -46,7 +46,7 @@ (define-module (gnu services backup)
restic-backup-configuration-fields
restic-backup-configuration-jobs
- restic-backup-job-program
+ restic-action-program
restic-backup-job->mcron-job
restic-guix
restic-guix-wrapper-package
@@ -107,15 +107,18 @@ (define-configuration/no-serialization restic-backup-configuration
(list-of-restic-backup-jobs '())
"The list of backup jobs for the current system."))
-(define (restic-backup-job-program config)
+(define %restic-guix-supported-actions
+ '("backup" "mount" "prune" "restore" "run" "unlock"))
+
+(define* (restic-action-program config action)
(let ((restic
(file-append (restic-backup-job-restic config) "/bin/restic"))
+ (name
+ (restic-backup-job-name config))
(repository
(restic-backup-job-repository config))
(password-file
(restic-backup-job-password-file config))
- (files
- (restic-backup-job-files config))
(extra-flags
(restic-backup-job-extra-flags config))
(verbose
@@ -123,55 +126,104 @@ (define (restic-backup-job-program config)
'("--verbose")
'())))
(program-file
- "restic-backup-job.scm"
+ (string-append "restic-" action "-" name "-program.scm")
#~(begin
(use-modules (ice-9 popen)
- (ice-9 rdelim))
+ (ice-9 rdelim)
+ (srfi srfi-1))
+
+ (define cli-arguments
+ (let* ((cl (command-line))
+ (argc (length cl)))
+ (if (> argc 1)
+ (take-right cl (- argc 1))
+ '())))
+
(setenv "RESTIC_PASSWORD"
(with-input-from-file #$password-file read-line))
- (execlp #$restic #$restic #$@verbose
- "-r" #$repository
- #$@extra-flags
- "backup" #$@files)))))
+ (apply execlp `(#$restic #$restic #$@verbose
+ "-r" #$repository
+ #$@extra-flags
+ #$action ,@cli-arguments))))))
+
+(define* (restic-guix jobs #:key (supported-actions
+ %restic-guix-supported-actions))
+ (define action-table
+ (map
+ (lambda (action)
+ (list action
+ (map (lambda (job)
+ (list (restic-backup-job-name job)
+ (restic-action-program job action)))
+ jobs)))
+ ;; run is an alias for backup
+ (filter (lambda (a) (not (string=? a "run"))) supported-actions)))
-(define (restic-guix jobs)
(program-file
"restic-guix"
#~(begin
(use-modules (ice-9 match)
(srfi srfi-1))
+ (define action-table '#$action-table)
+ (define (assoc-table key table)
+ (first
+ (filter-map
+ (match-lambda
+ ((k v)
+ (and (string=? key k) v)))
+ table)))
(define names '#$(map restic-backup-job-name jobs))
- (define programs '#$(map restic-backup-job-program jobs))
+ (define backup-files
+ '#$(map restic-backup-job-files jobs))
+
+ (define (get-program action name)
+ (assoc-table name (assoc-table action action-table)))
- (define (get-program name)
+ (define (get-backup-files name)
(define idx
(list-index (lambda (n) (string=? n name)) names))
- (unless idx
- (error (string-append "Unknown job name " name "\n\n"
- "Possible job names are: "
- (string-join names " "))))
- (list-ref programs idx))
-
- (define (backup args)
- (define name (third args))
- (define program (get-program name))
- (execlp program program))
+ (list-ref backup-files idx))
(define (validate-args args)
- (when (not (>= (length args) 3))
- (error (string-append "Usage: " (basename (car args))
- " backup NAME"))))
+ (unless (>= (length args) 2)
+ (error (string-append "Usage: " (basename (first args))
+ " ACTION [ARGS]\n\nSupported actions are: "
+ #$(string-join supported-actions ", ") ".")))
+ (unless (member (second args) '#$supported-actions)
+ (error (string-append "Unknown action: " (second args) ". Supported"
+ "actions are: "
+ #$(string-join supported-actions ", ") "."))))
+
+ (define (validate-action-args action args)
+ (define argc (length args))
+ (when (not (>= argc 3))
+ (error (string-append "Usage: " (basename (first args))
+ " " action " JOB_NAME [ARGS]\n\nPossible job "
+ "names are: " (string-join names ", ") ".")))
+ (define job-name (third args))
+ (unless (member job-name names)
+ (error (string-append "Unknown job name: " job-name ". Possible job "
+ "names are: " (string-join names ", ") ".")))
+ (let ((program
+ (get-program
+ ;; run is just backup called with restic-backup-job-files
+ (if (string=? action "run") "backup" action)
+ job-name))
+ (rest (if (> argc 3)
+ (take-right args (- argc 3))
+ '())))
+ (values program
+ (if (string=? action "run")
+ (append rest (get-backup-files job-name))
+ rest))))
(define (main args)
(validate-args args)
(define action (second args))
- (match action
- ("backup"
- (backup args))
- (_
- (error (string-append "Unknown action: " action)))))
+ (define-values (program action-args) (validate-action-args action args))
+ (apply execlp (append (list program program) action-args)))
(main (command-line)))))
@@ -183,7 +235,7 @@ (define (restic-backup-job->mcron-job config)
(name
(restic-backup-job-name config)))
#~(job #$schedule
- #$(string-append "restic-guix backup " name)
+ #$(string-append "restic-guix run " name)
#:user #$user)))
(define (restic-guix-wrapper-package jobs)

base-commit: d48af5cca84914d44b032d0bf0820640ebbe7a4b
--
2.45.2
F
F
Fabio Natali wrote on 3 Sep 00:50 +0200
Re: Add restic commands to the restic-guix package
(address . 72803@debbugs.gnu.org)(name . Giacomo Leidi)(address . goodoldpaul@autistici.org)
8734mhfz6e.fsf@fabionatali.com
Hi Giacomo,

Thanks for the patch and for the Restic service in the first place.

Toggle quote (2 lines)
> diff --git a/doc/guix.texi b/doc/guix.texi

In the manual, consider the "extra-flags" section where we say:

Toggle quote (3 lines)
> A list of values that are lowered to strings. These will be passed as
> command-line arguments to the current job restic backup invokation.

Perhaps this should now read "...the current job restic invokation..."
or "...the current restic invokation...", as the action is no longer
limited to "backup"?

Toggle quote (4 lines)
> (program-file
> - "restic-backup-job.scm"
> + (string-append "restic-" action "-" name "-program.scm")

Should 'name' be slug-ified in any way here? E.g. to avoid spaces,
capital letters, symbols that might be confusing when part of a file
name, etc.

Toggle quote (11 lines)
> + (define action-table
> + (map
> + (lambda (action)
> + (list action
> + (map (lambda (job)
> + (list (restic-backup-job-name job)
> + (restic-action-program job action)))
> + jobs)))
> + ;; run is an alias for backup
> + (filter (lambda (a) (not (string=? a "run"))) supported-actions)))

Could this be (marginally) simpler if we used two nested association
lists? That way, 'get-program' might simply use 'assoc-ref' (twice) and
'assoc-table' would be redundant?

Everything else looks fine to me. For what it's worth, here's how I've
been testing this.

Initialise a Restic repository as follows (warning: this overwrites
'/some-temporary-folder/password'):

Toggle snippet (6 lines)
mkdir /some-temporary-folder
export RESTIC_PASSWORD=password
restic init --repo=/some-temporary-folder/repository
echo "${RESTIC_PASSWORD}" > /some-temporary-folder/password

Save the following system definition as
'/some-temporary-folder/config.scm'.

Toggle snippet (28 lines)
(use-modules (gnu))
(use-package-modules backup)
(use-service-modules backup)

(operating-system
(host-name "host")
(bootloader (bootloader-configuration
(bootloader grub-bootloader)
(targets '("/dev/vda"))))
(file-systems (cons (file-system
(device "/dev/vda1")
(mount-point "/")
(type "ext4"))
%base-file-systems))
(packages (cons* restic %base-packages))
(services (cons*
(service restic-backup-service-type
(restic-backup-configuration
(jobs
(list (restic-backup-job
(name "test")
(repository "/restic/repository")
(password-file "/restic/password")
(schedule "* * * * *")
(files '("/root")))))))
%base-services)))

From a Guix checkout where this patch has been applied, launch a test VM
as follows:

Toggle snippet (7 lines)
$(./pre-inst-env guix system vm \
--no-graphic \
--share=/some-temporary-folder=/restic \
/tmp/config.scm) \
-m 2048 -smp 2

Log in as root, then check that the cron schedule is correctly defined
with 'herd schedule mcron', backup jobs should be scheduled every
minute.

Mount the Restic repository to see that snapshots have been actually
created every minute since boot. This can be done either on the guest or
on the host system. E.g. on the guest:

Toggle snippet (6 lines)
restic mount \
--password-file=/restic/password \
--repo=/restic/repository \
/mnt

Unfortunately I don't have commit access to push this, but hopefully
someone else will have a second look and push it soon.

It'd be nice to have a little test suite for this, but in case this can
be part of a future patch.

HTH, thanks, Fabio.
F
F
Fabio Natali wrote on 3 Sep 01:01 +0200
(address . 72803@debbugs.gnu.org)
87zfopek2m.fsf@fabionatali.com
Toggle quote (3 lines)
> Mount the Restic repository to see that snapshots have been actually
> created every minute since boot.

Ha, sorry, I should have mentioned the revised 'restic-guix' script too,
which I tested with various commands and that also seemed to be working
fine.

Thanks, cheers, Fabio.
P
2d83c75b-b750-b80d-5d7c-b4a6c89b4434@autistici.org
Hi Fabio,

thank you very much for your detailed testing and review.

On 9/3/24 00:50, Fabio Natali wrote:
Toggle quote (4 lines)
> Perhaps this should now read "...the current job restic invokation..."
> or "...the current restic invokation...", as the action is no longer
> limited to "backup"?

Definitely, good catch.

Toggle quote (6 lines)
>> (program-file
>> - "restic-backup-job.scm"
>> + (string-append "restic-" action "-" name "-program.scm")
> Should 'name' be slug-ified in any way here? E.g. to avoid spaces,
> capital letters, symbols that might be confusing when part of a file
> name, etc.
It should, right. I'll use the same approach used for the
home-dotfiles-service-type (i.e. replacing illegal characters with "-").
Toggle quote (13 lines)
>> + (define action-table
>> + (map
>> + (lambda (action)
>> + (list action
>> + (map (lambda (job)
>> + (list (restic-backup-job-name job)
>> + (restic-action-program job action)))
>> + jobs)))
>> + ;; run is an alias for backup
>> + (filter (lambda (a) (not (string=? a "run"))) supported-actions)))
> Could this be (marginally) simpler if we used two nested association
> lists? That way, 'get-program' might simply use 'assoc-ref' (twice) and
> 'assoc-table' would be redundant?
I thought that as well, in fact my first implementation was with Guile's
vhashes but it appears that neither alists nor vhashesh can be correctly
ungexped, or at least I didn't find a way to do so. This is why I'm
using plain lists and I need assoc-table. If you have some pointer where
I could look how to lower alists it would be very helpful.
Toggle quote (3 lines)
> It'd be nice to have a little test suite for this, but in case this can
> be part of a future patch.

There are already some tests Richard made at #71639 , once they get in
I'll make sure to expand them for additional restic-guix subcommands.

Thank your for your review Fabio, I'm sending a patchset addressing your
comments.

giacomo
G
G
Giacomo Leidi wrote on 5 Sep 00:29 +0200
[PATCH v2] services: restic-backup: Add more restic commands to the restic-guix package.
(address . 72803@debbugs.gnu.org)(name . Giacomo Leidi)(address . goodoldpaul@autistici.org)
cba87a67ff4b6dfdc7620b45c9fec4d1ee0321bc.1725488944.git.goodoldpaul@autistici.org
This patch refactors the way restic commands can be added to the
restic-guix package with a more general approach. This way new
subcommands for restic-guix can be added more easily.

* gnu/services/backup.scm (restic-backup-job-program): Generalize to
restic-action-program;
(restic-guix): allow for multiple actions.

* doc/guix.texi: Document it.

Change-Id: Ib2b5d74bebc51e35f1ae6e1aa32cedee0da59697
---
doc/guix.texi | 13 +++-
gnu/services/backup.scm | 127 +++++++++++++++++++++++++++++-----------
2 files changed, 104 insertions(+), 36 deletions(-)

Toggle diff (229 lines)
diff --git a/doc/guix.texi b/doc/guix.texi
index 16c697586a..8e3ecb80c2 100644
--- a/doc/guix.texi
+++ b/doc/guix.texi
@@ -41611,7 +41611,15 @@ Miscellaneous Services
configuration, without waiting for the scheduled job:
@example
-restic-guix backup remote-ftp
+restic-guix run remote-ftp
+@end example
+
+All arguments passed after the job name will be passed to the underlying
+@code{restic} command, together with the @code{extra-flags} field from the
+@code{restic-backup-job} record:
+
+@example
+restic-guix restore remote-ftp -t `pwd`/restored -i .config/guix/channels.scm latest
@end example
@c %start of fragment
@@ -41667,8 +41675,7 @@ Miscellaneous Services
@item @code{extra-flags} (default: @code{'()}) (type: list-of-lowerables)
A list of values that are lowered to strings. These will be passed as
-command-line arguments to the current job @command{restic backup}
-invokation.
+command-line arguments to the current @command{restic} invokation.
@end table
diff --git a/gnu/services/backup.scm b/gnu/services/backup.scm
index 555e9fc959..83d388143e 100644
--- a/gnu/services/backup.scm
+++ b/gnu/services/backup.scm
@@ -46,7 +46,7 @@ (define-module (gnu services backup)
restic-backup-configuration-fields
restic-backup-configuration-jobs
- restic-backup-job-program
+ restic-action-program
restic-backup-job->mcron-job
restic-guix
restic-guix-wrapper-package
@@ -97,7 +97,7 @@ (define-configuration/no-serialization restic-backup-job
(extra-flags
(list-of-lowerables '())
"A list of values that are lowered to strings. These will be passed as
-command-line arguments to the current job @command{restic backup} invokation."))
+command-line arguments to the current @command{restic} invokation."))
(define list-of-restic-backup-jobs?
(list-of restic-backup-job?))
@@ -107,15 +107,27 @@ (define-configuration/no-serialization restic-backup-configuration
(list-of-restic-backup-jobs '())
"The list of backup jobs for the current system."))
-(define (restic-backup-job-program config)
+(define %restic-guix-supported-actions
+ '("backup" "mount" "prune" "restore" "run" "snapshots" "unlock"))
+
+(define* (restic-action-program config action)
+ (define (format name)
+ ;; Remove from NAME characters that cannot be used in the store.
+ (string-map (lambda (chr)
+ (if (and (char-set-contains? char-set:ascii chr)
+ (char-set-contains? char-set:graphic chr)
+ (not (memv chr '(#\. #\/ #\space))))
+ chr
+ #\-))
+ name))
(let ((restic
(file-append (restic-backup-job-restic config) "/bin/restic"))
+ (name
+ (restic-backup-job-name config))
(repository
(restic-backup-job-repository config))
(password-file
(restic-backup-job-password-file config))
- (files
- (restic-backup-job-files config))
(extra-flags
(restic-backup-job-extra-flags config))
(verbose
@@ -123,55 +135,104 @@ (define (restic-backup-job-program config)
'("--verbose")
'())))
(program-file
- "restic-backup-job.scm"
+ (string-append "restic-" action "-" (format name) "-program.scm")
#~(begin
(use-modules (ice-9 popen)
- (ice-9 rdelim))
+ (ice-9 rdelim)
+ (srfi srfi-1))
+
+ (define cli-arguments
+ (let* ((cl (command-line))
+ (argc (length cl)))
+ (if (> argc 1)
+ (take-right cl (- argc 1))
+ '())))
+
(setenv "RESTIC_PASSWORD"
(with-input-from-file #$password-file read-line))
- (execlp #$restic #$restic #$@verbose
- "-r" #$repository
- #$@extra-flags
- "backup" #$@files)))))
+ (apply execlp `(#$restic #$restic #$@verbose
+ "-r" #$repository
+ #$@extra-flags
+ #$action ,@cli-arguments))))))
+
+(define* (restic-guix jobs #:key (supported-actions
+ %restic-guix-supported-actions))
+ (define action-table
+ (map
+ (lambda (action)
+ (list action
+ (map (lambda (job)
+ (list (restic-backup-job-name job)
+ (restic-action-program job action)))
+ jobs)))
+ ;; run is an alias for backup
+ (filter (lambda (a) (not (string=? a "run"))) supported-actions)))
-(define (restic-guix jobs)
(program-file
"restic-guix"
#~(begin
(use-modules (ice-9 match)
(srfi srfi-1))
+ (define action-table '#$action-table)
+ (define (assoc-table key table)
+ (first
+ (filter-map
+ (match-lambda
+ ((k v)
+ (and (string=? key k) v)))
+ table)))
(define names '#$(map restic-backup-job-name jobs))
- (define programs '#$(map restic-backup-job-program jobs))
+ (define backup-files
+ '#$(map restic-backup-job-files jobs))
+
+ (define (get-program action name)
+ (assoc-table name (assoc-table action action-table)))
- (define (get-program name)
+ (define (get-backup-files name)
(define idx
(list-index (lambda (n) (string=? n name)) names))
- (unless idx
- (error (string-append "Unknown job name " name "\n\n"
- "Possible job names are: "
- (string-join names " "))))
- (list-ref programs idx))
-
- (define (backup args)
- (define name (third args))
- (define program (get-program name))
- (execlp program program))
+ (list-ref backup-files idx))
(define (validate-args args)
- (when (not (>= (length args) 3))
- (error (string-append "Usage: " (basename (car args))
- " backup NAME"))))
+ (unless (>= (length args) 2)
+ (error (string-append "Usage: " (basename (first args))
+ " ACTION [ARGS]\n\nSupported actions are: "
+ #$(string-join supported-actions ", ") ".")))
+ (unless (member (second args) '#$supported-actions)
+ (error (string-append "Unknown action: " (second args) ". Supported"
+ "actions are: "
+ #$(string-join supported-actions ", ") "."))))
+
+ (define (validate-action-args action args)
+ (define argc (length args))
+ (when (not (>= argc 3))
+ (error (string-append "Usage: " (basename (first args))
+ " " action " JOB_NAME [ARGS]\n\nPossible job "
+ "names are: " (string-join names ", ") ".")))
+ (define job-name (third args))
+ (unless (member job-name names)
+ (error (string-append "Unknown job name: " job-name ". Possible job "
+ "names are: " (string-join names ", ") ".")))
+ (let ((program
+ (get-program
+ ;; run is just backup called with restic-backup-job-files
+ (if (string=? action "run") "backup" action)
+ job-name))
+ (rest (if (> argc 3)
+ (take-right args (- argc 3))
+ '())))
+ (values program
+ (if (string=? action "run")
+ (append rest (get-backup-files job-name))
+ rest))))
(define (main args)
(validate-args args)
(define action (second args))
- (match action
- ("backup"
- (backup args))
- (_
- (error (string-append "Unknown action: " action)))))
+ (define-values (program action-args) (validate-action-args action args))
+ (apply execlp (append (list program program) action-args)))
(main (command-line)))))
@@ -183,7 +244,7 @@ (define (restic-backup-job->mcron-job config)
(name
(restic-backup-job-name config)))
#~(job #$schedule
- #$(string-append "restic-guix backup " name)
+ #$(string-append "restic-guix run " name)
#:user #$user)))
(define (restic-guix-wrapper-package jobs)

base-commit: 9a03ab25ba889be27b34d5cebea05d5ac3b0a033
--
2.45.2
?
Your comment

Commenting via the web interface is currently disabled.

To comment on this conversation send an email to 72803@debbugs.gnu.org

To respond to this issue using the mumi CLI, first switch to it
mumi current 72803
Then, you may apply the latest patchset in this issue (with sign off)
mumi am -- -s
Or, compose a reply to this issue
mumi compose
Or, send patches to this issue
mumi send-email *.patch