Introduce unit tests for oci-container-service-type.

  • Done
  • quality assurance status badge
Details
2 participants
  • paul
  • Ludovic Courtès
Owner
unassigned
Submitted by
paul
Severity
normal
P
(address . guix-patches@gnu.org)(name . Ludovic Courtès)(address . ludo@gnu.org)
10a8cae4-a5a2-a2e0-fa64-95650ae2e703@autistici.org
Hi,

as discussed in issue #66160 and #67574 I'm sending a follow up with
some unit tests for most of the internals of oci-container-service-type.
These tests depend on the hotfix from #67574 since #66160 was merged
with a blocking bug due to a last minute feature I added during the
review process :( Hence if this gets merged before #67574 tests will fail .


Thank you for your help  and apologies for the noise,


giacomo
G
G
Giacomo Leidi wrote on 3 Dec 2023 22:56
[PATCH] tests: Add oci-container-service-type unit tests.
(address . 67613@debbugs.gnu.org)(name . Giacomo Leidi)(address . goodoldpaul@autistici.org)
20231203215630.28144-1-goodoldpaul@autistici.org
This patch is a followup to issue #66160 and issue #67574. It introduces
unit tests for the oci-container-service-type. 8 out 11 tests depend on
issue #67574 being merged since issue #66160 was merged with a blocking
bug from the beginning.

* gnu/services/docker.scm: Export
oci-container-configuration-container-user and
oci-container-configuration-workdir.
* tests/services/docker.scm: New file.
* Makefile.am (SCM_TESTS): Register it.

Change-Id: I47ed0fe36060ba84dd50b548a66f36e3df8a3710
---
Makefile.am | 1 +
gnu/services/docker.scm | 2 +
tests/services/docker.scm | 187 ++++++++++++++++++++++++++++++++++++++
3 files changed, 190 insertions(+)
create mode 100644 tests/services/docker.scm

Toggle diff (222 lines)
diff --git a/Makefile.am b/Makefile.am
index cbc3191dfc..91f7a77a94 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -564,6 +564,7 @@ SCM_TESTS = \
tests/services.scm \
tests/services/file-sharing.scm \
tests/services/configuration.scm \
+ tests/services/docker.scm \
tests/services/lightdm.scm \
tests/services/linux.scm \
tests/services/pam-mount.scm \
diff --git a/gnu/services/docker.scm b/gnu/services/docker.scm
index ebea0a473a..263cb41df3 100644
--- a/gnu/services/docker.scm
+++ b/gnu/services/docker.scm
@@ -58,6 +58,8 @@ (define-module (gnu services docker)
oci-container-configuration-network
oci-container-configuration-ports
oci-container-configuration-volumes
+ oci-container-configuration-container-user
+ oci-container-configuration-workdir
oci-container-service-type
oci-container-shepherd-service))
diff --git a/tests/services/docker.scm b/tests/services/docker.scm
new file mode 100644
index 0000000000..fad28a228c
--- /dev/null
+++ b/tests/services/docker.scm
@@ -0,0 +1,187 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2023 Giacomo Leidi <goodoldpaul@autistici.org>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+
+(define-module (tests services docker)
+ #:use-module (gnu packages docker)
+ #:use-module (gnu services docker)
+ #:use-module (guix derivations)
+ #:use-module (guix gexp)
+ #:use-module (guix monads)
+ #:use-module (guix packages)
+ #:use-module (guix store)
+ #:use-module (guix tests)
+ #:use-module (ice-9 match)
+ #:use-module (srfi srfi-1)
+ #:use-module (srfi srfi-64))
+
+
+;;; Commentary:
+;;;
+;;; Unit tests for the (gnu services docker) module.
+;;;
+;;; Code:
+
+
+;;;
+;;; Unit tests for the oci-container-service-type.
+;;;
+
+
+;;; Access some internals for whitebox testing.
+(define %store
+ (open-connection-for-tests))
+(define (gexp->sexp . x)
+ (apply (@@ (guix gexp) gexp->sexp) x))
+(define* (gexp->sexp* exp #:optional target)
+ (run-with-store %store (gexp->sexp exp (%current-system) target)
+ #:guile-for-build (%guile-for-build)))
+(define (list->sexp-list* lst)
+ (map (lambda (el)
+ (if (gexp? el)
+ (gexp->sexp* el)
+ el))
+ lst))
+(define oci-sanitize-mixed-list
+ (@@ (gnu services docker) oci-sanitize-mixed-list))
+(define (oci-container-configuration->options config)
+ (list->sexp-list*
+ ((@@ (gnu services docker) oci-container-configuration->options) config)))
+
+(test-begin "oci-containers-service")
+
+(test-group "oci-sanitize-mixed-list"
+ (define delimiter "=")
+ (define file-like-key
+ (plain-file "oci-tests-file-like-key" "some-content"))
+ (define mixed-list
+ `("any kind of string"
+ ("KEY" . "VALUE")
+ (,#~(string-append "COMPUTED" "_KEY") . "VALUE")
+ (,file-like-key . "VALUE")))
+
+ (test-assertm "successfully lower mixed values"
+ (mlet* %store-monad ((ml -> (oci-sanitize-mixed-list "field-name" mixed-list delimiter))
+ (actual -> (list->sexp-list* ml))
+ (file-like-item (lower-object file-like-key))
+ (expected -> `("any kind of string"
+ (string-append "KEY" "=" "VALUE")
+ (string-append (string-append "COMPUTED" "_KEY") "=" "VALUE")
+ (string-append ,file-like-item "=" "VALUE"))))
+ (mbegin %store-monad
+ (return
+ (every (lambda (pair)
+ (apply (if (string? (first pair))
+ string=?
+ equal?)
+ pair))
+ (zip expected actual))))))
+
+ (test-error
+ "illegal list values" #t
+ (oci-sanitize-mixed-list "field-name" '(("KEY" . "VALUE") #f) delimiter))
+
+ (test-error
+ "illegal pair member values" #t
+ (oci-sanitize-mixed-list "field-name" '(("KEY" . 1)) delimiter)))
+
+(test-group "oci-container-configuration->options"
+ (define config
+ (oci-container-configuration
+ (image "guix/guix:latest")))
+
+ (test-equal "entrypoint"
+ (list "--entrypoint" "entrypoint")
+ (oci-container-configuration->options
+ (oci-container-configuration
+ (inherit config)
+ (entrypoint "entrypoint"))))
+
+ (test-equal "environment"
+ (list "--env" '(string-append "key" "=" "value")
+ "--env" '(string-append "environment" "=" "variable"))
+ (oci-container-configuration->options
+ (oci-container-configuration
+ (inherit config)
+ (environment
+ '(("key" . "value")
+ ("environment" . "variable"))))))
+
+ (test-equal "network"
+ (list "--network" "host")
+ (oci-container-configuration->options
+ (oci-container-configuration
+ (inherit config)
+ (network "host"))))
+
+ (test-equal "container-user"
+ (list "--user" "service-account")
+ (oci-container-configuration->options
+ (oci-container-configuration
+ (inherit config)
+ (container-user "service-account"))))
+
+ (test-equal "workdir"
+ (list "--workdir" "/srv/http")
+ (oci-container-configuration->options
+ (oci-container-configuration
+ (inherit config)
+ (workdir "/srv/http"))))
+
+ (test-equal "ports"
+ (list "-p" '(string-append "10443" ":" "443")
+ "-p" '(string-append "9022" ":" "22"))
+ (oci-container-configuration->options
+ (oci-container-configuration
+ (inherit config)
+ (ports
+ '(("10443" . "443")
+ ("9022" . "22"))))))
+
+ (test-equal "volumes"
+ (list "-v" '(string-append "/gnu/store" ":" "/gnu/store")
+ "-v" '(string-append "/var/lib/guix" ":" "/var/lib/guix"))
+ (oci-container-configuration->options
+ (oci-container-configuration
+ (inherit config)
+ (volumes
+ '(("/gnu/store" . "/gnu/store")
+ ("/var/lib/guix" . "/var/lib/guix"))))))
+
+ (test-equal "complete configuration"
+ (list "--entrypoint" "entrypoint"
+ "--env" '(string-append "key" "=" "value")
+ "--network" "host"
+ "--user" "service-account"
+ "--workdir" "/srv/http"
+ "-p" '(string-append "10443" ":" "443")
+ "-v" '(string-append "/gnu/store" ":" "/gnu/store"))
+ (oci-container-configuration->options
+ (oci-container-configuration
+ (inherit config)
+ (entrypoint "entrypoint")
+ (environment
+ '(("key" . "value")))
+ (network "host")
+ (container-user "service-account")
+ (workdir "/srv/http")
+ (ports
+ '(("10443" . "443")))
+ (volumes
+ '(("/gnu/store" . "/gnu/store")))))))
+
+(test-end "oci-containers-service")

base-commit: 2c9ac9ab20c76abe570ff83f8746fa089fea3047
--
2.41.0
L
L
Ludovic Courtès wrote on 10 Dec 2023 22:47
Re: bug#67613: Introduce unit tests for oci-container-service-type.
(name . Giacomo Leidi)(address . goodoldpaul@autistici.org)(address . 67613@debbugs.gnu.org)
87lea13f3e.fsf_-_@gnu.org
Hello,

Giacomo Leidi <goodoldpaul@autistici.org> skribis:

Toggle quote (13 lines)
> This patch is a followup to issue #66160 and issue #67574. It introduces
> unit tests for the oci-container-service-type. 8 out 11 tests depend on
> issue #67574 being merged since issue #66160 was merged with a blocking
> bug from the beginning.
>
> * gnu/services/docker.scm: Export
> oci-container-configuration-container-user and
> oci-container-configuration-workdir.
> * tests/services/docker.scm: New file.
> * Makefile.am (SCM_TESTS): Register it.
>
> Change-Id: I47ed0fe36060ba84dd50b548a66f36e3df8a3710

Thanks for working on this!

To me, what’s really helpful is a system test: a test that spins up a VM
running an OCI service and makes sure said service is functional.
Apologies if I wasn’t clear!

Unit tests can be interesting too, but only if their “bug-finding
performance” is good. The tests below, for instance, are likely to be
mirroring the implementation too closely to be really able to find bugs:

Toggle quote (24 lines)
> + (test-equal "environment"
> + (list "--env" '(string-append "key" "=" "value")
> + "--env" '(string-append "environment" "=" "variable"))
> + (oci-container-configuration->options
> + (oci-container-configuration
> + (inherit config)
> + (environment
> + '(("key" . "value")
> + ("environment" . "variable"))))))
> +
> + (test-equal "network"
> + (list "--network" "host")
> + (oci-container-configuration->options
> + (oci-container-configuration
> + (inherit config)
> + (network "host"))))
> +
> + (test-equal "container-user"
> + (list "--user" "service-account")
> + (oci-container-configuration->options
> + (oci-container-configuration
> + (inherit config)
> + (container-user "service-account"))))

Thus my suggestion would be to instead focus on a system test, like
those in (gnu tests docker).

Does that make sense? WDYT?

Ludo’.
P
(name . Ludovic Courtès)(address . ludo@gnu.org)(address . 67613@debbugs.gnu.org)
cb50582a-b886-e6c8-59a0-d71285058e43@autistici.org
Hi Ludo’,

On 12/10/23 22:47, Ludovic Courtès wrote:
Toggle quote (5 lines)
> Thus my suggestion would be to instead focus on a system test, like
> those in (gnu tests docker).
>
> Does that make sense? WDYT?

I definitely misunderstood, I'll work also on system tests like those
you pointed out. Thank you, I was not aware of them, I was wondering how
do I run them?

guix shell --pure -D guix -- make check TESTS=gnu/tests/docker.scm

gives me

============================================================================
Testsuite summary for GNU Guix 1.3.0.50882-34e1c
============================================================================
# TOTAL: 0
# PASS:  0
# SKIP:  0
# XFAIL: 0
# FAIL:  0
# XPASS: 0
# ERROR: 0
============================================================================

Thank you,

giacomo
Attachment: file
L
L
Ludovic Courtès wrote on 14 Dec 2023 19:34
(name . paul)(address . goodoldpaul@autistici.org)(address . 67613@debbugs.gnu.org)
87wmtgtyy4.fsf@gnu.org
Hi,

paul <goodoldpaul@autistici.org> skribis:

Toggle quote (4 lines)
> I definitely misunderstood, I'll work also on system tests like those
> you pointed out. Thank you, I was not aware of them, I was wondering
> how do I run them?

With ‘make check-system TESTS=…’:


Apologies for the miscommunication!

Ludo’.
P
(name . Ludovic Courtès)(address . ludo@gnu.org)(address . 67613@debbugs.gnu.org)
05d4f2f7-01ff-65d1-107f-f71b8e103de0@autistici.org
Hi Ludo’ ,

I should have created a suitable system test for the
oci-container-service-type. Thanks to a nice input from
@graywolf@emacs.ch on mastodon, and actually to be able to run the test
since the vm doesn't have internet access and can't pull OCI images, I
implemented a new oci-image record that can be given some lowerable
value that can be lowered to an OCI tarballed image and passed to the
image field of the oci-container-configuration record. I'd like to point
out two things:

- It's the first time I use Guix internal API to build derivations, I
took most of my implementation from other places around Guix and I hope
is sound but I may have missed something. I'd like your feedback about it.

- I was tempted to make the image field of the
oci-container-configuration record directly only accept oci-image
records (hence making the value field of oci-image optional) but that
would break existing configurations. I'm not sure about the contract we
have for configuration records API, should I wait 1.5.0 for this change?


I'm sending an updated patchset, thank you for all your help and efforts.


giacomo
G
G
Giacomo Leidi wrote on 11 Jan 21:39 +0100
[PATCH v2 2/5] gnu: docker: Allow setting host environment variables in oci-container-configuration.
(address . 67613@debbugs.gnu.org)(name . Giacomo Leidi)(address . goodoldpaul@autistici.org)
20240111203954.29335-2-goodoldpaul@autistici.org
* gnu/services/docker.scm (oci-container-configuration)
[host-environment]: New field;
(oci-sanitize-host-environment): sanitize it;
(oci-container-shepherd-service): use it.

Change-Id: I4d54d37736cf09f042a71cb0b6e673abc0948d9c
---
gnu/services/docker.scm | 31 +++++++++++++++++++++++++++++--
1 file changed, 29 insertions(+), 2 deletions(-)

Toggle diff (72 lines)
diff --git a/gnu/services/docker.scm b/gnu/services/docker.scm
index b4fd94d1fd..7706b4a29a 100644
--- a/gnu/services/docker.scm
+++ b/gnu/services/docker.scm
@@ -5,7 +5,7 @@
;;; Copyright © 2020 Efraim Flashner <efraim@flashner.co.il>
;;; Copyright © 2020 Jesse Dowell <jessedowell@gmail.com>
;;; Copyright © 2021 Brice Waegeneire <brice@waegenei.re>
-;;; Copyright © 2023 Giacomo Leidi <goodoldpaul@autistici.org>
+;;; Copyright © 2023, 2024 Giacomo Leidi <goodoldpaul@autistici.org>
;;;
;;; This file is part of GNU Guix.
;;;
@@ -285,6 +285,11 @@ (define (oci-sanitize-mixed-list name value delimiter)
name el)))))
value))
+(define (oci-sanitize-host-environment value)
+ ;; Expected spec format:
+ ;; '(("HOME" . "/home/nobody") "JAVA_HOME=/java")
+ (oci-sanitize-mixed-list "host-environment" value "="))
+
(define (oci-sanitize-environment value)
;; Expected spec format:
;; '(("HOME" . "/home/nobody") "JAVA_HOME=/java")
@@ -330,6 +335,24 @@ (define-configuration/no-serialization oci-container-configuration
(entrypoint
(maybe-string)
"Overwrite the default entrypoint (@code{ENTRYPOINT}) of the image.")
+ (host-environment
+ (list '())
+ "Set environment variables in the host environment where @command{docker run}
+is invoked. This is especially useful to pass secrets from the host to the
+container without having them on the @command{docker run}'s command line: by
+setting the @{MYSQL_PASSWORD} on the host and by passing
+@code{--env MYSQL_PASSWORD} through the @code{extra-arguments} field, it is
+possible to securely set values in the container environment. This field's
+value can be a list of pairs or strings, even mixed:
+
+@lisp
+(list '(\"LANGUAGE\" . \"eo:ca:eu\")
+ \"JAVA_HOME=/opt/java\")
+@end lisp
+
+Pair members can be strings, gexps or file-like objects. Strings are passed
+directly to @code{make-forkexec-constructor}."
+ (sanitizer oci-sanitize-host-environment))
(environment
(list '())
"Set environment variables. This can be a list of pairs or strings, even
@@ -450,6 +473,8 @@ (define (guess-name name image)
(let* ((docker-command (file-append docker-cli "/bin/docker"))
(user (oci-container-configuration-user config))
(group (oci-container-configuration-group config))
+ (host-environment
+ (oci-container-configuration-host-environment config))
(command (oci-container-configuration-command config))
(provision (oci-container-configuration-provision config))
(image (oci-container-configuration-image config))
@@ -471,7 +496,9 @@ (define (guess-name name image)
"--name" #$name
#$@options #$@extra-arguments #$image #$@command)
#:user #$user
- #:group #$group))
+ #:group #$group
+ #:environment-variables
+ (list #$@host-environment)))
(stop
#~(lambda _
(invoke #$docker-command "rm" "-f" #$name)))
--
2.41.0
G
G
Giacomo Leidi wrote on 11 Jan 21:39 +0100
[PATCH v2 1/5] gnu: docker: Provide escape hatch in oci-container-configuration.
(address . 67613@debbugs.gnu.org)(name . Giacomo Leidi)(address . goodoldpaul@autistici.org)
20240111203954.29335-1-goodoldpaul@autistici.org
* gnu/services/docker.scm (oci-container-configuration)
[extra-arguments]: New field;
(oci-sanitize-extra-arguments): sanitize it;
(oci-container-shepherd-service): use it;
* doc/guix.texi: document it.

Change-Id: I54c74ac2fe0f5ca65ca5a1d0d7f3fb55ff428063
---
doc/guix.texi | 13 ++++++++++---
gnu/services/docker.scm | 42 ++++++++++++++++++++++++++++++++++-------
2 files changed, 45 insertions(+), 10 deletions(-)

Toggle diff (156 lines)
diff --git a/doc/guix.texi b/doc/guix.texi
index 395545bed7..ce239c603d 100644
--- a/doc/guix.texi
+++ b/doc/guix.texi
@@ -39844,7 +39844,8 @@ Set environment variables. This can be a list of pairs or strings, even mixed:
"JAVA_HOME=/opt/java")
@end lisp
-String are passed directly to the Docker CLI. You can refer to the
+Pair members can be strings, gexps or file-like objects.
+Strings are passed directly to the Docker CLI. You can refer to the
@uref{https://docs.docker.com/engine/reference/commandline/run/#env,upstream}
documentation for semantics.
@@ -39868,7 +39869,8 @@ list of pairs or strings, even mixed:
"10443:443")
@end lisp
-String are passed directly to the Docker CLI. You can refer to the
+Pair members can be strings, gexps or file-like objects.
+Strings are passed directly to the Docker CLI. You can refer to the
@uref{https://docs.docker.com/engine/reference/commandline/run/#publish,upstream}
documentation for semantics.
@@ -39881,7 +39883,8 @@ list of pairs or strings, even mixed:
"/gnu/store:/gnu/store")
@end lisp
-String are passed directly to the Docker CLI. You can refer to the
+Pair members can be strings, gexps or file-like objects.
+Strings are passed directly to the Docker CLI. You can refer to the
@uref{https://docs.docker.com/engine/reference/commandline/run/#volume,upstream}
documentation for semantics.
@@ -39896,6 +39899,10 @@ You can refer to the
@url{https://docs.docker.com/engine/reference/run/#workdir,upstream}
documentation for semantics.
+@item @code{extra-arguments} (default: @code{()}) (type: list)
+A list of strings, gexps or file-like objects that will be directly
+passed to the @command{docker run} invokation.
+
@end table
@end deftp
diff --git a/gnu/services/docker.scm b/gnu/services/docker.scm
index 4d32b96847..b4fd94d1fd 100644
--- a/gnu/services/docker.scm
+++ b/gnu/services/docker.scm
@@ -58,6 +58,9 @@ (define-module (gnu services docker)
oci-container-configuration-network
oci-container-configuration-ports
oci-container-configuration-volumes
+ oci-container-configuration-container-user
+ oci-container-configuration-workdir
+ oci-container-configuration-extra-arguments
oci-container-service-type
oci-container-shepherd-service))
@@ -297,6 +300,21 @@ (define (oci-sanitize-volumes value)
;; '(("/mnt/dir" . "/dir") "/run/current-system/profile:/java")
(oci-sanitize-mixed-list "volumes" value ":"))
+(define (oci-sanitize-extra-arguments value)
+ (define (valid? member)
+ (or (string? member)
+ (gexp? member)
+ (file-like? member)))
+ (map
+ (lambda (el)
+ (if (valid? el)
+ el
+ (raise
+ (formatted-message
+ (G_ "extra arguments may only be strings, gexps or file-like objects
+but ~a was found") el))))
+ value))
+
(define-maybe/no-serialization string)
(define-configuration/no-serialization oci-container-configuration
@@ -322,7 +340,8 @@ (define-configuration/no-serialization oci-container-configuration
\"JAVA_HOME=/opt/java\")
@end lisp
-String are passed directly to the Docker CLI. You can refer to the
+Pair members can be strings, gexps or file-like objects. Strings are passed
+directly to the Docker CLI. You can refer to the
@url{https://docs.docker.com/engine/reference/commandline/run/#env,upstream}
documentation for semantics."
(sanitizer oci-sanitize-environment))
@@ -347,7 +366,8 @@ (define-configuration/no-serialization oci-container-configuration
\"10443:443\")
@end lisp
-String are passed directly to the Docker CLI. You can refer to the
+Pair members can be strings, gexps or file-like objects. Strings are passed
+directly to the Docker CLI. You can refer to the
@url{https://docs.docker.com/engine/reference/commandline/run/#publish,upstream}
documentation for semantics."
(sanitizer oci-sanitize-ports))
@@ -361,7 +381,8 @@ (define-configuration/no-serialization oci-container-configuration
\"/gnu/store:/gnu/store\")
@end lisp
-String are passed directly to the Docker CLI. You can refer to the
+Pair members can be strings, gexps or file-like objects. Strings are passed
+directly to the Docker CLI. You can refer to the
@url{https://docs.docker.com/engine/reference/commandline/run/#volume,upstream}
documentation for semantics."
(sanitizer oci-sanitize-volumes))
@@ -375,7 +396,12 @@ (define-configuration/no-serialization oci-container-configuration
"Set the current working for the spawned Shepherd service.
You can refer to the
@url{https://docs.docker.com/engine/reference/run/#workdir,upstream}
-documentation for semantics."))
+documentation for semantics.")
+ (extra-arguments
+ (list '())
+ "A list of strings, gexps or file-like objects that will be directly passed
+to the @command{docker run} invokation."
+ (sanitizer oci-sanitize-extra-arguments)))
(define oci-container-configuration->options
(lambda (config)
@@ -428,7 +454,9 @@ (define (guess-name name image)
(provision (oci-container-configuration-provision config))
(image (oci-container-configuration-image config))
(options (oci-container-configuration->options config))
- (name (guess-name provision image)))
+ (name (guess-name provision image))
+ (extra-arguments
+ (oci-container-configuration-extra-arguments config)))
(shepherd-service (provision `(,(string->symbol name)))
(requirement '(dockerd user-processes))
@@ -441,7 +469,7 @@ (define (guess-name name image)
;; docker run [OPTIONS] IMAGE [COMMAND] [ARG...]
(list #$docker-command "run" "--rm"
"--name" #$name
- #$@options #$image #$@command)
+ #$@options #$@extra-arguments #$image #$@command)
#:user #$user
#:group #$group))
(stop
@@ -482,5 +510,5 @@ (define oci-container-service-type
(extend append)
(compose concatenate)
(description
- "This service allows the management of Docker and OCI
+ "This service allows the management of OCI
containers as Shepherd services.")))

base-commit: 637b72e2b83a6332849218ef1f193124fa8239eb
--
2.41.0
G
G
Giacomo Leidi wrote on 11 Jan 21:39 +0100
[PATCH v2 3/5] gnu: docker: Allow setting Shepherd dependencies in oci-container-configuration.
(address . 67613@debbugs.gnu.org)(name . Giacomo Leidi)(address . goodoldpaul@autistici.org)
20240111203954.29335-3-goodoldpaul@autistici.org
* gnu/services/docker.scm (oci-container-configuration)
[requirement]: New field;
(list-of-symbols): sanitize it;
(oci-container-shepherd-service): use it.

Change-Id: Ic0ba336a2257d6ef7c658cfc6cd630116661f581
---
gnu/services/docker.scm | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)

Toggle diff (44 lines)
diff --git a/gnu/services/docker.scm b/gnu/services/docker.scm
index 7706b4a29a..43ffb71901 100644
--- a/gnu/services/docker.scm
+++ b/gnu/services/docker.scm
@@ -320,6 +320,9 @@ (define (valid? member)
but ~a was found") el))))
value))
+(define list-of-symbols?
+ (list-of symbol?))
+
(define-maybe/no-serialization string)
(define-configuration/no-serialization oci-container-configuration
@@ -376,6 +379,10 @@ (define-configuration/no-serialization oci-container-configuration
(provision
(maybe-string)
"Set the name of the provisioned Shepherd service.")
+ (requirement
+ (list-of-symbols '())
+ "Set additional Shepherd services dependencies to the provisioned Shepherd
+service.")
(network
(maybe-string)
"Set a Docker network for the spawned container.")
@@ -477,6 +484,7 @@ (define (guess-name name image)
(oci-container-configuration-host-environment config))
(command (oci-container-configuration-command config))
(provision (oci-container-configuration-provision config))
+ (requirement (oci-container-configuration-requirement config))
(image (oci-container-configuration-image config))
(options (oci-container-configuration->options config))
(name (guess-name provision image))
@@ -484,7 +492,7 @@ (define (guess-name name image)
(oci-container-configuration-extra-arguments config)))
(shepherd-service (provision `(,(string->symbol name)))
- (requirement '(dockerd user-processes))
+ (requirement `(dockerd user-processes ,@requirement))
(respawn? #f)
(documentation
(string-append
--
2.41.0
G
G
Giacomo Leidi wrote on 11 Jan 21:39 +0100
[PATCH v2 5/5] gnu: Add tests and documentation for oci-container-service-type.
(address . 67613@debbugs.gnu.org)(name . Giacomo Leidi)(address . goodoldpaul@autistici.org)
20240111203954.29335-5-goodoldpaul@autistici.org
* doc/guix.texi: Add documentation for the oci-image record and update
the oci-container-configuration documentation.
* gnu/tests/docker.scm (run-oci-container-test): New variable;
(%test-oci-container): new variable.

Change-Id: Id8f4f5454aa3b88d8aa3fa47de823e921acece05
---
doc/guix.texi | 91 +++++++++++++++++++++++++++-
gnu/services/docker.scm | 6 +-
gnu/tests/docker.scm | 131 +++++++++++++++++++++++++++++++++++++++-
3 files changed, 221 insertions(+), 7 deletions(-)

Toggle diff (314 lines)
diff --git a/doc/guix.texi b/doc/guix.texi
index ce239c603d..1916a00412 100644
--- a/doc/guix.texi
+++ b/doc/guix.texi
@@ -39790,6 +39790,17 @@ processes as Shepherd Services.
@lisp
(service oci-container-service-type
(list
+ (oci-container-configuration
+ (image
+ (oci-image
+ (repository "guile")
+ (tag "3")
+ (value (specifications->manifest '("guile")))
+ (pack-options '(#:symlinks (("/bin/guile" -> "bin/guile"))
+ #:max-layers 2))))
+ (entrypoint "/bin/guile")
+ (command
+ '("-c" "(display \"hello!\n\")")))
(oci-container-configuration
(image "prom/prometheus")
(network "host")
@@ -39836,6 +39847,23 @@ Overwrite the default command (@code{CMD}) of the image.
@item @code{entrypoint} (default: @code{""}) (type: string)
Overwrite the default entrypoint (@code{ENTRYPOINT}) of the image.
+@item @code{host-environment} (default: @code{()}) (type: list)
+Set environment variables in the host environment where @command{docker
+run} is invoked. This is especially useful to pass secrets from the
+host to the container without having them on the @command{docker run}'s
+command line: by setting the @code{MYSQL_PASSWORD} on the host and by passing
+@code{--env MYSQL_PASSWORD} through the @code{extra-arguments} field, it is
+possible to securely set values in the container environment. This field's
+value can be a list of pairs or strings, even mixed:
+
+@lisp
+(list '(\"LANGUAGE\" . \"eo:ca:eu\")
+ \"JAVA_HOME=/opt/java\")
+@end lisp
+
+Pair members can be strings, gexps or file-like objects. Strings are passed
+directly to @code{make-forkexec-constructor}.
+
@item @code{environment} (default: @code{()}) (type: list)
Set environment variables. This can be a list of pairs or strings, even mixed:
@@ -39849,14 +39877,19 @@ Strings are passed directly to the Docker CLI. You can refer to the
@uref{https://docs.docker.com/engine/reference/commandline/run/#env,upstream}
documentation for semantics.
-@item @code{image} (type: string)
-The image used to build the container. Images are resolved by the
-Docker Engine, and follow the usual format
+@item @code{image} (type: string-or-oci-image)
+The image used to build the container. It can be a string or an
+@code{oci-image} record. Strings are resolved by the Docker Engine, and
+follow the usual format
@code{myregistry.local:5000/testing/test-image:tag}.
@item @code{provision} (default: @code{""}) (type: string)
Set the name of the provisioned Shepherd service.
+@item @code{requirement} (default: @code{()}) (type: list-of-symbols)
+Set additional Shepherd services dependencies to the provisioned
+Shepherd service.
+
@item @code{network} (default: @code{""}) (type: string)
Set a Docker network for the spawned container.
@@ -39908,6 +39941,58 @@ passed to the @command{docker run} invokation.
@end deftp
+@c %end of fragment
+
+@c %start of fragment
+
+@deftp {Data Type} oci-image
+Available @code{oci-image} fields are:
+
+@table @asis
+@item @code{repository} (type: string)
+A string like @code{myregistry.local:5000/testing/test-image} that names
+the OCI image.
+
+@item @code{tag} (default: @code{"latest"}) (type: string)
+A string representing the OCI image tag. Defaults to @code{latest}.
+
+@item @code{value} (type: oci-lowerable-image)
+A @code{manifest} or @code{operating-system} record that will be lowered
+into an OCI compatible tarball. Otherwise this field's value can be a
+gexp or a file-like object that evaluates to an OCI compatible tarball.
+
+@item @code{pack-options} (default: @code{()}) (type: list)
+An optional set of keyword arguments that will be passed to the
+@code{docker-image} procedure from @code{guix scripts pack}. They can
+be used to replicate @command{guix pack} behavior:
+
+@lisp
+(oci-image
+ (repository "guile")
+ (tag "3")
+ (value
+ (specifications->manifest '("guile")))
+ (pack-options '(#:symlinks (("/bin/guile" -> "bin/guile"))
+ #:max-layers 2)))
+@end lisp
+
+If the @code{value} field is an @code{operating-system} record, this field's
+value will be ignored.
+
+@item @code{system} (default: @code{""}) (type: string)
+Attempt to build for a given system, e.g. "i686-linux"
+
+@item @code{target} (default: @code{""}) (type: string)
+Attempt to cross-build for a given triple, e.g. "aarch64-linux-gnu"
+
+@item @code{grafts?} (default: @code{#f}) (type: boolean)
+Whether to allow grafting or not in the pack build.
+
+@end table
+
+@end deftp
+
+
@c %end of fragment
@cindex Audit
diff --git a/gnu/services/docker.scm b/gnu/services/docker.scm
index 58a725737c..7aff8dcc5f 100644
--- a/gnu/services/docker.scm
+++ b/gnu/services/docker.scm
@@ -420,7 +420,7 @@ (define-configuration/no-serialization oci-container-configuration
"Set environment variables in the host environment where @command{docker run}
is invoked. This is especially useful to pass secrets from the host to the
container without having them on the @command{docker run}'s command line: by
-setting the @{MYSQL_PASSWORD} on the host and by passing
+setting the @code{MYSQL_PASSWORD} on the host and by passing
@code{--env MYSQL_PASSWORD} through the @code{extra-arguments} field, it is
possible to securely set values in the container environment. This field's
value can be a list of pairs or strings, even mixed:
@@ -435,8 +435,8 @@ (define-configuration/no-serialization oci-container-configuration
(sanitizer oci-sanitize-host-environment))
(environment
(list '())
- "Set environment variables. This can be a list of pairs or strings, even
-mixed:
+ "Set environment variables inside the container. This can be a list of pairs
+or strings, even mixed:
@lisp
(list '(\"LANGUAGE\" . \"eo:ca:eu\")
diff --git a/gnu/tests/docker.scm b/gnu/tests/docker.scm
index 9e9d2e2d07..d550136b4a 100644
--- a/gnu/tests/docker.scm
+++ b/gnu/tests/docker.scm
@@ -1,6 +1,7 @@
;;; GNU Guix --- Functional package management for GNU
;;; Copyright © 2019 Danny Milosavljevic <dannym@scratchpost.org>
;;; Copyright © 2019-2023 Ludovic Courtès <ludo@gnu.org>
+;;; Copyright © 2024 Giacomo Leidi <goodoldpaul@autistici.org>
;;;
;;; This file is part of GNU Guix.
;;;
@@ -29,6 +30,7 @@ (define-module (gnu tests docker)
#:use-module (gnu services networking)
#:use-module (gnu services docker)
#:use-module (gnu services desktop)
+ #:use-module (gnu packages)
#:use-module ((gnu packages base) #:select (glibc))
#:use-module (gnu packages guile)
#:use-module (gnu packages docker)
@@ -43,7 +45,8 @@ (define-module (gnu tests docker)
#:use-module (guix build-system trivial)
#:use-module ((guix licenses) #:prefix license:)
#:export (%test-docker
- %test-docker-system))
+ %test-docker-system
+ %test-oci-container))
(define %docker-os
(simple-operating-system
@@ -316,3 +319,129 @@ (define %test-docker-system
(locale-libcs (list glibc)))
#:type docker-image-type)))
run-docker-system-test)))))
+
+
+(define %oci-os
+ (simple-operating-system
+ (service dhcp-client-service-type)
+ (service dbus-root-service-type)
+ (service polkit-service-type)
+ (service elogind-service-type)
+ (service docker-service-type)
+ (extra-special-file "/shared.txt"
+ (plain-file "shared.txt" "hello"))
+ (service oci-container-service-type
+ (list
+ (oci-container-configuration
+ (image
+ (oci-image
+ (repository "guile")
+ (value
+ (specifications->manifest '("guile")))
+ (pack-options
+ '(#:symlinks (("/bin" -> "bin"))))))
+ (entrypoint
+ "/bin/guile")
+ (command
+ '("-c" "(let l ((c 300))(display c)(sleep 1)(when(positive? c)(l (- c 1))))"))
+ (host-environment
+ '(("VARIABLE" . "value")))
+ (volumes
+ '(("/shared.txt" . "/shared.txt:ro")))
+ (extra-arguments
+ '("--env" "VARIABLE")))))))
+
+(define (run-oci-container-test)
+ "Run IMAGE as an OCI backed Shepherd service, inside OS."
+
+ (define os
+ (marionette-operating-system
+ (operating-system-with-gc-roots
+ %oci-os
+ (list))
+ #:imported-modules '((gnu services herd)
+ (guix combinators))))
+
+ (define vm
+ (virtual-machine
+ (operating-system os)
+ (volatile? #f)
+ (memory-size 1024)
+ (disk-image-size (* 3000 (expt 2 20)))
+ (port-forwardings '())))
+
+ (define test
+ (with-imported-modules '((gnu build marionette))
+ #~(begin
+ (use-modules (srfi srfi-11) (srfi srfi-64)
+ (gnu build marionette))
+
+ (define marionette
+ ;; Relax timeout to accommodate older systems and
+ ;; allow for pulling the image.
+ (make-marionette (list #$vm) #:timeout 60))
+
+ (test-runner-current (system-test-runner #$output))
+ (test-begin "oci-container")
+
+ (test-assert "dockerd running"
+ (marionette-eval
+ '(begin
+ (use-modules (gnu services herd))
+ (match (start-service 'dockerd)
+ (#f #f)
+ (('service response-parts ...)
+ (match (assq-ref response-parts 'running)
+ ((pid) (number? pid))))))
+ marionette))
+
+ (sleep 10) ; let service start
+
+ (test-assert "docker-guile running"
+ (marionette-eval
+ '(begin
+ (use-modules (gnu services herd))
+ (match (start-service 'docker-guile)
+ (#f #f)
+ (('service response-parts ...)
+ (match (assq-ref response-parts 'running)
+ ((pid) (number? pid))))))
+ marionette))
+
+ (test-equal "passing host environment variables and volumes"
+ '("value" "hello")
+ (marionette-eval
+ `(begin
+ (use-modules (ice-9 popen)
+ (ice-9 rdelim))
+
+ (define slurp
+ (lambda args
+ (let* ((port (apply open-pipe* OPEN_READ args))
+ (output (let ((line (read-line port)))
+ (if (eof-object? line)
+ ""
+ line)))
+ (status (close-pipe port)))
+ output)))
+ (let* ((response1 (slurp
+ ,(string-append #$docker-cli "/bin/docker")
+ "exec" "docker-guile"
+ "/bin/guile" "-c" "(display (getenv \"VARIABLE\"))"))
+ (response2 (slurp
+ ,(string-append #$docker-cli "/bin/docker")
+ "exec" "docker-guile"
+ "/bin/guile" "-c" "(begin (use-modules (ice-9 popen) (ice-9 rdelim))
+(display (call-with-input-file \"/shared.txt\" read-line)))")))
+ (list response1 response2)))
+ marionette))
+
+ (test-end))))
+
+ (gexp->derivation "oci-container-test" test))
+
+(define %test-oci-container
+ (system-test
+ (name "oci-container")
+ (description "Test OCI backed Shepherd service.")
+ (value (run-oci-container-test))))
--
2.41.0
G
G
Giacomo Leidi wrote on 11 Jan 21:39 +0100
[PATCH v2 4/5] gnu: docker: Allow passing tarballs for images in oci-container-configuration.
(address . 67613@debbugs.gnu.org)(name . Giacomo Leidi)(address . goodoldpaul@autistici.org)
20240111203954.29335-4-goodoldpaul@autistici.org
This commit allows for loading an OCI image tarball before running an
OCI backed Shepherd service. It does so by adding a one shot Shepherd
service to the dependencies of the OCI backed service that at boot runs
docker load on the tarball.

* gnu/services/docker.scm (oci-image): New record;
(lower-oci-image): new variable, lower it;
(string-or-oci-image?): sanitize it;
(oci-container-configuration)[image]: allow also for oci-image records;
(oci-container-shepherd-service): use it;
(%oci-image-loader): new variable.

Change-Id: Ie504f479ea0d47f74b0ec5df9085673ffd3f639d
---
gnu/services/docker.scm | 244 ++++++++++++++++++++++++++++++++++++----
1 file changed, 219 insertions(+), 25 deletions(-)

Toggle diff (344 lines)
diff --git a/gnu/services/docker.scm b/gnu/services/docker.scm
index 43ffb71901..58a725737c 100644
--- a/gnu/services/docker.scm
+++ b/gnu/services/docker.scm
@@ -23,11 +23,14 @@
;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
(define-module (gnu services docker)
+ #:use-module (gnu image)
#:use-module (gnu services)
#:use-module (gnu services configuration)
#:use-module (gnu services base)
#:use-module (gnu services dbus)
#:use-module (gnu services shepherd)
+ #:use-module (gnu system)
+ #:use-module (gnu system image)
#:use-module (gnu system setuid)
#:use-module (gnu system shadow)
#:use-module (gnu packages admin) ;shadow
@@ -37,7 +40,11 @@ (define-module (gnu services docker)
#:use-module (guix diagnostics)
#:use-module (guix gexp)
#:use-module (guix i18n)
+ #:use-module (guix monads)
#:use-module (guix packages)
+ #:use-module (guix profiles)
+ #:use-module ((guix scripts pack) #:prefix pack:)
+ #:use-module (guix store)
#:use-module (srfi srfi-1)
#:use-module (ice-9 format)
#:use-module (ice-9 match)
@@ -45,6 +52,16 @@ (define-module (gnu services docker)
#:export (docker-configuration
docker-service-type
singularity-service-type
+ oci-image
+ oci-image?
+ oci-image-fields
+ oci-image-repository
+ oci-image-tag
+ oci-image-value
+ oci-image-pack-options
+ oci-image-target
+ oci-image-system
+ oci-image-grafts?
oci-container-configuration
oci-container-configuration?
oci-container-configuration-fields
@@ -52,9 +69,11 @@ (define-module (gnu services docker)
oci-container-configuration-group
oci-container-configuration-command
oci-container-configuration-entrypoint
+ oci-container-configuration-host-environment
oci-container-configuration-environment
oci-container-configuration-image
oci-container-configuration-provision
+ oci-container-configuration-requirement
oci-container-configuration-network
oci-container-configuration-ports
oci-container-configuration-volumes
@@ -62,7 +81,8 @@ (define-module (gnu services docker)
oci-container-configuration-workdir
oci-container-configuration-extra-arguments
oci-container-service-type
- oci-container-shepherd-service))
+ oci-container-shepherd-service
+ %oci-container-accounts))
(define-maybe file-like)
@@ -320,11 +340,68 @@ (define (valid? member)
but ~a was found") el))))
value))
+(define (oci-image-reference image)
+ (if (string? image)
+ image
+ (string-append (oci-image-repository image)
+ ":" (oci-image-tag image))))
+
+(define (oci-lowerable-image? image)
+ (or (manifest? image)
+ (operating-system? image)
+ (gexp? image)
+ (file-like? image)))
+
+(define (string-or-oci-image? image)
+ (or (string? image)
+ (oci-image? image)))
+
(define list-of-symbols?
(list-of symbol?))
(define-maybe/no-serialization string)
+(define-configuration/no-serialization oci-image
+ (repository
+ (string)
+ "A string like @code{myregistry.local:5000/testing/test-image} that names
+the OCI image.")
+ (tag
+ (string "latest")
+ "A string representing the OCI image tag. Defaults to @code{latest}.")
+ (value
+ (oci-lowerable-image)
+ "A @code{manifest} or @code{operating-system} record that will be lowered
+into an OCI compatible tarball. Otherwise this field's value can be a gexp
+or a file-like object that evaluates to an OCI compatible tarball.")
+ (pack-options
+ (list '())
+ "An optional set of keyword arguments that will be passed to the
+@code{docker-image} procedure from @code{guix scripts pack}. They can be used
+to replicate @command{guix pack} behavior:
+
+@lisp
+(oci-image
+ (repository \"guile\")
+ (tag \"3\")
+ (manifest (specifications->manifest '(\"guile\")))
+ (pack-options
+ '(#:symlinks ((\"/bin/guile\" -> \"bin/guile\"))
+ #:max-layers 2)))
+@end lisp
+
+If the @code{value} field is an @code{operating-system} record, this field's
+value will be ignored.")
+ (system
+ (maybe-string)
+ "Attempt to build for a given system, e.g. \"i686-linux\"")
+ (target
+ (maybe-string)
+ "Attempt to cross-build for a given triple, e.g. \"aarch64-linux-gnu\"")
+ (grafts?
+ (boolean #f)
+ "Whether to allow grafting or not in the pack build."))
+
(define-configuration/no-serialization oci-container-configuration
(user
(string "oci-container")
@@ -372,8 +449,9 @@ (define-configuration/no-serialization oci-container-configuration
documentation for semantics."
(sanitizer oci-sanitize-environment))
(image
- (string)
- "The image used to build the container. Images are resolved by the Docker
+ (string-or-oci-image)
+ "The image used to build the container. It can be a string or an
+@code{oci-image} record. Strings are resolved by the Docker
Engine, and follow the usual format
@code{myregistry.local:5000/testing/test-image:tag}.")
(provision
@@ -470,14 +548,122 @@ (define oci-container-configuration->options
(list "-v" spec))
(oci-container-configuration-volumes config))))))))
+(define* (get-keyword-value args keyword #:key (default #f))
+ (let ((kv (memq keyword args)))
+ (if (and kv (>= (length kv) 2))
+ (cadr kv)
+ default)))
+
+(define (lower-operating-system os target system)
+ (mlet* %store-monad
+ ((tarball
+ (lower-object
+ (system-image (os->image os #:type docker-image-type))
+ system
+ #:target target)))
+ (return tarball)))
+
+(define (lower-manifest name image target system)
+ (define value (oci-image-value image))
+ (define options (oci-image-pack-options image))
+ (define image-reference
+ (oci-image-reference image))
+ (define image-tag
+ (let* ((extra-options
+ (get-keyword-value options #:extra-options))
+ (image-tag-option
+ (and extra-options
+ (get-keyword-value extra-options #:image-tag))))
+ (if image-tag-option
+ '()
+ `(#:extra-options (#:image-tag ,image-reference)))))
+
+ (mlet* %store-monad
+ ((_ (set-grafting
+ (oci-image-grafts? image)))
+ (guile (set-guile-for-build (default-guile)))
+ (profile
+ (profile-derivation value
+ #:target target
+ #:system system
+ #:hooks '()
+ #:locales? #f))
+ (tarball (apply pack:docker-image
+ `(,name ,profile
+ ,@options
+ ,@image-tag
+ #:localstatedir? #t))))
+ (return tarball)))
+
+(define (lower-oci-image name image)
+ (define value (oci-image-value image))
+ (define image-target (oci-image-target image))
+ (define image-system (oci-image-system image))
+ (define target
+ (if (maybe-value-set? image-target)
+ image-target
+ (%current-target-system)))
+ (define system
+ (if (maybe-value-set? image-system)
+ image-system
+ (%current-system)))
+ (with-store store
+ (run-with-store store
+ (match value
+ ((? manifest? value)
+ (lower-manifest name image target system))
+ ((? operating-system? value)
+ (lower-operating-system value target system))
+ ((or (? gexp? value)
+ (? file-like? value))
+ value)
+ (_
+ (raise
+ (formatted-message
+ (G_ "oci-image value must contain only manifest,
+operating-system, gexp or file-like records but ~a was found")
+ value))))
+ #:target target
+ #:system system)))
+
+(define (%oci-image-loader name image tag)
+ (let ((docker (file-append docker-cli "/bin/docker"))
+ (tarball (lower-oci-image name image)))
+ (with-imported-modules '((guix build utils))
+ (program-file (format #f "~a-image-loader" name)
+ #~(begin
+ (use-modules (guix build utils)
+ (ice-9 popen)
+ (ice-9 rdelim))
+
+ (format #t "Loading image for ~a from ~a...~%" #$name #$tarball)
+ (define line
+ (read-line
+ (open-input-pipe
+ (string-append #$docker " load -i " #$tarball))))
+
+ (unless (or (eof-object? line)
+ (string-null? line))
+ (format #t "~a~%" line)
+ (let ((repository&tag
+ (string-drop line
+ (string-length
+ "Loaded image: "))))
+
+ (invoke #$docker "tag" repository&tag #$tag)
+ (format #t "Tagged ~a with ~a...~%" #$tarball #$tag))))))))
+
(define (oci-container-shepherd-service config)
(define (guess-name name image)
(if (maybe-value-set? name)
name
(string-append "docker-"
- (basename (car (string-split image #\:))))))
+ (basename
+ (if (string? image)
+ (first (string-split image #\:))
+ (oci-image-repository image))))))
- (let* ((docker-command (file-append docker-cli "/bin/docker"))
+ (let* ((docker (file-append docker-cli "/bin/docker"))
(user (oci-container-configuration-user config))
(group (oci-container-configuration-group config))
(host-environment
@@ -486,6 +672,7 @@ (define (guess-name name image)
(provision (oci-container-configuration-provision config))
(requirement (oci-container-configuration-requirement config))
(image (oci-container-configuration-image config))
+ (image-reference (oci-image-reference image))
(options (oci-container-configuration->options config))
(name (guess-name provision image))
(extra-arguments
@@ -496,30 +683,37 @@ (define (guess-name name image)
(respawn? #f)
(documentation
(string-append
- "Docker backed Shepherd service for image: " image))
+ "Docker backed Shepherd service for "
+ (if (oci-image? image) name image) "."))
(start
- #~(make-forkexec-constructor
- ;; docker run [OPTIONS] IMAGE [COMMAND] [ARG...]
- (list #$docker-command "run" "--rm"
- "--name" #$name
- #$@options #$@extra-arguments #$image #$@command)
- #:user #$user
- #:group #$group
- #:environment-variables
- (list #$@host-environment)))
+ #~(lambda ()
+ (when #$(oci-image? image)
+ (invoke #$(%oci-image-loader
+ name image image-reference)))
+ (fork+exec-command
+ ;; docker run [OPTIONS] IMAGE [COMMAND] [ARG...]
+ (list #$docker "run" "--rm" "--name" #$name
+ #$@options #$@extra-arguments
+ #$image-reference #$@command)
+ #:user #$user
+ #:group #$group
+ #:environment-variables
+ (list #$@host-environment))))
(stop
#~(lambda _
- (invoke #$docker-command "rm" "-f" #$name)))
+ (invoke #$docker "rm" "-f" #$name)))
(actions
- (list
- (shepherd-action
- (name 'pull)
- (documentation
- (format #f "Pull ~a's image (~a)."
- name image))
- (procedure
- #~(lambda _
- (invoke #$docker-command "pull" #$image)))))))))
+ (if (oci-image? image)
+ '()
+ (list
+ (shepherd-action
+ (name 'pull)
+ (documentation
+ (format #f "Pull ~a's image (~a)."
+ name image))
+ (procedure
+ #~(lambda _
+ (invoke #$docker "pull" #$image))))))))))
(define %oci-container-accounts
(list (user-account
--
2.41.0
P
Re: bug#67613: Introduce unit tests for oci-container-service-type.
(name . Ludovic Courtès)(address . ludo@gnu.org)(address . 67613@debbugs.gnu.org)
237dfee6-5280-7d4f-4fec-c07f4f083880@autistici.org
Hi Ludo’ ,

I'm sending a patchset rebased on current master. I hope patch 1-3 are
non-controversial enough to be directly merged as they add features
without breaking existing configurations. About patch 4 and 5 I stand by
my request for help in my last email.

Thank you for your work,


giacomo
G
G
Giacomo Leidi wrote on 4 May 00:11 +0200
[PATCH v3 4/5] gnu: docker: Allow passing tarballs for images in oci-container-configuration.
(address . 67613@debbugs.gnu.org)(name . Giacomo Leidi)(address . goodoldpaul@autistici.org)
0c4d11411cd8efcea9c3252146cbc69c9155f2e9.1714774277.git.goodoldpaul@autistici.org
This commit allows for loading an OCI image tarball before running an
OCI backed Shepherd service. It does so by adding a one shot Shepherd
service to the dependencies of the OCI backed service that at boot runs
docker load on the tarball.

* gnu/services/docker.scm (oci-image): New record;
(lower-oci-image): new variable, lower it;
(string-or-oci-image?): sanitize it;
(oci-container-configuration)[image]: allow also for oci-image records;
(oci-container-shepherd-service): use it;
(%oci-image-loader): new variable.

Change-Id: Ie504f479ea0d47f74b0ec5df9085673ffd3f639d
---
doc/guix.texi | 70 +++++++++++-
gnu/services/docker.scm | 244 ++++++++++++++++++++++++++++++++++++----
2 files changed, 286 insertions(+), 28 deletions(-)

Toggle diff (439 lines)
diff --git a/doc/guix.texi b/doc/guix.texi
index 2b9cc5602c..451bee5615 100644
--- a/doc/guix.texi
+++ b/doc/guix.texi
@@ -40390,6 +40390,17 @@ Miscellaneous Services
@lisp
(service oci-container-service-type
(list
+ (oci-container-configuration
+ (image
+ (oci-image
+ (repository "guile")
+ (tag "3")
+ (value (specifications->manifest '("guile")))
+ (pack-options '(#:symlinks (("/bin/guile" -> "bin/guile"))
+ #:max-layers 2))))
+ (entrypoint "/bin/guile")
+ (command
+ '("-c" "(display \"hello!\n\")")))
(oci-container-configuration
(image "prom/prometheus")
(network "host")
@@ -40466,9 +40477,10 @@ Miscellaneous Services
@uref{https://docs.docker.com/engine/reference/commandline/run/#env,upstream}
documentation for semantics.
-@item @code{image} (type: string)
-The image used to build the container. Images are resolved by the
-Docker Engine, and follow the usual format
+@item @code{image} (type: string-or-oci-image)
+The image used to build the container. It can be a string or an
+@code{oci-image} record. Strings are resolved by the Docker Engine, and
+follow the usual format
@code{myregistry.local:5000/testing/test-image:tag}.
@item @code{provision} (default: @code{""}) (type: string)
@@ -40529,6 +40541,58 @@ Miscellaneous Services
@end deftp
+@c %end of fragment
+
+@c %start of fragment
+
+@deftp {Data Type} oci-image
+Available @code{oci-image} fields are:
+
+@table @asis
+@item @code{repository} (type: string)
+A string like @code{myregistry.local:5000/testing/test-image} that names
+the OCI image.
+
+@item @code{tag} (default: @code{"latest"}) (type: string)
+A string representing the OCI image tag. Defaults to @code{latest}.
+
+@item @code{value} (type: oci-lowerable-image)
+A @code{manifest} or @code{operating-system} record that will be lowered
+into an OCI compatible tarball. Otherwise this field's value can be a
+gexp or a file-like object that evaluates to an OCI compatible tarball.
+
+@item @code{pack-options} (default: @code{'()}) (type: list)
+An optional set of keyword arguments that will be passed to the
+@code{docker-image} procedure from @code{guix scripts pack}. They can
+be used to replicate @command{guix pack} behavior:
+
+@lisp
+(oci-image
+ (repository "guile")
+ (tag "3")
+ (value
+ (specifications->manifest '("guile")))
+ (pack-options '(#:symlinks (("/bin/guile" -> "bin/guile"))
+ #:max-layers 2)))
+@end lisp
+
+If the @code{value} field is an @code{operating-system} record, this field's
+value will be ignored.
+
+@item @code{system} (default: @code{""}) (type: string)
+Attempt to build for a given system, e.g. "i686-linux"
+
+@item @code{target} (default: @code{""}) (type: string)
+Attempt to cross-build for a given triple, e.g. "aarch64-linux-gnu"
+
+@item @code{grafts?} (default: @code{#f}) (type: boolean)
+Whether to allow grafting or not in the pack build.
+
+@end table
+
+@end deftp
+
+
@c %end of fragment
@cindex Audit
diff --git a/gnu/services/docker.scm b/gnu/services/docker.scm
index a5b1614fa9..7aff8dcc5f 100644
--- a/gnu/services/docker.scm
+++ b/gnu/services/docker.scm
@@ -23,11 +23,14 @@
;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
(define-module (gnu services docker)
+ #:use-module (gnu image)
#:use-module (gnu services)
#:use-module (gnu services configuration)
#:use-module (gnu services base)
#:use-module (gnu services dbus)
#:use-module (gnu services shepherd)
+ #:use-module (gnu system)
+ #:use-module (gnu system image)
#:use-module (gnu system setuid)
#:use-module (gnu system shadow)
#:use-module (gnu packages admin) ;shadow
@@ -37,7 +40,11 @@ (define-module (gnu services docker)
#:use-module (guix diagnostics)
#:use-module (guix gexp)
#:use-module (guix i18n)
+ #:use-module (guix monads)
#:use-module (guix packages)
+ #:use-module (guix profiles)
+ #:use-module ((guix scripts pack) #:prefix pack:)
+ #:use-module (guix store)
#:use-module (srfi srfi-1)
#:use-module (ice-9 format)
#:use-module (ice-9 match)
@@ -45,6 +52,16 @@ (define-module (gnu services docker)
#:export (docker-configuration
docker-service-type
singularity-service-type
+ oci-image
+ oci-image?
+ oci-image-fields
+ oci-image-repository
+ oci-image-tag
+ oci-image-value
+ oci-image-pack-options
+ oci-image-target
+ oci-image-system
+ oci-image-grafts?
oci-container-configuration
oci-container-configuration?
oci-container-configuration-fields
@@ -52,9 +69,11 @@ (define-module (gnu services docker)
oci-container-configuration-group
oci-container-configuration-command
oci-container-configuration-entrypoint
+ oci-container-configuration-host-environment
oci-container-configuration-environment
oci-container-configuration-image
oci-container-configuration-provision
+ oci-container-configuration-requirement
oci-container-configuration-network
oci-container-configuration-ports
oci-container-configuration-volumes
@@ -62,7 +81,8 @@ (define-module (gnu services docker)
oci-container-configuration-workdir
oci-container-configuration-extra-arguments
oci-container-service-type
- oci-container-shepherd-service))
+ oci-container-shepherd-service
+ %oci-container-accounts))
(define-maybe file-like)
@@ -320,11 +340,68 @@ (define (oci-sanitize-extra-arguments value)
but ~a was found") el))))
value))
+(define (oci-image-reference image)
+ (if (string? image)
+ image
+ (string-append (oci-image-repository image)
+ ":" (oci-image-tag image))))
+
+(define (oci-lowerable-image? image)
+ (or (manifest? image)
+ (operating-system? image)
+ (gexp? image)
+ (file-like? image)))
+
+(define (string-or-oci-image? image)
+ (or (string? image)
+ (oci-image? image)))
+
(define list-of-symbols?
(list-of symbol?))
(define-maybe/no-serialization string)
+(define-configuration/no-serialization oci-image
+ (repository
+ (string)
+ "A string like @code{myregistry.local:5000/testing/test-image} that names
+the OCI image.")
+ (tag
+ (string "latest")
+ "A string representing the OCI image tag. Defaults to @code{latest}.")
+ (value
+ (oci-lowerable-image)
+ "A @code{manifest} or @code{operating-system} record that will be lowered
+into an OCI compatible tarball. Otherwise this field's value can be a gexp
+or a file-like object that evaluates to an OCI compatible tarball.")
+ (pack-options
+ (list '())
+ "An optional set of keyword arguments that will be passed to the
+@code{docker-image} procedure from @code{guix scripts pack}. They can be used
+to replicate @command{guix pack} behavior:
+
+@lisp
+(oci-image
+ (repository \"guile\")
+ (tag \"3\")
+ (manifest (specifications->manifest '(\"guile\")))
+ (pack-options
+ '(#:symlinks ((\"/bin/guile\" -> \"bin/guile\"))
+ #:max-layers 2)))
+@end lisp
+
+If the @code{value} field is an @code{operating-system} record, this field's
+value will be ignored.")
+ (system
+ (maybe-string)
+ "Attempt to build for a given system, e.g. \"i686-linux\"")
+ (target
+ (maybe-string)
+ "Attempt to cross-build for a given triple, e.g. \"aarch64-linux-gnu\"")
+ (grafts?
+ (boolean #f)
+ "Whether to allow grafting or not in the pack build."))
+
(define-configuration/no-serialization oci-container-configuration
(user
(string "oci-container")
@@ -372,8 +449,9 @@ (define-configuration/no-serialization oci-container-configuration
documentation for semantics."
(sanitizer oci-sanitize-environment))
(image
- (string)
- "The image used to build the container. Images are resolved by the Docker
+ (string-or-oci-image)
+ "The image used to build the container. It can be a string or an
+@code{oci-image} record. Strings are resolved by the Docker
Engine, and follow the usual format
@code{myregistry.local:5000/testing/test-image:tag}.")
(provision
@@ -470,14 +548,122 @@ (define oci-container-configuration->options
(list "-v" spec))
(oci-container-configuration-volumes config))))))))
+(define* (get-keyword-value args keyword #:key (default #f))
+ (let ((kv (memq keyword args)))
+ (if (and kv (>= (length kv) 2))
+ (cadr kv)
+ default)))
+
+(define (lower-operating-system os target system)
+ (mlet* %store-monad
+ ((tarball
+ (lower-object
+ (system-image (os->image os #:type docker-image-type))
+ system
+ #:target target)))
+ (return tarball)))
+
+(define (lower-manifest name image target system)
+ (define value (oci-image-value image))
+ (define options (oci-image-pack-options image))
+ (define image-reference
+ (oci-image-reference image))
+ (define image-tag
+ (let* ((extra-options
+ (get-keyword-value options #:extra-options))
+ (image-tag-option
+ (and extra-options
+ (get-keyword-value extra-options #:image-tag))))
+ (if image-tag-option
+ '()
+ `(#:extra-options (#:image-tag ,image-reference)))))
+
+ (mlet* %store-monad
+ ((_ (set-grafting
+ (oci-image-grafts? image)))
+ (guile (set-guile-for-build (default-guile)))
+ (profile
+ (profile-derivation value
+ #:target target
+ #:system system
+ #:hooks '()
+ #:locales? #f))
+ (tarball (apply pack:docker-image
+ `(,name ,profile
+ ,@options
+ ,@image-tag
+ #:localstatedir? #t))))
+ (return tarball)))
+
+(define (lower-oci-image name image)
+ (define value (oci-image-value image))
+ (define image-target (oci-image-target image))
+ (define image-system (oci-image-system image))
+ (define target
+ (if (maybe-value-set? image-target)
+ image-target
+ (%current-target-system)))
+ (define system
+ (if (maybe-value-set? image-system)
+ image-system
+ (%current-system)))
+ (with-store store
+ (run-with-store store
+ (match value
+ ((? manifest? value)
+ (lower-manifest name image target system))
+ ((? operating-system? value)
+ (lower-operating-system value target system))
+ ((or (? gexp? value)
+ (? file-like? value))
+ value)
+ (_
+ (raise
+ (formatted-message
+ (G_ "oci-image value must contain only manifest,
+operating-system, gexp or file-like records but ~a was found")
+ value))))
+ #:target target
+ #:system system)))
+
+(define (%oci-image-loader name image tag)
+ (let ((docker (file-append docker-cli "/bin/docker"))
+ (tarball (lower-oci-image name image)))
+ (with-imported-modules '((guix build utils))
+ (program-file (format #f "~a-image-loader" name)
+ #~(begin
+ (use-modules (guix build utils)
+ (ice-9 popen)
+ (ice-9 rdelim))
+
+ (format #t "Loading image for ~a from ~a...~%" #$name #$tarball)
+ (define line
+ (read-line
+ (open-input-pipe
+ (string-append #$docker " load -i " #$tarball))))
+
+ (unless (or (eof-object? line)
+ (string-null? line))
+ (format #t "~a~%" line)
+ (let ((repository&tag
+ (string-drop line
+ (string-length
+ "Loaded image: "))))
+
+ (invoke #$docker "tag" repository&tag #$tag)
+ (format #t "Tagged ~a with ~a...~%" #$tarball #$tag))))))))
+
(define (oci-container-shepherd-service config)
(define (guess-name name image)
(if (maybe-value-set? name)
name
(string-append "docker-"
- (basename (car (string-split image #\:))))))
+ (basename
+ (if (string? image)
+ (first (string-split image #\:))
+ (oci-image-repository image))))))
- (let* ((docker-command (file-append docker-cli "/bin/docker"))
+ (let* ((docker (file-append docker-cli "/bin/docker"))
(user (oci-container-configuration-user config))
(group (oci-container-configuration-group config))
(host-environment
@@ -486,6 +672,7 @@ (define (oci-container-shepherd-service config)
(provision (oci-container-configuration-provision config))
(requirement (oci-container-configuration-requirement config))
(image (oci-container-configuration-image config))
+ (image-reference (oci-image-reference image))
(options (oci-container-configuration->options config))
(name (guess-name provision image))
(extra-arguments
@@ -496,30 +683,37 @@ (define (oci-container-shepherd-service config)
(respawn? #f)
(documentation
(string-append
- "Docker backed Shepherd service for image: " image))
+ "Docker backed Shepherd service for "
+ (if (oci-image? image) name image) "."))
(start
- #~(make-forkexec-constructor
- ;; docker run [OPTIONS] IMAGE [COMMAND] [ARG...]
- (list #$docker-command "run" "--rm"
- "--name" #$name
- #$@options #$@extra-arguments #$image #$@command)
- #:user #$user
- #:group #$group
- #:environment-variables
- (list #$@host-environment)))
+ #~(lambda ()
+ (when #$(oci-image? image)
+ (invoke #$(%oci-image-loader
+ name image image-reference)))
+ (fork+exec-command
+ ;; docker run [OPTIONS] IMAGE [COMMAND] [ARG...]
+ (list #$docker "run" "--rm" "--name" #$name
+ #$@options #$@extra-arguments
+ #$image-reference #$@command)
+ #:user #$user
+ #:group #$group
+ #:environment-variables
+ (list #$@host-environment))))
(stop
#~(lambda _
- (invoke #$docker-command "rm" "-f" #$name)))
+ (invoke #$docker "rm" "-f" #$name)))
(actions
- (list
- (shepherd-action
- (name 'pull)
- (documentation
- (format #f "Pull ~a's image (~a)."
- name image))
- (procedure
- #~(lambda _
- (invoke #$docker-command "pull" #$image)))))))))
+ (if (oci-image? image)
+ '()
+ (list
+ (shepherd-action
+ (name 'pull)
+ (documentation
+ (format #f "Pull ~a's image (~a)."
+ name image))
+ (procedure
+ #~(lambda _
+ (invoke #$docker "pull" #$image))))))))))
(define %oci-container-accounts
(list (user-account
--
2.41.0
G
G
Giacomo Leidi wrote on 4 May 00:11 +0200
[PATCH v3 3/5] gnu: docker: Allow setting Shepherd dependencies in oci-container-configuration.
(address . 67613@debbugs.gnu.org)(name . Giacomo Leidi)(address . goodoldpaul@autistici.org)
9506edfb6edd9d3f2a8b832caf39005883a04ac2.1714774277.git.goodoldpaul@autistici.org
* gnu/services/docker.scm (oci-container-configuration)
[requirement]: New field;
(list-of-symbols): sanitize it;
(oci-container-shepherd-service): use it.
* doc/guix.texi: Document it.

Change-Id: Ic0ba336a2257d6ef7c658cfc6cd630116661f581
---
doc/guix.texi | 4 ++++
gnu/services/docker.scm | 10 +++++++++-
2 files changed, 13 insertions(+), 1 deletion(-)

Toggle diff (59 lines)
diff --git a/doc/guix.texi b/doc/guix.texi
index fad0bf8c7c..2b9cc5602c 100644
--- a/doc/guix.texi
+++ b/doc/guix.texi
@@ -40474,6 +40474,10 @@ Miscellaneous Services
@item @code{provision} (default: @code{""}) (type: string)
Set the name of the provisioned Shepherd service.
+@item @code{requirement} (default: @code{'()}) (type: list-of-symbols)
+Set additional Shepherd services dependencies to the provisioned
+Shepherd service.
+
@item @code{network} (default: @code{""}) (type: string)
Set a Docker network for the spawned container.
diff --git a/gnu/services/docker.scm b/gnu/services/docker.scm
index df5884aca1..a5b1614fa9 100644
--- a/gnu/services/docker.scm
+++ b/gnu/services/docker.scm
@@ -320,6 +320,9 @@ (define (oci-sanitize-extra-arguments value)
but ~a was found") el))))
value))
+(define list-of-symbols?
+ (list-of symbol?))
+
(define-maybe/no-serialization string)
(define-configuration/no-serialization oci-container-configuration
@@ -376,6 +379,10 @@ (define-configuration/no-serialization oci-container-configuration
(provision
(maybe-string)
"Set the name of the provisioned Shepherd service.")
+ (requirement
+ (list-of-symbols '())
+ "Set additional Shepherd services dependencies to the provisioned Shepherd
+service.")
(network
(maybe-string)
"Set a Docker network for the spawned container.")
@@ -477,6 +484,7 @@ (define (oci-container-shepherd-service config)
(oci-container-configuration-host-environment config))
(command (oci-container-configuration-command config))
(provision (oci-container-configuration-provision config))
+ (requirement (oci-container-configuration-requirement config))
(image (oci-container-configuration-image config))
(options (oci-container-configuration->options config))
(name (guess-name provision image))
@@ -484,7 +492,7 @@ (define (oci-container-shepherd-service config)
(oci-container-configuration-extra-arguments config)))
(shepherd-service (provision `(,(string->symbol name)))
- (requirement '(dockerd user-processes))
+ (requirement `(dockerd user-processes ,@requirement))
(respawn? #f)
(documentation
(string-append
--
2.41.0
G
G
Giacomo Leidi wrote on 4 May 00:11 +0200
[PATCH v3 5/5] gnu: Add tests for oci-container-service-type.
(address . 67613@debbugs.gnu.org)(name . Giacomo Leidi)(address . goodoldpaul@autistici.org)
24a83c33df8bf8f71df9fea01b0485982799bdc1.1714774277.git.goodoldpaul@autistici.org
* gnu/tests/docker.scm (run-oci-container-test): New variable;
(%test-oci-container): new variable.

Change-Id: Idefc3840bdc6e0ed4264e8f27373cd9a670f87a0
---
gnu/tests/docker.scm | 131 ++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 130 insertions(+), 1 deletion(-)

Toggle diff (162 lines)
diff --git a/gnu/tests/docker.scm b/gnu/tests/docker.scm
index 9e9d2e2d07..d550136b4a 100644
--- a/gnu/tests/docker.scm
+++ b/gnu/tests/docker.scm
@@ -1,6 +1,7 @@
;;; GNU Guix --- Functional package management for GNU
;;; Copyright © 2019 Danny Milosavljevic <dannym@scratchpost.org>
;;; Copyright © 2019-2023 Ludovic Courtès <ludo@gnu.org>
+;;; Copyright © 2024 Giacomo Leidi <goodoldpaul@autistici.org>
;;;
;;; This file is part of GNU Guix.
;;;
@@ -29,6 +30,7 @@ (define-module (gnu tests docker)
#:use-module (gnu services networking)
#:use-module (gnu services docker)
#:use-module (gnu services desktop)
+ #:use-module (gnu packages)
#:use-module ((gnu packages base) #:select (glibc))
#:use-module (gnu packages guile)
#:use-module (gnu packages docker)
@@ -43,7 +45,8 @@ (define-module (gnu tests docker)
#:use-module (guix build-system trivial)
#:use-module ((guix licenses) #:prefix license:)
#:export (%test-docker
- %test-docker-system))
+ %test-docker-system
+ %test-oci-container))
(define %docker-os
(simple-operating-system
@@ -316,3 +319,129 @@ (define %test-docker-system
(locale-libcs (list glibc)))
#:type docker-image-type)))
run-docker-system-test)))))
+
+
+(define %oci-os
+ (simple-operating-system
+ (service dhcp-client-service-type)
+ (service dbus-root-service-type)
+ (service polkit-service-type)
+ (service elogind-service-type)
+ (service docker-service-type)
+ (extra-special-file "/shared.txt"
+ (plain-file "shared.txt" "hello"))
+ (service oci-container-service-type
+ (list
+ (oci-container-configuration
+ (image
+ (oci-image
+ (repository "guile")
+ (value
+ (specifications->manifest '("guile")))
+ (pack-options
+ '(#:symlinks (("/bin" -> "bin"))))))
+ (entrypoint
+ "/bin/guile")
+ (command
+ '("-c" "(let l ((c 300))(display c)(sleep 1)(when(positive? c)(l (- c 1))))"))
+ (host-environment
+ '(("VARIABLE" . "value")))
+ (volumes
+ '(("/shared.txt" . "/shared.txt:ro")))
+ (extra-arguments
+ '("--env" "VARIABLE")))))))
+
+(define (run-oci-container-test)
+ "Run IMAGE as an OCI backed Shepherd service, inside OS."
+
+ (define os
+ (marionette-operating-system
+ (operating-system-with-gc-roots
+ %oci-os
+ (list))
+ #:imported-modules '((gnu services herd)
+ (guix combinators))))
+
+ (define vm
+ (virtual-machine
+ (operating-system os)
+ (volatile? #f)
+ (memory-size 1024)
+ (disk-image-size (* 3000 (expt 2 20)))
+ (port-forwardings '())))
+
+ (define test
+ (with-imported-modules '((gnu build marionette))
+ #~(begin
+ (use-modules (srfi srfi-11) (srfi srfi-64)
+ (gnu build marionette))
+
+ (define marionette
+ ;; Relax timeout to accommodate older systems and
+ ;; allow for pulling the image.
+ (make-marionette (list #$vm) #:timeout 60))
+
+ (test-runner-current (system-test-runner #$output))
+ (test-begin "oci-container")
+
+ (test-assert "dockerd running"
+ (marionette-eval
+ '(begin
+ (use-modules (gnu services herd))
+ (match (start-service 'dockerd)
+ (#f #f)
+ (('service response-parts ...)
+ (match (assq-ref response-parts 'running)
+ ((pid) (number? pid))))))
+ marionette))
+
+ (sleep 10) ; let service start
+
+ (test-assert "docker-guile running"
+ (marionette-eval
+ '(begin
+ (use-modules (gnu services herd))
+ (match (start-service 'docker-guile)
+ (#f #f)
+ (('service response-parts ...)
+ (match (assq-ref response-parts 'running)
+ ((pid) (number? pid))))))
+ marionette))
+
+ (test-equal "passing host environment variables and volumes"
+ '("value" "hello")
+ (marionette-eval
+ `(begin
+ (use-modules (ice-9 popen)
+ (ice-9 rdelim))
+
+ (define slurp
+ (lambda args
+ (let* ((port (apply open-pipe* OPEN_READ args))
+ (output (let ((line (read-line port)))
+ (if (eof-object? line)
+ ""
+ line)))
+ (status (close-pipe port)))
+ output)))
+ (let* ((response1 (slurp
+ ,(string-append #$docker-cli "/bin/docker")
+ "exec" "docker-guile"
+ "/bin/guile" "-c" "(display (getenv \"VARIABLE\"))"))
+ (response2 (slurp
+ ,(string-append #$docker-cli "/bin/docker")
+ "exec" "docker-guile"
+ "/bin/guile" "-c" "(begin (use-modules (ice-9 popen) (ice-9 rdelim))
+(display (call-with-input-file \"/shared.txt\" read-line)))")))
+ (list response1 response2)))
+ marionette))
+
+ (test-end))))
+
+ (gexp->derivation "oci-container-test" test))
+
+(define %test-oci-container
+ (system-test
+ (name "oci-container")
+ (description "Test OCI backed Shepherd service.")
+ (value (run-oci-container-test))))
--
2.41.0
G
G
Giacomo Leidi wrote on 4 May 00:11 +0200
[PATCH v3 2/5] gnu: docker: Allow setting host environment variables in oci-container-configuration.
(address . 67613@debbugs.gnu.org)(name . Giacomo Leidi)(address . goodoldpaul@autistici.org)
0136493778f7159c4cdddf460b71f9d9f5094115.1714774277.git.goodoldpaul@autistici.org
* gnu/services/docker.scm (oci-container-configuration)
[host-environment]: New field;
(oci-sanitize-host-environment): sanitize it;
(oci-container-shepherd-service): use it.
* doc/guix.texi: Document it.

Change-Id: I4d54d37736cf09f042a71cb0b6e673abc0948d9c
---
doc/guix.texi | 17 +++++++++++++++++
gnu/services/docker.scm | 31 +++++++++++++++++++++++++++++--
2 files changed, 46 insertions(+), 2 deletions(-)

Toggle diff (100 lines)
diff --git a/doc/guix.texi b/doc/guix.texi
index 19b7563916..fad0bf8c7c 100644
--- a/doc/guix.texi
+++ b/doc/guix.texi
@@ -40436,6 +40436,23 @@ Miscellaneous Services
@item @code{entrypoint} (default: @code{""}) (type: string)
Overwrite the default entrypoint (@code{ENTRYPOINT}) of the image.
+@item @code{host-environment} (default: @code{'()}) (type: list)
+Set environment variables in the host environment where @command{docker
+run} is invoked. This is especially useful to pass secrets from the
+host to the container without having them on the @command{docker run}'s
+command line: by setting the @code{MYSQL_PASSWORD} on the host and by passing
+@code{--env MYSQL_PASSWORD} through the @code{extra-arguments} field, it is
+possible to securely set values in the container environment. This field's
+value can be a list of pairs or strings, even mixed:
+
+@lisp
+(list '(\"LANGUAGE\" . \"eo:ca:eu\")
+ \"JAVA_HOME=/opt/java\")
+@end lisp
+
+Pair members can be strings, gexps or file-like objects. Strings are passed
+directly to @code{make-forkexec-constructor}.
+
@item @code{environment} (default: @code{'()}) (type: list)
Set environment variables. This can be a list of pairs or strings, even mixed:
diff --git a/gnu/services/docker.scm b/gnu/services/docker.scm
index 824c4ecbe6..df5884aca1 100644
--- a/gnu/services/docker.scm
+++ b/gnu/services/docker.scm
@@ -5,7 +5,7 @@
;;; Copyright © 2020 Efraim Flashner <efraim@flashner.co.il>
;;; Copyright © 2020 Jesse Dowell <jessedowell@gmail.com>
;;; Copyright © 2021 Brice Waegeneire <brice@waegenei.re>
-;;; Copyright © 2023 Giacomo Leidi <goodoldpaul@autistici.org>
+;;; Copyright © 2023, 2024 Giacomo Leidi <goodoldpaul@autistici.org>
;;;
;;; This file is part of GNU Guix.
;;;
@@ -285,6 +285,11 @@ (define (oci-sanitize-mixed-list name value delimiter)
name el)))))
value))
+(define (oci-sanitize-host-environment value)
+ ;; Expected spec format:
+ ;; '(("HOME" . "/home/nobody") "JAVA_HOME=/java")
+ (oci-sanitize-mixed-list "host-environment" value "="))
+
(define (oci-sanitize-environment value)
;; Expected spec format:
;; '(("HOME" . "/home/nobody") "JAVA_HOME=/java")
@@ -330,6 +335,24 @@ (define-configuration/no-serialization oci-container-configuration
(entrypoint
(maybe-string)
"Overwrite the default entrypoint (@code{ENTRYPOINT}) of the image.")
+ (host-environment
+ (list '())
+ "Set environment variables in the host environment where @command{docker run}
+is invoked. This is especially useful to pass secrets from the host to the
+container without having them on the @command{docker run}'s command line: by
+setting the @code{MYSQL_PASSWORD} on the host and by passing
+@code{--env MYSQL_PASSWORD} through the @code{extra-arguments} field, it is
+possible to securely set values in the container environment. This field's
+value can be a list of pairs or strings, even mixed:
+
+@lisp
+(list '(\"LANGUAGE\" . \"eo:ca:eu\")
+ \"JAVA_HOME=/opt/java\")
+@end lisp
+
+Pair members can be strings, gexps or file-like objects. Strings are passed
+directly to @code{make-forkexec-constructor}."
+ (sanitizer oci-sanitize-host-environment))
(environment
(list '())
"Set environment variables inside the container. This can be a list of pairs
@@ -450,6 +473,8 @@ (define (oci-container-shepherd-service config)
(let* ((docker-command (file-append docker-cli "/bin/docker"))
(user (oci-container-configuration-user config))
(group (oci-container-configuration-group config))
+ (host-environment
+ (oci-container-configuration-host-environment config))
(command (oci-container-configuration-command config))
(provision (oci-container-configuration-provision config))
(image (oci-container-configuration-image config))
@@ -471,7 +496,9 @@ (define (oci-container-shepherd-service config)
"--name" #$name
#$@options #$@extra-arguments #$image #$@command)
#:user #$user
- #:group #$group))
+ #:group #$group
+ #:environment-variables
+ (list #$@host-environment)))
(stop
#~(lambda _
(invoke #$docker-command "rm" "-f" #$name)))
--
2.41.0
G
G
Giacomo Leidi wrote on 4 May 00:11 +0200
[PATCH v3 1/5] gnu: docker: Provide escape hatch in oci-container-configuration.
(address . 67613@debbugs.gnu.org)(name . Giacomo Leidi)(address . goodoldpaul@autistici.org)
306393d4c3b28f242fb735d793ef127427f3d072.1714774276.git.goodoldpaul@autistici.org
* gnu/services/docker.scm (exports): Add missing procedures;
(oci-container-service-type)[description]: Docker and OCI images should
mean the same thing;
(oci-container-configuration): clarify field types;
[extra-arguments]: new field;
(oci-sanitize-extra-arguments): sanitize it;
(oci-container-shepherd-service): use it.
* doc/guix.texi: Document it.

Change-Id: I64e9d82c8ae538d59d1c482f23070a880156ddf7
---
doc/guix.texi | 21 ++++++++++++-------
gnu/services/docker.scm | 46 +++++++++++++++++++++++++++++++++--------
2 files changed, 51 insertions(+), 16 deletions(-)

Toggle diff (196 lines)
diff --git a/doc/guix.texi b/doc/guix.texi
index 3f5d4e7f0d..19b7563916 100644
--- a/doc/guix.texi
+++ b/doc/guix.texi
@@ -40430,13 +40430,13 @@ Miscellaneous Services
@item @code{group} (default: @code{"docker"}) (type: string)
The group under whose authority docker commands will be run.
-@item @code{command} (default: @code{()}) (type: list-of-strings)
+@item @code{command} (default: @code{'()}) (type: list-of-strings)
Overwrite the default command (@code{CMD}) of the image.
@item @code{entrypoint} (default: @code{""}) (type: string)
Overwrite the default entrypoint (@code{ENTRYPOINT}) of the image.
-@item @code{environment} (default: @code{()}) (type: list)
+@item @code{environment} (default: @code{'()}) (type: list)
Set environment variables. This can be a list of pairs or strings, even mixed:
@lisp
@@ -40444,7 +40444,8 @@ Miscellaneous Services
"JAVA_HOME=/opt/java")
@end lisp
-String are passed directly to the Docker CLI. You can refer to the
+Pair members can be strings, gexps or file-like objects.
+Strings are passed directly to the Docker CLI. You can refer to the
@uref{https://docs.docker.com/engine/reference/commandline/run/#env,upstream}
documentation for semantics.
@@ -40459,7 +40460,7 @@ Miscellaneous Services
@item @code{network} (default: @code{""}) (type: string)
Set a Docker network for the spawned container.
-@item @code{ports} (default: @code{()}) (type: list)
+@item @code{ports} (default: @code{'()}) (type: list)
Set the port or port ranges to expose from the spawned container. This can be a
list of pairs or strings, even mixed:
@@ -40468,11 +40469,12 @@ Miscellaneous Services
"10443:443")
@end lisp
-String are passed directly to the Docker CLI. You can refer to the
+Pair members can be strings, gexps or file-like objects.
+Strings are passed directly to the Docker CLI. You can refer to the
@uref{https://docs.docker.com/engine/reference/commandline/run/#publish,upstream}
documentation for semantics.
-@item @code{volumes} (default: @code{()}) (type: list)
+@item @code{volumes} (default: @code{'()}) (type: list)
Set volume mappings for the spawned container. This can be a
list of pairs or strings, even mixed:
@@ -40481,7 +40483,8 @@ Miscellaneous Services
"/gnu/store:/gnu/store")
@end lisp
-String are passed directly to the Docker CLI. You can refer to the
+Pair members can be strings, gexps or file-like objects.
+Strings are passed directly to the Docker CLI. You can refer to the
@uref{https://docs.docker.com/engine/reference/commandline/run/#volume,upstream}
documentation for semantics.
@@ -40496,6 +40499,10 @@ Miscellaneous Services
@url{https://docs.docker.com/engine/reference/run/#workdir,upstream}
documentation for semantics.
+@item @code{extra-arguments} (default: @code{'()}) (type: list)
+A list of strings, gexps or file-like objects that will be directly
+passed to the @command{docker run} invokation.
+
@end table
@end deftp
diff --git a/gnu/services/docker.scm b/gnu/services/docker.scm
index 4d32b96847..824c4ecbe6 100644
--- a/gnu/services/docker.scm
+++ b/gnu/services/docker.scm
@@ -58,6 +58,9 @@ (define-module (gnu services docker)
oci-container-configuration-network
oci-container-configuration-ports
oci-container-configuration-volumes
+ oci-container-configuration-container-user
+ oci-container-configuration-workdir
+ oci-container-configuration-extra-arguments
oci-container-service-type
oci-container-shepherd-service))
@@ -297,6 +300,21 @@ (define (oci-sanitize-volumes value)
;; '(("/mnt/dir" . "/dir") "/run/current-system/profile:/java")
(oci-sanitize-mixed-list "volumes" value ":"))
+(define (oci-sanitize-extra-arguments value)
+ (define (valid? member)
+ (or (string? member)
+ (gexp? member)
+ (file-like? member)))
+ (map
+ (lambda (el)
+ (if (valid? el)
+ el
+ (raise
+ (formatted-message
+ (G_ "extra arguments may only be strings, gexps or file-like objects
+but ~a was found") el))))
+ value))
+
(define-maybe/no-serialization string)
(define-configuration/no-serialization oci-container-configuration
@@ -314,15 +332,16 @@ (define-configuration/no-serialization oci-container-configuration
"Overwrite the default entrypoint (@code{ENTRYPOINT}) of the image.")
(environment
(list '())
- "Set environment variables. This can be a list of pairs or strings, even
-mixed:
+ "Set environment variables inside the container. This can be a list of pairs
+or strings, even mixed:
@lisp
(list '(\"LANGUAGE\" . \"eo:ca:eu\")
\"JAVA_HOME=/opt/java\")
@end lisp
-String are passed directly to the Docker CLI. You can refer to the
+Pair members can be strings, gexps or file-like objects. Strings are passed
+directly to the Docker CLI. You can refer to the
@url{https://docs.docker.com/engine/reference/commandline/run/#env,upstream}
documentation for semantics."
(sanitizer oci-sanitize-environment))
@@ -347,7 +366,8 @@ (define-configuration/no-serialization oci-container-configuration
\"10443:443\")
@end lisp
-String are passed directly to the Docker CLI. You can refer to the
+Pair members can be strings, gexps or file-like objects. Strings are passed
+directly to the Docker CLI. You can refer to the
@url{https://docs.docker.com/engine/reference/commandline/run/#publish,upstream}
documentation for semantics."
(sanitizer oci-sanitize-ports))
@@ -361,7 +381,8 @@ (define-configuration/no-serialization oci-container-configuration
\"/gnu/store:/gnu/store\")
@end lisp
-String are passed directly to the Docker CLI. You can refer to the
+Pair members can be strings, gexps or file-like objects. Strings are passed
+directly to the Docker CLI. You can refer to the
@url{https://docs.docker.com/engine/reference/commandline/run/#volume,upstream}
documentation for semantics."
(sanitizer oci-sanitize-volumes))
@@ -375,7 +396,12 @@ (define-configuration/no-serialization oci-container-configuration
"Set the current working for the spawned Shepherd service.
You can refer to the
@url{https://docs.docker.com/engine/reference/run/#workdir,upstream}
-documentation for semantics."))
+documentation for semantics.")
+ (extra-arguments
+ (list '())
+ "A list of strings, gexps or file-like objects that will be directly passed
+to the @command{docker run} invokation."
+ (sanitizer oci-sanitize-extra-arguments)))
(define oci-container-configuration->options
(lambda (config)
@@ -428,7 +454,9 @@ (define (oci-container-shepherd-service config)
(provision (oci-container-configuration-provision config))
(image (oci-container-configuration-image config))
(options (oci-container-configuration->options config))
- (name (guess-name provision image)))
+ (name (guess-name provision image))
+ (extra-arguments
+ (oci-container-configuration-extra-arguments config)))
(shepherd-service (provision `(,(string->symbol name)))
(requirement '(dockerd user-processes))
@@ -441,7 +469,7 @@ (define (oci-container-shepherd-service config)
;; docker run [OPTIONS] IMAGE [COMMAND] [ARG...]
(list #$docker-command "run" "--rm"
"--name" #$name
- #$@options #$image #$@command)
+ #$@options #$@extra-arguments #$image #$@command)
#:user #$user
#:group #$group))
(stop
@@ -482,5 +510,5 @@ (define oci-container-service-type
(extend append)
(compose concatenate)
(description
- "This service allows the management of Docker and OCI
+ "This service allows the management of OCI
containers as Shepherd services.")))

base-commit: 7d4ae2fca723114fb1df56de33b82177fbc4d0a6
--
2.41.0
L
L
Ludovic Courtès wrote on 25 May 15:58 +0200
Re: bug#67613: Introduce unit tests for oci-container-service-type.
(name . Giacomo Leidi)(address . goodoldpaul@autistici.org)(address . 67613-done@debbugs.gnu.org)
87ikz20z9w.fsf_-_@gnu.org
Hi,

I went ahead and finally applied this series, thanks!

(I’m still waiting for the system test to complete on my laptop: it
requires quite a lot of disk space so I had to GC and try again a couple
of times already…)

Thanks,
Ludo’.
Closed
?
Your comment

This issue is archived.

To comment on this conversation send an email to 67613@debbugs.gnu.org

To respond to this issue using the mumi CLI, first switch to it
mumi current 67613
Then, you may apply the latest patchset in this issue (with sign off)
mumi am -- -s
Or, compose a reply to this issue
mumi compose
Or, send patches to this issue
mumi send-email *.patch