(address . guix-patches@gnu.org)(name . Jørgen Kvalsvik)(address . j@lambda.is)
20241115211106.2759121-2-j@lambda.is
Add a new fetcher for go programs, based on go mod download. This is
largely how go programs *want* to be fetched, and some pseudo-private
dependencies [1] are difficult to wrangle without letting go do the
work. This patch only adds support for git as the root module fetch.
The approach is conceptually simple:
1. Fetch the root package to build (usually a git repo)
2. Find all the go.mod files in this tree and run go mod download
3. Store the sources of the package, its direct- and transitive
dependencies as the source.
The source is in source/, the dependencies in go/pkg as encouraged by
the toolchain. Go tooling is generally very opinionated on how things
are supposed to be done, and fighting it usually brings pain.
This approach is not without drawbacks. Go programs tend to be very
liberal with dependencies and specific with versions, which will in
practice leads to a **large** set of sources. At a system level there
will be a lot of duplicated sources, and a lot of slightly different
versions which could be compatible. This is not guix specific but
rather how go programs design their environments.
Another is libraries. Go wants to statically link everything, and
generally likes to work with source, and every program pins dependencies
to different revisions. The go-mod-fetch getting the source of all
libraries the libraries breaks the general packaing approach of
packaging libraries separately, but this was never really leveraged by
go anyway. This switches the focus to packaging applications rather
than libraries.
Finally, the package.source needs two hashes - one for the direct origin
(e.g. git), and one for the sum of libraries fetched by go.mod.
[1] github.com/bufbuild/buf 1.46 requires
buf.build/gen/go/bufbuild/bufplugin/protocolbuffers/go
v1.35.1-20241031151143-70f632351282.1 which is only available as a
.zip and has a layout that go mod understands, but is alien to the
go-build-system.
* guix/go-mod-download.scm: New file.
* gnu/local.mk: Register it.
Change-Id: I84c00df07393a9978124667e3e2497aec7009252
---
Makefile.am | 1 +
guix/go-mod-download.scm | 146 +++++++++++++++++++++++++++++++++++++++
2 files changed, 147 insertions(+)
create mode 100644 guix/go-mod-download.scm
Toggle diff (166 lines)
diff --git a/Makefile.am b/Makefile.am
index 3a35b7becd..fc00947f4f 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -102,6 +102,7 @@ MODULES = \
guix/android-repo-download.scm \
guix/bzr-download.scm \
guix/git-download.scm \
+ guix/go-mod-download.scm \
guix/hg-download.scm \
guix/hash.scm \
guix/swh.scm \
diff --git a/guix/go-mod-download.scm b/guix/go-mod-download.scm
new file mode 100644
index 0000000000..1a7ffbb6ac
--- /dev/null
+++ b/guix/go-mod-download.scm
@@ -0,0 +1,146 @@
+;;; GNU Guix --- Functional package management for GNU
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+
+;;; TODOs:
+;;; 1. Support non-git root repositories
+;;; 2. Store/cache individual module downloads
+
+(define-module (guix go-mod-download)
+ #:use-module (guix build utils)
+ #:use-module (guix derivations)
+ #:use-module (guix packages)
+ #:use-module (guix gexp)
+ #:use-module (guix modules)
+ #:use-module (guix monads)
+ #:use-module (guix records)
+ #:use-module (guix store)
+ #:use-module (guix git-download)
+
+ #:export (go-mod-reference
+ go-mod-reference?
+ go-mod-source
+ go-mod-go
+
+ go-mod-fetch))
+
+(define (go-package)
+ "Return the default Go package."
+ (let ((distro (resolve-interface '(gnu packages golang))))
+ (module-ref distro 'go)))
+
+(define (nss-certs-package)
+ "Return the default nss-certs package."
+ (let ((distro (resolve-interface '(gnu packages certs))))
+ (module-ref distro 'nss-certs)))
+
+;; The source key accepts the same kinds as the package record, but mostly go
+;; use git. The go key specifies which go version to use, which might be
+;; necessary if any module sets a newer toolchain in go.mod
+(define-record-type* <go-mod-reference>
+ go-mod-reference make-go-mod-reference
+ go-mod-reference?
+ (source go-mod-source)
+ (go go-mod-go (default (go-package)) (thunked)))
+
+;; Fetch all direct and indirect dependencies of the go modules in the source
+;; tree (usually a git repo) using go mod download.
+(define* (go-mod-fetch source hash-algo hash
+ #:optional name
+ #:key (system (%current-system))
+ (guile (default-guile))
+ (go (go-package))
+ (nss-certs (nss-certs-package)))
+ (define guile-json
+ (module-ref (resolve-interface '(gnu packages guile)) 'guile-json-4))
+
+ (define* (build source go)
+ (with-imported-modules (source-module-closure
+ '((guix build utils)
+ (guix build download)
+ (srfi srfi-34)))
+ (with-extensions (list guile-json)
+ #~(begin
+ (use-modules
+ (guix build download)
+ (guix build utils)
+ (srfi srfi-34))
+ (let* ((cert-dir (string-append #$nss-certs "/etc/ssl/certs"))
+ (src-dir (string-append #$output "/source"))
+ (mod-cache (string-append #$output "/go/pkg"))
+ (xgo (string-append #$go "/bin/go")))
+ ;; go.mod files can specify a minimum required toolchain which could
+ ;; cause go mod download to fetch and install a newer compiler
+ ;; if invoked with an older one.
+ (setenv "GOTOOLCHAIN" (string-append "go" (getenv "go toolchain")))
+ (setenv "SSL_CERT_DIR" cert-dir)
+ (setenv "GOCACHE" "/homeless-shelter")
+ (setenv "GOPATH" "/homeless-shelter")
+ (setenv "GOMODCACHE" mod-cache)
+
+ (mkdir-p src-dir)
+ (mkdir-p mod-cache)
+ (copy-recursively #$source src-dir)
+ (let* ((go-mods (find-files src-dir "go.mod")))
+ ;; go mod will update the go.mod with transitive dependencies if
+ ;; they are not set, and fail with an error if the file is not
+ ;; writable.
+ (for-each make-file-writable go-mods)
+ (with-throw-handler
+ #t
+ (lambda _
+ (for-each
+ (lambda (go-mod)
+ (with-directory-excursion (dirname go-mod)
+ ;; go mod download must be run twice - the first
+ ;; fetches direct dependencies and *records*
+ ;; transitive dependencies, the second run fetches
+ ;; the transitive dependencies.
+ (and
+ (invoke xgo "mod" "download")
+ (invoke xgo "mod" "download"))))
+ go-mods)
+ #t)
+ (lambda (key . args)
+ (display (string-append "Fetching modules failed.\n"
+ "Here are the results of `go env`:\n"))
+ (invoke xgo "env")))))))))
+
+ (let* ((mod-source (go-mod-source source))
+ (mod-ref (origin-uri mod-source))
+ (mod-go (go-mod-go source))
+ (mod-hash (origin-hash mod-source))
+ (mod-hash-value (content-hash-value mod-hash))
+ (mod-hash-algo (content-hash-algorithm mod-hash)))
+
+ (mlet* %store-monad ((guile-for-build (package->derivation guile system))
+ (git-source (git-fetch mod-ref mod-hash-algo
+ mod-hash-value
+ #:system system
+ #:guile guile-for-build)))
+ (gexp->derivation (or name "go-mod-fetch") (build git-source mod-go)
+ #:script-name "go-mod-fetch"
+ #:env-vars
+ `(("go toolchain" . ,(package-version mod-go)))
+ #:leaked-env-vars '("http_proxy" "https_proxy"
+ "LC_ALL" "LC_MESSAGES" "LANG"
+ "COLUMNS")
+ #:system system
+ #:local-build? #t ;don't offload repo cloning
+ #:recursive? #t
+ #:hash-algo hash-algo
+ #:hash hash
+ #:guile-for-build guile-for-build))))
--
2.39.5