Add a Go Module Importer

OpenSubmitted by Katherine Cox-Buday.
Details
5 participants
  • Helio Machado
  • Katherine Cox-Buday
  • dftxbs3e
  • Ludovic Courtès
  • Marius Bakke
Owner
unassigned
Severity
normal
K
K
Katherine Cox-Buday wrote on 23 Oct 2020 16:06
(address . guix-patches@gnu.org)
87sga5kpdp.fsf@gmail.com
From cc92cbcf5ae89891f478f319e955419800bdfcf9 Mon Sep 17 00:00:00 2001From: Katherine Cox-Buday <cox.katherine.e@gmail.com>Date: Thu, 22 Oct 2020 19:40:17 -0500Subject: [PATCH] * guix/import/go.scm: Created Go Importer * guix/scripts/import.scm: Created Go Importer Subcommand * guix/import/go.scm (importers): Added Go Importer Subcommand
--- guix/import/go.scm | 276 +++++++++++++++++++++++++++++++++++++ guix/scripts/import.scm | 2 +- guix/scripts/import/go.scm | 118 ++++++++++++++++ 3 files changed, 395 insertions(+), 1 deletion(-) create mode 100644 guix/import/go.scm create mode 100644 guix/scripts/import/go.scm
Toggle diff (421 lines)diff --git a/guix/import/go.scm b/guix/import/go.scmnew file mode 100644index 0000000000..61009f3565--- /dev/null+++ b/guix/import/go.scm@@ -0,0 +1,276 @@+;;; GNU Guix --- Functional package management for GNU+;;; Copyright © 2020 Katherine Cox-Buday <cox.katherine.e@gmail.com>+;;;+;;; This file is part of GNU Guix.+;;;+;;; GNU Guix is free software; you can redistribute it and/or modify it+;;; under the terms of the GNU General Public License as published by+;;; the Free Software Foundation; either version 3 of the License, or (at+;;; your option) any later version.+;;;+;;; GNU Guix is distributed in the hope that it will be useful, but+;;; WITHOUT ANY WARRANTY; without even the implied warranty of+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+;;; GNU General Public License for more details.+;;;+;;; You should have received a copy of the GNU General Public License+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.++(define-module (guix import go)+ #:use-module (ice-9 match)+ #:use-module (ice-9 rdelim)+ #:use-module (ice-9 receive)+ #:use-module (ice-9 regex)+ #:use-module (srfi srfi-1)+ #:use-module (srfi srfi-9)+ #:use-module (guix json)+ #:use-module ((guix download) #:prefix download:)+ #:use-module (guix import utils)+ #:use-module (guix import json)+ #:use-module (guix packages)+ #:use-module (guix upstream)+ #:use-module (guix utils)+ #:use-module ((guix licenses) #:prefix license:)+ #:use-module (guix base16)+ #:use-module (guix base32)+ #:use-module (guix build download)+ #:use-module (web uri)++ #:export (go-module->guix-package+ go-module-recursive-import+ infer-module-root))++(define (escape-capital-letters s)+ "To avoid ambiguity when serving from case-insensitive file systems, the+$module and $version elements are case-encoded by replacing every uppercase+letter with an exclamation mark followed by the corresponding lower-case+letter."+ (let ((escaped-string (string)))+ (string-for-each-index+ (lambda (i)+ (let ((c (string-ref s i)))+ (set! escaped-string+ (string-concatenate+ (list escaped-string+ (if (char-upper-case? c) "!" "")+ (string (char-downcase c)))))))+ s)+ escaped-string))++(define (fetch-latest-version goproxy-url module-path)+ "Fetches the version number of the latest version for MODULE-PATH from the+given GOPROXY-URL server."+ (assoc-ref+ (json-fetch (format #f "~a/~a/@latest" goproxy-url+ (escape-capital-letters module-path)))+ "Version"))++(define (fetch-go.mod goproxy-url module-path version file)+ "Fetches go.mod from the given GOPROXY-URL server for the given MODULE-PATH+and VERSION."+ (url-fetch (format #f "~a/~a/@v/~a.mod" goproxy-url+ (escape-capital-letters module-path)+ (escape-capital-letters version))+ file+ #:print-build-trace? #f))++(define (parse-go.mod go.mod-path)+ "Parses a go.mod file and returns an alist of module path to version."+ (with-input-from-file go.mod-path+ (lambda ()+ (let ((in-require? #f)+ (requirements (list)))+ (do ((line (read-line) (read-line)))+ ((eof-object? line))+ (set! line (string-trim line))+ ;; The parser is either entering, within, exiting, or after the+ ;; require block. The Go toolchain is trustworthy so edge-cases like+ ;; double-entry, etc. need not complect the parser.+ (cond+ ((string=? line "require (")+ (set! in-require? #t))+ ((and in-require? (string=? line ")"))+ (set! in-require? #f))+ (in-require?+ (let* ((requirement (string-split line #\space))+ ;; Modules should be unquoted+ (module-path (string-delete #\" (car requirement)))+ (version (list-ref requirement 1)))+ (set! requirements (acons module-path version requirements))))+ ((string-prefix? "replace" line)+ (let* ((requirement (string-split line #\space))+ (module-path (list-ref requirement 1))+ (new-module-path (list-ref requirement 3))+ (version (list-ref requirement 4)))+ (set! requirements (assoc-remove! requirements module-path))+ (set! requirements (acons new-module-path version requirements))))))+ requirements))))++(define (module-path-without-major-version module-path)+ "Go modules can be appended with a major version indicator,+e.g. /v3. Sometimes it is desirable to work with the root module path. For+instance, for a module path github.com/foo/bar/v3 this function returns+github.com/foo/bar."+ (let ((m (string-match "(.*)\\/v[0-9]+$" module-path)))+ (if m+ (match:substring m 1)+ module-path)))++(define (infer-module-root module-path)+ "Go modules can be defined at any level of a repository's tree, but querying+for the meta tag usually can only be done at the webpage at the root of the+repository. Therefore, it is sometimes necessary to try and derive a module's+root path from its path. For a set of well-known forges, the pattern of what+consists of a module's root page is known before hand."+ ;; See the following URL for the official Go equivalent:+ ;; https://github.com/golang/go/blob/846dce9d05f19a1f53465e62a304dea21b99f910/src/cmd/go/internal/vcs/vcs.go#L1026-L1087+ (define-record-type <scs>+ (make-scs url-prefix root-regex type)+ scs?+ (url-prefix scs-url-prefix)+ (root-regex scs-root-regex)+ (type scs-type))+ (let* ((known-scs+ (list+ (make-scs+ "github.com"+ "^(github\\.com/[A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+)(/[A-Za-z0-9_.\\-]+)*$"+ 'git)+ (make-scs+ "bitbucket.org"+ "^(bitbucket\\.org/([A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+))(/[A-Za-z0-9_.\\-]+)*$`"+ 'unknown)+ (make-scs+ "hub.jazz.net/git/"+ "^(hub\\.jazz\\.net/git/[a-z0-9]+/[A-Za-z0-9_.\\-]+)(/[A-Za-z0-9_.\\-]+)*$"+ 'git)+ (make-scs+ "git.apache.org"+ "^(git\\.apache\\.org/[a-z0-9_.\\-]+\\.git)(/[A-Za-z0-9_.\\-]+)*$"+ 'git)+ (make-scs+ "git.openstack.org"+ "^(git\\.openstack\\.org/[A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+)(\\.git)?(/[A-Za-z0-9_.\\-]+)*$"+ 'git)))+ (scs (find (lambda (scs) (string-prefix? (scs-url-prefix scs) module-path))+ known-scs)))+ (if scs+ (match:substring (string-match (scs-root-regex scs) module-path) 1)+ module-path)))++(define (to-guix-package-name module-path)+ "Converts a module's path to the canonical Guix format for Go packages."+ (string-downcase+ (string-append "go-"+ (string-replace-substring+ (string-replace-substring+ ;; Guix has its own field for version+ (module-path-without-major-version module-path)+ "." "-")+ "/" "-"))))++(define (fetch-module-meta-data module-path)+ "Fetches module meta-data from a module's landing page. This is necessary+because goproxy servers don't currently provide all the information needed to+build a package."+ (let* ((port (http-fetch (string->uri (format #f "https://~a?go-get=1" module-path))))+ (module-metadata #f)+ (meta-tag-prefix "<meta name=\"go-import\" content=\"")+ (meta-tag-prefix-length (string-length meta-tag-prefix)))+ (do ((line (read-line port) (read-line port)))+ ((or (eof-object? line)+ module-metadata))+ (let ((meta-tag-index (string-contains line meta-tag-prefix)))+ (when meta-tag-index+ (let* ((start (+ meta-tag-index meta-tag-prefix-length))+ (end (string-index line #\" start)))+ (set! module-metadata+ (string-split (substring/shared line start end) #\space))))))+ (close-port port)+ module-metadata))++(define (module-meta-data-scs meta-data)+ "Return the source control system specified by a module's meta-data."+ (string->symbol (list-ref meta-data 1)))++(define (module-meta-data-repo-url meta-data goproxy-url)+ "Return the URL where the fetcher which will be used can download the source+control."+ (if (member (module-meta-data-scs meta-data) '(fossil mod))+ goproxy-url+ (list-ref meta-data 2)))++(define (source-uri scs-type scs-repo-url file)+ "Generate the `origin' block of a package depending on what type of source+control system is being used."+ (case scs-type+ ((git)+ `(origin+ (method git-fetch)+ (uri (git-reference+ (url ,scs-repo-url)+ (commit (string-append "v" version))))+ (file-name (git-file-name name version))+ (sha256+ (base32+ ,(guix-hash-url file)))))+ ((hg)+ `(origin+ (method hg-fetch)+ (uri (hg-reference+ (url ,scs-repo-url)+ (changeset ,version)))+ (file-name (format #f "~a-~a-checkout" name version))))+ ((svn)+ `(origin+ (method svn-fetch)+ (uri (svn-reference+ (url ,scs-repo-url)+ (revision (string->number version))+ (recursive? #f)))+ (file-name (format #f "~a-~a-checkout" name version))+ (sha256+ (base32+ ,(guix-hash-url file)))))+ (else+ (raise-exception (format #f "unsupported scs type: ~a" scs-type)))))++(define* (go-module->guix-package module-path #:key (goproxy-url "https://proxy.golang.org"))+ (call-with-temporary-output-file+ (lambda (temp port)+ (let* ((latest-version (fetch-latest-version goproxy-url module-path))+ (go.mod-path (fetch-go.mod goproxy-url module-path latest-version+ temp))+ (dependencies (map car (parse-go.mod temp)))+ (guix-name (to-guix-package-name module-path))+ (root-module-path (infer-module-root module-path))+ ;; SCS type and URL are not included in goproxy information. For+ ;; this we need to fetch it from the official module page.+ (meta-data (fetch-module-meta-data root-module-path))+ (scs-type (module-meta-data-scs meta-data))+ (scs-repo-url (module-meta-data-repo-url meta-data goproxy-url)))+ (values+ `(package+ (name ,guix-name)+ ;; Elide the "v" prefix Go uses+ (version ,(string-trim latest-version #\v))+ (source+ ,(source-uri scs-type scs-repo-url temp))+ (build-system go-build-system)+ ,@(maybe-inputs (map to-guix-package-name dependencies))+ ;; TODO(katco): It would be nice to make an effort to fetch this+ ;; from known forges, e.g. GitHub+ (home-page ,(format #f "https://~a" root-module-path))+ (synopsis "A Go package")+ (description ,(format #f "~a is a Go package." guix-name))+ (license #f))+ dependencies)))))++(define* (go-module-recursive-import package-name+ #:key (goproxy-url "https://proxy.golang.org"))+ (recursive-import package-name #f+ #:repo->guix-package+ (lambda (name _)+ (go-module->guix-package name+ #:goproxy-url goproxy-url))+ #:guix-name to-guix-package-name))diff --git a/guix/scripts/import.scm b/guix/scripts/import.scmindex 0a3863f965..1d2b45d942 100644--- a/guix/scripts/import.scm+++ b/guix/scripts/import.scm@@ -77,7 +77,7 @@ rather than \\n." ;;; (define importers '("gnu" "nix" "pypi" "cpan" "hackage" "stackage" "elpa" "gem"- "cran" "crate" "texlive" "json" "opam"))+ "go" "cran" "crate" "texlive" "json" "opam")) (define (resolve-importer name) (let ((module (resolve-interfacediff --git a/guix/scripts/import/go.scm b/guix/scripts/import/go.scmnew file mode 100644index 0000000000..000039769c--- /dev/null+++ b/guix/scripts/import/go.scm@@ -0,0 +1,118 @@+;;; GNU Guix --- Functional package management for GNU+;;; Copyright © 2020 Katherine Cox-Buday <cox.katherine.e@gmail.com>+;;;+;;; This file is part of GNU Guix.+;;;+;;; GNU Guix is free software; you can redistribute it and/or modify it+;;; under the terms of the GNU General Public License as published by+;;; the Free Software Foundation; either version 3 of the License, or (at+;;; your option) any later version.+;;;+;;; GNU Guix is distributed in the hope that it will be useful, but+;;; WITHOUT ANY WARRANTY; without even the implied warranty of+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+;;; GNU General Public License for more details.+;;;+;;; You should have received a copy of the GNU General Public License+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.++(define-module (guix scripts import go)+ #:use-module (guix ui)+ #:use-module (guix utils)+ #:use-module (guix scripts)+ #:use-module (guix import go)+ #:use-module (guix scripts import)+ #:use-module (srfi srfi-1)+ #:use-module (srfi srfi-11)+ #:use-module (srfi srfi-37)+ #:use-module (ice-9 match)+ #:use-module (ice-9 format)+ #:export (guix-import-go))++ +;;;+;;; Command-line options.+;;;++(define %default-options+ '())++(define (show-help)+ (display (G_ "Usage: guix import go PACKAGE-PATH+Import and convert the Go module for PACKAGE-PATH.\n"))+ (display (G_ "+ -h, --help display this help and exit"))+ (display (G_ "+ -V, --version display version information and exit"))+ (display (G_ "+ -r, --recursive generate package expressions for all Go modules\+ that are not yet in Guix"))+ (display (G_ "+ -p, --goproxy=GOPROXY specify which goproxy server to use"))+ (newline)+ (show-bug-report-information))++(define %options+ ;; Specification of the command-line options.+ (cons* (option '(#\h "help") #f #f+ (lambda args+ (show-help)+ (exit 0)))+ (option '(#\V "version") #f #f+ (lambda args+ (show-version-and-exit "guix import go")))+ (option '(#\r "recursive") #f #f+ (lambda (opt name arg result)+ (alist-cons 'recursive #t result)))+ (option '(#\p "goproxy") #t #f+ (lambda (opt name arg result)+ (alist-cons 'goproxy+ (string->symbol arg)+ (alist-delete 'goproxy result))))+ %standard-import-options))++ +;;;+;;; Entry point.+;;;++(define (guix-import-go . args)+ (define (parse-options)+ ;; Return the alist of option values.+ (args-fold* args %options+ (lambda (opt name arg result)+ (leave (G_ "~A: unrecognized option~%") name))+ (lambda (arg result)+ (alist-cons 'argument arg result))+ %default-options))++ (let* ((opts (parse-options))+ (args (filter-map (match-lambda+ (('argument . value)+ value)+ (_ #f))+ (reverse opts))))+ (match args+ ((module-name)+ (if (assoc-ref opts 'recursive)+ (map (match-lambda+ ((and ('package ('name name) . rest) pkg)+ `(define-public ,(string->symbol name)+ ,pkg))+ (_ #f))+ (go-module-recursive-import module-name+ #:goproxy-url+ (or (assoc-ref opts 'goproxy)+ "https://proxy.golang.org")))+ (let ((sexp (go-module->guix-package module-name+ #:goproxy-url+ (or (assoc-ref opts 'goproxy)+ "https://proxy.golang.org"))))+ (unless sexp+ (leave (G_ "failed to download meta-data for module '~a'~%")+ module-name))+ sexp)))+ (()+ (leave (G_ "too few arguments~%")))+ ((many ...)+ (leave (G_ "too many arguments~%"))))))-- 2.28.0
-- Katherine
L
L
Ludovic Courtès wrote on 28 Oct 2020 11:41
(name . Katherine Cox-Buday)(address . cox.katherine.e@gmail.com)(address . 44178@debbugs.gnu.org)
87a6w64opi.fsf@gnu.org
Hi Katherine,
Katherine Cox-Buday <cox.katherine.e@gmail.com> skribis:
Toggle quote (7 lines)>>From cc92cbcf5ae89891f478f319e955419800bdfcf9 Mon Sep 17 00:00:00 2001> From: Katherine Cox-Buday <cox.katherine.e@gmail.com>> Date: Thu, 22 Oct 2020 19:40:17 -0500> Subject: [PATCH] * guix/import/go.scm: Created Go Importer *> guix/scripts/import.scm: Created Go Importer Subcommand * guix/import/go.scm> (importers): Added Go Importer Subcommand
Nice! I think that can make a lot of people happy. :-)
Here’s a quick review. I won’t promise I can reply to followups in thecoming days because with the release preparation going on, I’d ratherfocus on that. So perhaps this patch will have to wait until after thisrelease, but certainly before the next one!
Toggle quote (17 lines)> +(define (escape-capital-letters s)> + "To avoid ambiguity when serving from case-insensitive file systems, the> +$module and $version elements are case-encoded by replacing every uppercase> +letter with an exclamation mark followed by the corresponding lower-case> +letter."> + (let ((escaped-string (string)))> + (string-for-each-index> + (lambda (i)> + (let ((c (string-ref s i)))> + (set! escaped-string> + (string-concatenate> + (list escaped-string> + (if (char-upper-case? c) "!" "")> + (string (char-downcase c)))))))> + s)> + escaped-string))
As a general comment, the coding style in Guix is functional “bydefault” (info "(guix) Coding Style"). That means we almost never use‘set!’ and procedures that modify their arguments.
We also avoid idioms like car/cdr and ‘do’, which are more commonly usedin other Lisps, as you know very well. ;-)
In the case above, I’d probably use ‘string-fold’. The resulting codeshould be easier to reason about and likely more efficient.
Toggle quote (8 lines)> +(define (fetch-latest-version goproxy-url module-path)> + "Fetches the version number of the latest version for MODULE-PATH from the> +given GOPROXY-URL server."> + (assoc-ref> + (json-fetch (format #f "~a/~a/@latest" goproxy-url> + (escape-capital-letters module-path)))> + "Version"))
I’d suggest using ‘define-json-mapping’ from (json) like in the otherimporters.
Toggle quote (15 lines)> +(define (infer-module-root module-path)> + "Go modules can be defined at any level of a repository's tree, but querying> +for the meta tag usually can only be done at the webpage at the root of the> +repository. Therefore, it is sometimes necessary to try and derive a module's> +root path from its path. For a set of well-known forges, the pattern of what> +consists of a module's root page is known before hand."> + ;; See the following URL for the official Go equivalent:> + ;; https://github.com/golang/go/blob/846dce9d05f19a1f53465e62a304dea21b99f910/src/cmd/go/internal/vcs/vcs.go#L1026-L1087> + (define-record-type <scs>> + (make-scs url-prefix root-regex type)> + scs?> + (url-prefix scs-url-prefix)> + (root-regex scs-root-regex)> + (type scs-type))
Maybe VCS as “version control system”? (It took me a while to guesswhat “SCS” meant.)
Toggle quote (18 lines)> +(define (fetch-module-meta-data module-path)> + "Fetches module meta-data from a module's landing page. This is necessary> +because goproxy servers don't currently provide all the information needed to> +build a package."> + (let* ((port (http-fetch (string->uri (format #f "https://~a?go-get=1" module-path))))> + (module-metadata #f)> + (meta-tag-prefix "<meta name=\"go-import\" content=\"")> + (meta-tag-prefix-length (string-length meta-tag-prefix)))> + (do ((line (read-line port) (read-line port)))> + ((or (eof-object? line)> + module-metadata))> + (let ((meta-tag-index (string-contains line meta-tag-prefix)))> + (when meta-tag-index> + (let* ((start (+ meta-tag-index meta-tag-prefix-length))> + (end (string-index line #\" start)))> + (set! module-metadata> + (string-split (substring/shared line start end) #\space))))))
I’d suggest a named ‘let’ or ‘fold’ here.
Likewise, instead of concatenating XML strings (which could lead tomalformed XML), I recommend using SXML: you would create an sexp like
(meta (@ (name "go-import") (content …)))
and at the end pass it to ‘sxml->sxml’ (info "(guile) Reading andWriting XML").
Toggle quote (3 lines)> + (else> + (raise-exception (format #f "unsupported scs type: ~a" scs-type)))))
‘raise-exception’ takes an error condition. In this case, we should use(srfi srfi-34) for ‘raise’ write something like:
(raise (condition (formatted-message (G_ "…" …))))
Toggle quote (7 lines)> +(define* (go-module->guix-package module-path #:key (goproxy-url "https://proxy.golang.org"))> + (call-with-temporary-output-file> + (lambda (temp port)> + (let* ((latest-version (fetch-latest-version goproxy-url module-path))> + (go.mod-path (fetch-go.mod goproxy-url module-path latest-version> + temp))
It seems that ‘go.mod-path’ isn’t used, and thus ‘fetch-go.mod’ &co. aren’t used either, or am I overlooking something?
Toggle quote (2 lines)> + (dependencies (map car (parse-go.mod temp)))
Please use ‘match’ instead, or perhaps define a record type for theabstraction at hand.
Toggle quote (22 lines)> + (guix-name (to-guix-package-name module-path))> + (root-module-path (infer-module-root module-path))> + ;; SCS type and URL are not included in goproxy information. For> + ;; this we need to fetch it from the official module page.> + (meta-data (fetch-module-meta-data root-module-path))> + (scs-type (module-meta-data-scs meta-data))> + (scs-repo-url (module-meta-data-repo-url meta-data goproxy-url)))> + (values> + `(package> + (name ,guix-name)> + ;; Elide the "v" prefix Go uses> + (version ,(string-trim latest-version #\v))> + (source> + ,(source-uri scs-type scs-repo-url temp))> + (build-system go-build-system)> + ,@(maybe-inputs (map to-guix-package-name dependencies))> + ;; TODO(katco): It would be nice to make an effort to fetch this> + ;; from known forges, e.g. GitHub> + (home-page ,(format #f "https://~a" root-module-path))> + (synopsis "A Go package")> + (description ,(format #f "~a is a Go package." guix-name))
Maybe something like “fill it out!” so we don’t get patch submissionswith the default synopsis/description. :-)
Toggle quote (2 lines)> + (license #f))
Likewise.
Two more things: could you (1) and an entry under “Invoking guix import”in doc/guix.texi, and (2) add tests, taking inspiration from theexisting importer tests?
Thank you!
Ludo’.
L
L
Ludovic Courtès wrote on 28 Oct 2020 11:42
(name . Katherine Cox-Buday)(address . cox.katherine.e@gmail.com)(address . 44178@debbugs.gnu.org)
878sbq4oof.fsf@gnu.org
Hi Katherine,
Katherine Cox-Buday <cox.katherine.e@gmail.com> skribis:
Toggle quote (7 lines)>>From cc92cbcf5ae89891f478f319e955419800bdfcf9 Mon Sep 17 00:00:00 2001> From: Katherine Cox-Buday <cox.katherine.e@gmail.com>> Date: Thu, 22 Oct 2020 19:40:17 -0500> Subject: [PATCH] * guix/import/go.scm: Created Go Importer *> guix/scripts/import.scm: Created Go Importer Subcommand * guix/import/go.scm> (importers): Added Go Importer Subcommand
Nice! I think that can make a lot of people happy. :-)
Here’s a quick review. I won’t promise I can reply to followups in thecoming days because with the release preparation going on, I’d ratherfocus on that. So perhaps this patch will have to wait until after thisrelease, but certainly before the next one!
Toggle quote (17 lines)> +(define (escape-capital-letters s)> + "To avoid ambiguity when serving from case-insensitive file systems, the> +$module and $version elements are case-encoded by replacing every uppercase> +letter with an exclamation mark followed by the corresponding lower-case> +letter."> + (let ((escaped-string (string)))> + (string-for-each-index> + (lambda (i)> + (let ((c (string-ref s i)))> + (set! escaped-string> + (string-concatenate> + (list escaped-string> + (if (char-upper-case? c) "!" "")> + (string (char-downcase c)))))))> + s)> + escaped-string))
As a general comment, the coding style in Guix is functional “bydefault” (info "(guix) Coding Style"). That means we almost never use‘set!’ and procedures that modify their arguments.
We also avoid idioms like car/cdr and ‘do’, which are more commonly usedin other Lisps, as you know very well. ;-)
In the case above, I’d probably use ‘string-fold’. The resulting codeshould be easier to reason about and likely more efficient.
Toggle quote (8 lines)> +(define (fetch-latest-version goproxy-url module-path)> + "Fetches the version number of the latest version for MODULE-PATH from the> +given GOPROXY-URL server."> + (assoc-ref> + (json-fetch (format #f "~a/~a/@latest" goproxy-url> + (escape-capital-letters module-path)))> + "Version"))
I’d suggest using ‘define-json-mapping’ from (json) like in the otherimporters.
Toggle quote (15 lines)> +(define (infer-module-root module-path)> + "Go modules can be defined at any level of a repository's tree, but querying> +for the meta tag usually can only be done at the webpage at the root of the> +repository. Therefore, it is sometimes necessary to try and derive a module's> +root path from its path. For a set of well-known forges, the pattern of what> +consists of a module's root page is known before hand."> + ;; See the following URL for the official Go equivalent:> + ;; https://github.com/golang/go/blob/846dce9d05f19a1f53465e62a304dea21b99f910/src/cmd/go/internal/vcs/vcs.go#L1026-L1087> + (define-record-type <scs>> + (make-scs url-prefix root-regex type)> + scs?> + (url-prefix scs-url-prefix)> + (root-regex scs-root-regex)> + (type scs-type))
Maybe VCS as “version control system”? (It took me a while to guesswhat “SCS” meant.)
Toggle quote (18 lines)> +(define (fetch-module-meta-data module-path)> + "Fetches module meta-data from a module's landing page. This is necessary> +because goproxy servers don't currently provide all the information needed to> +build a package."> + (let* ((port (http-fetch (string->uri (format #f "https://~a?go-get=1" module-path))))> + (module-metadata #f)> + (meta-tag-prefix "<meta name=\"go-import\" content=\"")> + (meta-tag-prefix-length (string-length meta-tag-prefix)))> + (do ((line (read-line port) (read-line port)))> + ((or (eof-object? line)> + module-metadata))> + (let ((meta-tag-index (string-contains line meta-tag-prefix)))> + (when meta-tag-index> + (let* ((start (+ meta-tag-index meta-tag-prefix-length))> + (end (string-index line #\" start)))> + (set! module-metadata> + (string-split (substring/shared line start end) #\space))))))
I’d suggest a named ‘let’ or ‘fold’ here.
Likewise, instead of concatenating XML strings (which could lead tomalformed XML), I recommend using SXML: you would create an sexp like
(meta (@ (name "go-import") (content …)))
and at the end pass it to ‘sxml->sxml’ (info "(guile) Reading andWriting XML").
Toggle quote (3 lines)> + (else> + (raise-exception (format #f "unsupported scs type: ~a" scs-type)))))
‘raise-exception’ takes an error condition. In this case, we should use(srfi srfi-34) for ‘raise’ write something like:
(raise (condition (formatted-message (G_ "…" …))))
Toggle quote (7 lines)> +(define* (go-module->guix-package module-path #:key (goproxy-url "https://proxy.golang.org"))> + (call-with-temporary-output-file> + (lambda (temp port)> + (let* ((latest-version (fetch-latest-version goproxy-url module-path))> + (go.mod-path (fetch-go.mod goproxy-url module-path latest-version> + temp))
It seems that ‘go.mod-path’ isn’t used, and thus ‘fetch-go.mod’ &co. aren’t used either, or am I overlooking something?
Toggle quote (2 lines)> + (dependencies (map car (parse-go.mod temp)))
Please use ‘match’ instead, or perhaps define a record type for theabstraction at hand.
Toggle quote (22 lines)> + (guix-name (to-guix-package-name module-path))> + (root-module-path (infer-module-root module-path))> + ;; SCS type and URL are not included in goproxy information. For> + ;; this we need to fetch it from the official module page.> + (meta-data (fetch-module-meta-data root-module-path))> + (scs-type (module-meta-data-scs meta-data))> + (scs-repo-url (module-meta-data-repo-url meta-data goproxy-url)))> + (values> + `(package> + (name ,guix-name)> + ;; Elide the "v" prefix Go uses> + (version ,(string-trim latest-version #\v))> + (source> + ,(source-uri scs-type scs-repo-url temp))> + (build-system go-build-system)> + ,@(maybe-inputs (map to-guix-package-name dependencies))> + ;; TODO(katco): It would be nice to make an effort to fetch this> + ;; from known forges, e.g. GitHub> + (home-page ,(format #f "https://~a" root-module-path))> + (synopsis "A Go package")> + (description ,(format #f "~a is a Go package." guix-name))
Maybe something like “fill it out!” so we don’t get patch submissionswith the default synopsis/description. :-)
Toggle quote (2 lines)> + (license #f))
Likewise.
Two more things: could you (1) and an entry under “Invoking guix import”in doc/guix.texi, and (2) add tests, taking inspiration from theexisting importer tests?
Thank you!
Ludo’.
M
M
Marius Bakke wrote on 10 Nov 2020 21:26
(name . Helio Machado)(address . 0x2b3bfa0@gmail.com)
87blg5dkla.fsf@gnu.org
Katherine Cox-Buday <cox.katherine.e@gmail.com> writes:
Toggle quote (7 lines)>>From cc92cbcf5ae89891f478f319e955419800bdfcf9 Mon Sep 17 00:00:00 2001> From: Katherine Cox-Buday <cox.katherine.e@gmail.com>> Date: Thu, 22 Oct 2020 19:40:17 -0500> Subject: [PATCH] * guix/import/go.scm: Created Go Importer *> guix/scripts/import.scm: Created Go Importer Subcommand * guix/import/go.scm> (importers): Added Go Importer Subcommand
I just want to say thanks a lot for this! I tested it, and it prettymuch works as advertised.
Cc'ing Helio who was working on a Go importer as well recently.
-----BEGIN PGP SIGNATURE-----
iQFDBAEBCgAtFiEEu7At3yzq9qgNHeZDoqBt8qM6VPoFAl+q94EPHG1hcml1c0BnbnUub3JnAAoJEKKgbfKjOlT6smgH/ibghemWudkskluP5si3VgUb9JO49uiUgDv3CaKbs/SOvySUPFgl+8ceLWKgLTQeNVGBgcRBA+aiL0hgAYw/JTBndMHEJ1aZMevwWUDhIMojlbtfw8wBJuVROjPKcXZW5WXZIKQ1GyqpQ/AP72uZcgI7xB0Qbh0ztQq4uhZcEiDvgCGfsFWOphpuqxoB2rUpEruLaUj4kkEhccArt79WbgVLF3lTijM1FhffMk3r+4MCdmCL3pZsnDcoY/qNl4jGxqP2mj+rszQFqR6da8gIEhkM52NI3U9VoWLpgaLq4y4jQp4cdMPY4r2WE5jm7KryGcP/4KHFmf4g/8cvD0Lj5dk==hW6A-----END PGP SIGNATURE-----
H
H
Helio Machado wrote on 11 Nov 2020 02:23
(address . 44178@debbugs.gnu.org)
CANe01w5DxwrF+APLbcNMaUhs4h05EXC1HdRJ_YT8CRy6ULxUYw@mail.gmail.com
Thanks for the ping, Marius! I've been quite busy with some yak shavingtasks, but my importer is already working and has some interestingimprovements, like elegant module fetching from the official module proxy,license extraction and recursive import support.
I need to fix an esoteric bug that trips the kernel's out-of-memory killerwhen building a derivation with dependencies, but the importer part workspretty well.
You can take a look to [my changes][1] for some readily backportable ideas,like [the compact algorithm for uppercase path escaping][2].
Please forgive the code quality and the possible backwards-compatibilitymistakes; this is an unfinished proof of concept.
$ guix import go-modules -r github.com/FiloSottile/age # Please refer tothe issue 43872 for more information about the resting environment
[1]:https://github.com/0x2b3bfa0/guix-go-modules/commit/5defe897065c5d3e63740932b360474132c77877[2]:https://github.com/0x2b3bfa0/guix-go-modules/blob/main/guix/build-system/go.scm#L65-L71
On Wed, 11 Nov 2020 at 02:19, Helio Machado <0x2b3bfa0@gmail.com> wrote:
Toggle quote (38 lines)> Thanks for the ping, Marius! I've been quite busy with some yak shaving> tasks, but my importer is already working and has some interesting> improvements, like elegant module fetching from the official module proxy,> license extraction and recursive import support.>> I need to fix an esoteric bug that trips the kernel's out-of-memory killer> when building a derivation with dependencies, but the importer part works> pretty well.>> You can take a look to [my changes][1] for some readily backportable> ideas, like [the compact algorithm for uppercase path escaping][2].>> Please forgive the code quality and the possible backwards-compatibility> mistakes; this is an unfinished proof of concept.>> [1]:> https://github.com/0x2b3bfa0/guix-go-modules/commit/5defe897065c5d3e63740932b360474132c77877> [2]:> https://github.com/0x2b3bfa0/guix-go-modules/blob/main/guix/build-system/go.scm#L65-L71>> On Tue, 10 Nov 2020 at 21:26, Marius Bakke <marius@gnu.org> wrote:>>> Katherine Cox-Buday <cox.katherine.e@gmail.com> writes:>>>> >>From cc92cbcf5ae89891f478f319e955419800bdfcf9 Mon Sep 17 00:00:00 2001>> > From: Katherine Cox-Buday <cox.katherine.e@gmail.com>>> > Date: Thu, 22 Oct 2020 19:40:17 -0500>> > Subject: [PATCH] * guix/import/go.scm: Created Go Importer *>> > guix/scripts/import.scm: Created Go Importer Subcommand *>> guix/import/go.scm>> > (importers): Added Go Importer Subcommand>>>> I just want to say thanks a lot for this! I tested it, and it pretty>> much works as advertised.>>>> Cc'ing Helio who was working on a Go importer as well recently.>>>
Attachment: file
K
K
Katherine Cox-Buday wrote on 11 Nov 2020 21:48
(name . Marius Bakke)(address . marius@gnu.org)
87v9ebpqlx.fsf@gmail.com
Marius Bakke <marius@gnu.org> writes:
Toggle quote (3 lines)> I just want to say thanks a lot for this! I tested it, and it pretty> much works as advertised.
You're very welcome!
I have more changes locally which fix some edge-cases. I'm using`go-ethereum` as my test case since someone mentioned that. Plus I needto make some of the changes Ludovic pointed out. Still, we're underway!
-- Katherine
D
D
dftxbs3e wrote on 9 Dec 2020 15:22
(address . 44178@debbugs.gnu.org)
9395ba7c7499d6c4423982486a0a0fb31cb76e93.camel@free.fr
Thanks a lot for this!
I'm getting some error trying to use it (patching on top of8e2aad26ae9b7365db83d4f6c74e9e79c57766a6), maybe that's fixed in yourlocal changes?
$ ./pre-inst-env guix import go -r github.com/syncthing/syncthingWARNING: (guix import go): `url-fetch' imported from both (guix importutils) and (guix build download)Backtrace:In ice-9/boot-9.scm: 1736:10 7 (with-exception-handler _ _ #:unwind? _ # _)In unknown file: 6 (apply-smob/0 #<thunk 7ff10807d9a0>)In ice-9/boot-9.scm: 718:2 5 (call-with-prompt _ _ #<procedure default-prompt-handle…>)In ice-9/eval.scm: 619:8 4 (_ #(#(#<directory (guile-user) 7ff107cadf00>)))In guix/ui.scm: 2127:12 3 (run-guix-command _ . _)In guix/scripts/import.scm: 120:11 2 (guix-import . _)In ice-9/eval.scm: 159:9 1 (_ #(#(#(#(#(#(#(#(#(#<directo…>) …) …) …) …) …) …) …) …))In guix/import/utils.scm: 429:0 0 (recursive-import _ #:repo->guix-package _ #:guix-name _…)
guix/import/utils.scm:429:0: In procedure recursive-import:Invalid keyword: #f
D
D
dftxbs3e wrote on 10 Dec 2020 03:42
(address . 44178@debbugs.gnu.org)
bc9fc787f49a9722396ed97554e2a7c634f830a0.camel@free.fr
On Wed, 2020-12-09 at 15:22 +0100, dftxbs3e wrote:
Toggle quote (34 lines)> Thanks a lot for this!> > I'm getting some error trying to use it (patching on top of> 8e2aad26ae9b7365db83d4f6c74e9e79c57766a6), maybe that's fixed in your> local changes?> > $ ./pre-inst-env guix import go -r github.com/syncthing/syncthing> WARNING: (guix import go): `url-fetch' imported from both (guix> import> utils) and (guix build download)> Backtrace:> In ice-9/boot-9.scm:> 1736:10 7 (with-exception-handler _ _ #:unwind? _ # _)> In unknown file:> 6 (apply-smob/0 #<thunk 7ff10807d9a0>)> In ice-9/boot-9.scm:> 718:2 5 (call-with-prompt _ _ #<procedure default-prompt-> handle…>)> In ice-9/eval.scm:> 619:8 4 (_ #(#(#<directory (guile-user) 7ff107cadf00>)))> In guix/ui.scm:> 2127:12 3 (run-guix-command _ . _)> In guix/scripts/import.scm:> 120:11 2 (guix-import . _)> In ice-9/eval.scm:> 159:9 1 (_ #(#(#(#(#(#(#(#(#(#<directo…>) …) …) …) …) …) …) …)> …))> In guix/import/utils.scm:> 429:0 0 (recursive-import _ #:repo->guix-package _ #:guix-name _> …)> > guix/import/utils.scm:429:0: In procedure recursive-import:> Invalid keyword: #f
I could fix it using the attached patch!
However, I noticed it doesnt pin versions in GNU Guix to what they arein go.mod file, is that expected? It always takes the latest. It mightwork but I am thinking it might cause breakage at some point?
Thank you!
Toggle diff (25 lines)diff --git a/guix/import/go.scm b/guix/import/go.scmindex 61009f3565..c7a1b1a5d4 100644--- a/guix/import/go.scm+++ b/guix/import/go.scm@@ -23,7 +23,7 @@ #:use-module (ice-9 regex) #:use-module (srfi srfi-1) #:use-module (srfi srfi-9)- #:use-module (guix json)+ #:use-module (json) #:use-module ((guix download) #:prefix download:) #:use-module (guix import utils) #:use-module (guix import json)@@ -268,9 +268,9 @@ control system is being used." (define* (go-module-recursive-import package-name #:key (goproxy-url "https://proxy.golang.org"))- (recursive-import package-name #f+ (recursive-import package-name #:repo->guix-package- (lambda (name _)+ (lambda* (name #:key repo version) (go-module->guix-package name #:goproxy-url goproxy-url)) #:guix-name to-guix-package-name))
D
D
dftxbs3e wrote on 10 Dec 2020 04:14
(address . 44178@debbugs.gnu.org)
97498b5966a4e1cd64bc85ec30bbd8007df09173.camel@free.fr
It now fails with:
$ ./pre-inst-env guix import go -r github.com/hashicorp/consul/apiWARNING: (guix import go): `url-fetch' imported from both (guix importutils) and (guix build download)
Starting download of /tmp/guix-file.i8tqa2From https://proxy.golang.org/github.com/hashicorp/consul/api/@v/v1.8.0.mod... v1.8.0.mod 424B 334KiB/s 00:00[##################] 100.0%Backtrace:In ice-9/boot-9.scm: 1736:10 17 (with-exception-handler _ _ #:unwind? _ # _)In unknown file: 16 (apply-smob/0 #<thunk 7fc5871054a0>)In ice-9/boot-9.scm: 718:2 15 (call-with-prompt _ _ #<procedure default-prompt-handle…>)In ice-9/eval.scm: 619:8 14 (_ #(#(#<directory (guile-user) 7fc586d40f00>)))In guix/ui.scm: 2127:12 13 (run-guix-command _ . _)In guix/scripts/import.scm: 120:11 12 (guix-import . _)In ice-9/eval.scm: 159:9 11 (_ _)In guix/import/utils.scm: 458:31 10 (recursive-import _ #:repo->guix-package _ #:guix-name _…) 449:33 9 (lookup-node "github.com/hashicorp/consul/api" #f)In guix/utils.scm: 697:8 8 (call-with-temporary-output-file _)In ice-9/eval.scm: 293:34 7 (_ #(#(#(#(#<directory (guix import go) 7fc…> …) …) …) …)) 159:9 6 (_ #(#(#(#(#<directory (guix import go) 7fc…> …) …) …) …))In ice-9/ports.scm: 445:17 5 (call-with-input-file _ _ #:binary _ #:encoding _ # _) 470:4 4 (_ _)In ice-9/eval.scm: 619:8 3 (_ #(#(#(#<directory (guix import go) 7fc584cbef00>)) …)) 619:8 2 (_ #(#(#<directory (guix import go) 7fc584cbef00> # …) …)) 293:34 1 (_ #(#(#(#(#(#<directory (guix import go…> …) …) …) …) …))In unknown file: 0 (list-ref ("replace" "github.com/hashicorp/consul/s…" …)…)
ERROR: In procedure list-ref:In procedure list-ref: Argument 2 out of range: 4

It's probably because the go.mod file contains a self-referencingreplace line (seems unsupported by the code):
module github.com/hashicorp/consul/api
go 1.12
replace github.com/hashicorp/consul/sdk => ../sdk
require ( github.com/hashicorp/consul/sdk v0.7.0 github.com/hashicorp/go-cleanhttp v0.5.1 github.com/hashicorp/go-hclog v0.12.0 github.com/hashicorp/go-rootcerts v1.0.2 github.com/hashicorp/go-uuid v1.0.1 github.com/hashicorp/serf v0.9.5 github.com/mitchellh/mapstructure v1.1.2 github.com/stretchr/testify v1.4.0)
?