[PATCH] Add golang-importer

  • Done
  • quality assurance status badge
Details
2 participants
  • Maxim Cournoyer
  • nly
Owner
unassigned
Submitted by
nly
Severity
normal
N
(address . guix-patches@gnu.org)
87lfzhjx1s.fsf@disroot.org
Hi. I've attached some patches for a golang importer

To use:
1. (load "golang.scm")
2. (define gopkg (make-go-package go-name*))
3. (package-sexp gopkg)
;; go-name looks like "golang.org/auth/pkg"

;; TODO
1. reliant on github urls
2. extract license
3. Recursive?

Suggestions welcome

Thanks,
Amar
From 98fc69e26354ddf8a7111fbe6560d998d01e4415 Mon Sep 17 00:00:00 2001
From: Amar Singh <nly@disroot.org>
Date: Wed, 1 May 2019 04:13:05 +0530
Subject: [PATCH 02/10] golang Import: formatting

Signed-off-by: Amar Singh <nly@disroot.org>
---
guix/import/golang.scm | 16 ++++++++++------
1 file changed, 10 insertions(+), 6 deletions(-)

Toggle diff (43 lines)
diff --git a/guix/import/golang.scm b/guix/import/golang.scm
index ad822f6b69..f9872ee8e3 100644
--- a/guix/import/golang.scm
+++ b/guix/import/golang.scm
@@ -88,20 +88,23 @@
(sha256 (go-name->sha256 go-name version))
(readme-string (go-name->readme-string go-name)))
(package (inherit go-github-com-alsm-ioprogress)
- (name (string-append "go-" (go-name->name go-name)))
+ (name (string-append "go-"
+ (go-name->name go-name)))
(version version)
(source (origin (method url-fetch)
- (uri (go-name->tarball go-name version))
- (sha256 (base32
- sha256))))
+ (uri (go-name->tarball go-name
+ version))
+ (sha256 (base32 sha256))))
(home-page (go-name->url go-name))
(build-system go-build-system)
(arguments
`(#:import-path ,go-name))
;; TODO: inputs
(synopsis
- (go-name->synopsis go-name readme-string))
- (description (go-name->description go-name readme-string))
+ (go-name->synopsis go-name
+ readme-string))
+ (description (go-name->description go-name
+ readme-string))
;; TODO: license
)))
@@ -117,5 +120,6 @@
;;; 6-7.b try to extract sentences. TODO
;;; 8. go-name->license TODO
;;; 9. go-name->inputs TODO
+;;; 10. package-print TODO
;;; golang.scm ends here
--
2.21.0
From 9cb6242b32f640887a1a7516ae469a4b6bee97cc Mon Sep 17 00:00:00 2001
From: Amar Singh <nly@disroot.org>
Date: Wed, 1 May 2019 04:13:40 +0530
Subject: [PATCH 03/10] Golang-Import: Add go-name->inputs.

* guix/import/golang.scm (go-name->inputs): New variable.

Signed-off-by: Amar Singh <nly@disroot.org>
---
guix/import/golang.scm | 8 ++++++++
1 file changed, 8 insertions(+)

Toggle diff (21 lines)
diff --git a/guix/import/golang.scm b/guix/import/golang.scm
index f9872ee8e3..e11d1c2230 100644
--- a/guix/import/golang.scm
+++ b/guix/import/golang.scm
@@ -82,6 +82,14 @@
(string-append (go-name->name go-name)
(substring readme-string 100 300)))
+;;; go list -f '{{ join .Deps "\n" }}',recursively find dependencies
+;;; go list -f '{{ join .Imports "\n" }}' ,non recursive
+(define (go-name->inputs go-name)
+ (let ((tmp (tmpnam)))
+ (and (zero? (system (string-append "go list -f '{{ join .Deps \"\\n\" }}'" " "
+ go-name " > " tmp)))
+ (call-with-input-file tmp get-string-all))))
+
(define-public (make-go-package go-name)
;; Do the expensive operations only once; query network for latest version
(let* ((version (latest-release go-name))
--
2.21.0
From 912e6fb0d35b43e3d1cf540702fa1905e704ce93 Mon Sep 17 00:00:00 2001
From: Amar Singh <nly@disroot.org>
Date: Thu, 2 May 2019 00:46:56 +0530
Subject: [PATCH 06/10] Formatting

Use: format-inputs instead of alist-sexp

Signed-off-by: Amar Singh <nly@disroot.org>
---
guix/import/golang.scm | 14 ++++++--------
1 file changed, 6 insertions(+), 8 deletions(-)

Toggle diff (38 lines)
diff --git a/guix/import/golang.scm b/guix/import/golang.scm
index e0ffca4b42..c0679990e1 100644
--- a/guix/import/golang.scm
+++ b/guix/import/golang.scm
@@ -136,8 +136,8 @@
(version version)
(source
(origin (method url-fetch)
- (uri (go-name->tarball go-name version))
- (sha256 (base32 sha256))))
+ (uri (go-name->tarball go-name version))
+ (sha256 (base32 sha256))))
(home-page
(go-name->url go-name))
(build-system
@@ -146,16 +146,14 @@
`(#:import-path ,go-name))
;; TODO: make inputs into (unquote ..) form
(inputs
- (alist-sexp (go-name->inputs go-name)))
- (synopsis
- (go-name->synopsis go-name readme-string))
- (description
- (go-name->description go-name readme-string))
+ (format-inputs (map go-name->guix-name (go-name->inputs go-name))))
+ (synopsis (go-name->synopsis go-name))
+ (description (go-name->description go-name))
;; TODO: license
)))
(define (filter-newlines string)
- (string-filter (lambda (x) (not (equal? #\ x))) string))
+ (string-filter (lambda (x) (not (equal? x #\newline))) string))
;;; Ask the upstream to export variable?
(define bv->nix-base32 (@@ (guix packages)
--
2.21.0
From 3132d5e9a7293ba934111079617498ab68a61c89 Mon Sep 17 00:00:00 2001
From: Amar Singh <nly@disroot.org>
Date: Thu, 2 May 2019 00:48:06 +0530
Subject: [PATCH 07/10] Formatting

Update: TODO License; Meaningful Description and Synopsis

Signed-off-by: Amar Singh <nly@disroot.org>
---
guix/import/golang.scm | 12 +++++++-----
1 file changed, 7 insertions(+), 5 deletions(-)

Toggle diff (38 lines)
diff --git a/guix/import/golang.scm b/guix/import/golang.scm
index c0679990e1..319807df88 100644
--- a/guix/import/golang.scm
+++ b/guix/import/golang.scm
@@ -185,24 +185,26 @@
(arguments ,(package-arguments package))
(synopsis ,(filter-newlines (package-synopsis package)))
(description ,(filter-newlines (package-description package)))
- (inputs ,(alist-sexp (package-inputs package)))
- (native-inputs ,(alist-sexp (package-native-inputs package)))
- (propagated-inputs ,(alist-sexp (package-propagated-inputs package)))))
+ (inputs ,(format-inputs (map car (package-inputs package))))
+ (native-inputs ,(format-inputs (map car (package-native-inputs package))))
+ (propagated-inputs ,(format-inputs (map car (package-propagated-inputs package))))))
;;; STATUS
;;; 1. latest-release DONE
;;; 1.b latest-commit PENDING/STALLED
-;;; 2. go-name->name DONE
+;;; 2. go-name->guix-name DONE
+;;; 2.b style go-github-com-user-project TODO
;;; 4. go-name->url DONE
;;; 4.b go-name->tarball DONE
;;; 5. go-name->sha256 (go-name version) DONE
;;; 6. go-name->synopsis DONE
;;; 7. go-name->description DONE
+;;; 4-7.b. Memoize, network procedures DONE
;;; 6-7.b try to extract sentences. TODO
;;; 8. go-name->license TODO
;;; 9. go-name->inputs DONE
-;;; 9.b. inputs alist TODO
+;;; 9.b. inputs alist DONE
;;; 10. package-sexp DONE
;;; 10.a origin-sexp DONE
--
2.21.0
From 65814de01030fdbf1d9d4c349f5149d645dae299 Mon Sep 17 00:00:00 2001
From: Amar Singh <nly@disroot.org>
Date: Thu, 2 May 2019 01:23:04 +0530
Subject: [PATCH 08/10] Mark DONE: go-name->guix-name

Signed-off-by: Amar Singh <nly@disroot.org>
---
guix/import/golang.scm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)

Toggle diff (15 lines)
diff --git a/guix/import/golang.scm b/guix/import/golang.scm
index 319807df88..2f0aca3331 100644
--- a/guix/import/golang.scm
+++ b/guix/import/golang.scm
@@ -194,7 +194,7 @@
;;; 1. latest-release DONE
;;; 1.b latest-commit PENDING/STALLED
;;; 2. go-name->guix-name DONE
-;;; 2.b style go-github-com-user-project TODO
+;;; 2.b style go-github-com-user-project DONE
;;; 4. go-name->url DONE
;;; 4.b go-name->tarball DONE
;;; 5. go-name->sha256 (go-name version) DONE
--
2.21.0
From 1e5386ca1369dc45390203819a3e7179381dbad2 Mon Sep 17 00:00:00 2001
From: Amar Singh <nly@disroot.org>
Date: Thu, 2 May 2019 01:31:43 +0530
Subject: [PATCH 09/10] Move todo to top, Add usage instructions

Fix: Also removed a leftover test expression

Signed-off-by: Amar Singh <nly@disroot.org>
---
guix/import/golang.scm | 49 ++++++++++++++++++++++--------------------
1 file changed, 26 insertions(+), 23 deletions(-)

Toggle diff (86 lines)
diff --git a/guix/import/golang.scm b/guix/import/golang.scm
index 2f0aca3331..3eb74d4984 100644
--- a/guix/import/golang.scm
+++ b/guix/import/golang.scm
@@ -36,6 +36,30 @@
(srfi srfi-26) ;; cut
)
+;;; To use, simply:
+;;; 1. (load "golang.scm")
+;;; 2. (define go-package (make-package go-name*))
+;;; 3. (package-sexp go-package)
+
+;;; STATUS
+;;; 1. latest-release DONE
+;;; 1.b latest-commit PENDING/STALLED
+;;; 2. go-name->guix-name DONE
+;;; 2.b style go-github-com-user-project DONE
+;;; 4. go-name->url DONE
+;;; 4.b go-name->tarball DONE
+;;; 5. go-name->sha256 (go-name version) DONE
+;;; 6. go-name->synopsis DONE
+;;; 7. go-name->description DONE
+;;; 4-7.b. Memoize, network procedures DONE
+;;; 6-7.b try to extract sentences. TODO
+;;; 8. go-name->license TODO
+;;; 9. go-name->inputs DONE
+;;; 9.b. inputs alist DONE
+;;; 10. package-sexp DONE
+;;; 10.a origin-sexp DONE
+;;; 11. Package Builds TODO
+
(define-public go-name* "github.com/gohugoio/hugo") ;; for tests
(define* (go-name->url go-name #:rest args)
@@ -91,14 +115,13 @@
(call-with-input-file (url->store (go-name->readme go-name))
read-string))))
-;;; Maybe try to match the first sentence.
+;;; TODO: try to match the first sentence.
(define (go-name->synopsis go-name)
(substring (go-name->readme-string go-name) 0 100))
-;;; Maybe try to match the the next two sentences.
+;;; TODO: try to match the the next two sentences.
(define (go-name->description go-name)
(substring (go-name->readme-string go-name) 100 300))
-(go-name->description go-name*)
(define shell-command
(lambda* (command #:rest args)
@@ -155,7 +178,6 @@
(define (filter-newlines string)
(string-filter (lambda (x) (not (equal? x #\newline))) string))
-;;; Ask the upstream to export variable?
(define bv->nix-base32 (@@ (guix packages)
bytevector->nix-base32-string))
@@ -189,23 +211,4 @@
(native-inputs ,(format-inputs (map car (package-native-inputs package))))
(propagated-inputs ,(format-inputs (map car (package-propagated-inputs package))))))
-
-;;; STATUS
-;;; 1. latest-release DONE
-;;; 1.b latest-commit PENDING/STALLED
-;;; 2. go-name->guix-name DONE
-;;; 2.b style go-github-com-user-project DONE
-;;; 4. go-name->url DONE
-;;; 4.b go-name->tarball DONE
-;;; 5. go-name->sha256 (go-name version) DONE
-;;; 6. go-name->synopsis DONE
-;;; 7. go-name->description DONE
-;;; 4-7.b. Memoize, network procedures DONE
-;;; 6-7.b try to extract sentences. TODO
-;;; 8. go-name->license TODO
-;;; 9. go-name->inputs DONE
-;;; 9.b. inputs alist DONE
-;;; 10. package-sexp DONE
-;;; 10.a origin-sexp DONE
-
;;; golang.scm ends here
--
2.21.0
From 05b03042d1bd20969ff8d352372e3a9e58977276 Mon Sep 17 00:00:00 2001
From: Amar Singh <nly@disroot.org>
Date: Thu, 2 May 2019 19:52:51 +0530
Subject: [PATCH 10/10] Bootstrap: don't use (gnu packages golang)

---
guix/import/golang.scm | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)

Toggle diff (37 lines)
diff --git a/guix/import/golang.scm b/guix/import/golang.scm
index 3eb74d4984..b51d496602 100644
--- a/guix/import/golang.scm
+++ b/guix/import/golang.scm
@@ -26,10 +26,11 @@
(guix download) ;; download-to-store
((guix import utils) #:prefix utils:) ;; hash
(guix packages) ;; packages
+ ((guix licenses) #:prefix license:) ;; licenses
(guix build-system) ;; build-system printer
(guix build-system go) ;; go-build-system
(guix store) ;; with-store
- (gnu packages golang) ;; inherit (simple) go package
+ ;; (gnu packages golang) ;; inherit (simple) go package
(ice-9 textual-ports) ;; to parse readme.md
(ice-9 popen) ;; open-input-ouput-pipe
(web uri) ;; uri->string
@@ -153,7 +154,8 @@
(let* ((version (latest-release go-name))
(sha256 (go-name->sha256 go-name version))
(readme-string (go-name->readme-string go-name)))
- (package (inherit go-github-com-alsm-ioprogress)
+ (package
+ ;; (inherit go-github-com-alsm-ioprogress)
(name
(string-append "go-" go-name))
(version version)
@@ -173,6 +175,7 @@
(synopsis (go-name->synopsis go-name))
(description (go-name->description go-name))
;; TODO: license
+ (license license:expat)
)))
(define (filter-newlines string)
--
2.21.0
A
A
Amar Singh wrote on 26 May 2019 18:44
[PATCH] Importer: Add golang
(address . 35627@debbugs.gnu.org)
85blzp6u88.fsf@disroot.org
This Go importer is flaky. It's a rebased patch.

From f54e0aa0ae3f553701b1253a6e8605a493e4d4ac Mon Sep 17 00:00:00 2001
From: Amar Singh <nly@disroot.org>
Date: Tue, 30 Apr 2019 23:17:51 +0530
Subject: [PATCH] Importer: Add golang

* guix/import/golang.scm.

Signed-off-by: Amar Singh <nly@disroot.org>
---
guix/import/golang.scm | 217 +++++++++++++++++++++++++++++++++++++++++
1 file changed, 217 insertions(+)
create mode 100644 guix/import/golang.scm

Toggle diff (225 lines)
diff --git a/guix/import/golang.scm b/guix/import/golang.scm
new file mode 100644
index 0000000000..b51d496602
--- /dev/null
+++ b/guix/import/golang.scm
@@ -0,0 +1,217 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2019 by Amar Singh <nly@disroot.org>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; This program is free software: you can redistribute it and/or modify
+;;; it under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation, either version 3 of the License, or
+;;; (at your option) any later version.
+;;;
+;;; This program is distributed in the hope that it will be useful,
+;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+(define-module (guix import golang))
+(use-modules
+ (srfi srfi-1) ;; fold
+ (ice-9 rdelim) ;; read-string
+ (guix import github) ;; latest-release
+ (guix utils) ;; string-replace-substring
+ (guix memoization) ;; memoize network operations
+ (guix download) ;; download-to-store
+ ((guix import utils) #:prefix utils:) ;; hash
+ (guix packages) ;; packages
+ ((guix licenses) #:prefix license:) ;; licenses
+ (guix build-system) ;; build-system printer
+ (guix build-system go) ;; go-build-system
+ (guix store) ;; with-store
+ ;; (gnu packages golang) ;; inherit (simple) go package
+ (ice-9 textual-ports) ;; to parse readme.md
+ (ice-9 popen) ;; open-input-ouput-pipe
+ (web uri) ;; uri->string
+ (srfi srfi-26) ;; cut
+ )
+
+;;; To use, simply:
+;;; 1. (load "golang.scm")
+;;; 2. (define go-package (make-package go-name*))
+;;; 3. (package-sexp go-package)
+
+;;; STATUS
+;;; 1. latest-release DONE
+;;; 1.b latest-commit PENDING/STALLED
+;;; 2. go-name->guix-name DONE
+;;; 2.b style go-github-com-user-project DONE
+;;; 4. go-name->url DONE
+;;; 4.b go-name->tarball DONE
+;;; 5. go-name->sha256 (go-name version) DONE
+;;; 6. go-name->synopsis DONE
+;;; 7. go-name->description DONE
+;;; 4-7.b. Memoize, network procedures DONE
+;;; 6-7.b try to extract sentences. TODO
+;;; 8. go-name->license TODO
+;;; 9. go-name->inputs DONE
+;;; 9.b. inputs alist DONE
+;;; 10. package-sexp DONE
+;;; 10.a origin-sexp DONE
+;;; 11. Package Builds TODO
+
+(define-public go-name* "github.com/gohugoio/hugo") ;; for tests
+
+(define* (go-name->url go-name #:rest args)
+ (if (string-contains go-name ".")
+ (uri->string (string->uri (apply string-append
+ "https://" go-name args)))
+ #f))
+
+(define (go-name->tarball go-name version)
+ (go-name->url go-name "/archive/v"
+ version ".tar.gz"))
+
+(define* (string-replace-substrings string substrings
+ #:optional (replacement "-"))
+ (if (null-list? substrings)
+ string
+ ((cut string-replace-substring <> (car substrings) replacement)
+ (string-replace-substrings string (cdr substrings)))))
+
+;;; Possible remove @@ if upstream exports the symbols
+(define (go-name->guix-name go-name)
+ (string-append "go-"
+ (string-replace-substrings go-name '("." "/") "-")))
+
+;;; Slow; accesses the network; memoized
+(define latest-release
+ (memoize
+ (lambda (go-name)
+ ((@@ (guix import github) latest-released-version)
+ (go-name->url go-name)
+ (go-name->guix-name go-name)))))
+
+;;; Slow; downloads the url from network; memoized
+(define url->store
+ (@@ (guix import cran) download))
+
+;;; Slow; download src tarball from network, returns base32 nix-hash;
+;;; memoized
+(define (go-name->sha256 go-name version)
+ (utils:guix-hash-url (url->store (go-name->tarball go-name version))))
+
+;;; Slow; network access; memoized
+(define go-name->readme-string
+ (memoize
+ (lambda (go-name)
+ (define (go-name->readme go-name)
+ (go-name->url "raw.githubusercontent.com"
+ ;; TODO, detect the domain
+ (substring go-name
+ (string-length "github.com"))
+ "/master/"
+ "README.md"))
+ (call-with-input-file (url->store (go-name->readme go-name))
+ read-string))))
+
+;;; TODO: try to match the first sentence.
+(define (go-name->synopsis go-name)
+ (substring (go-name->readme-string go-name) 0 100))
+
+;;; TODO: try to match the the next two sentences.
+(define (go-name->description go-name)
+ (substring (go-name->readme-string go-name) 100 300))
+
+(define shell-command
+ (lambda* (command #:rest args)
+ (let* ((cmd (string-join (cons command (delete #f (delete '() args))) " "))
+ (port (open-input-output-pipe cmd))
+ (result (read-string port))
+ (exit-code (close-pipe port)))
+ (and (zero? exit-code)
+ (string-split (string-trim-right result) #\newline)))))
+
+(define go-name->inputs
+ (lambda (go-name)
+ (let ((recursive-depends "-f '{{ join .Deps \"\\n\" }}'")
+ (direct-depends "-f '{{ join .Imports \"\\n\" }}'")
+ (go-command (car (shell-command "which go"))))
+ (shell-command go-command "list" direct-depends go-name))))
+
+;;; License
+(define (string->license license-string)
+ ((@@ (guix import cran) string->license) (string-upcase license-string)))
+
+;;; For inputs
+(define format-inputs
+ (@@ (guix import cran) format-inputs))
+
+(define-public (make-go-package go-name)
+ ;; Do the expensive operations only once; query network for latest
+ ;; version
+ (let* ((version (latest-release go-name))
+ (sha256 (go-name->sha256 go-name version))
+ (readme-string (go-name->readme-string go-name)))
+ (package
+ ;; (inherit go-github-com-alsm-ioprogress)
+ (name
+ (string-append "go-" go-name))
+ (version version)
+ (source
+ (origin (method url-fetch)
+ (uri (go-name->tarball go-name version))
+ (sha256 (base32 sha256))))
+ (home-page
+ (go-name->url go-name))
+ (build-system
+ go-build-system)
+ (arguments
+ `(#:import-path ,go-name))
+ ;; TODO: make inputs into (unquote ..) form
+ (inputs
+ (format-inputs (map go-name->guix-name (go-name->inputs go-name))))
+ (synopsis (go-name->synopsis go-name))
+ (description (go-name->description go-name))
+ ;; TODO: license
+ (license license:expat)
+ )))
+
+(define (filter-newlines string)
+ (string-filter (lambda (x) (not (equal? x #\newline))) string))
+
+(define bv->nix-base32 (@@ (guix packages)
+ bytevector->nix-base32-string))
+
+(define (origin-sexp origin)
+ `(origin
+ (method url-fetch)
+ (uri ,(origin-uri origin))
+ (sha256 (base32 ,(bv->nix-base32 (origin-sha256 origin))))
+ (file-name ,(origin-file-name origin))
+ (patches ,(origin-patches origin))
+ (snippet ,(origin-snippet origin))
+ (patch-flags ,(origin-patch-flags origin))
+ (patch-inputs ,(origin-patch-inputs origin))
+ (modules ,(origin-modules origin))
+ (patch-guile ,(origin-patch-guile origin))))
+
+(define (build-system-sexp build-system)
+ (symbol-append (build-system-name build-system) '-build-system))
+
+(define-public (package-sexp package)
+ `(package
+ (name ,(package-name package))
+ (version ,(package-version package))
+ (source ,(origin-sexp (package-source package)))
+ (home-page ,(package-home-page package))
+ (build-system ,(build-system-sexp (package-build-system package)))
+ (arguments ,(package-arguments package))
+ (synopsis ,(filter-newlines (package-synopsis package)))
+ (description ,(filter-newlines (package-description package)))
+ (inputs ,(format-inputs (map car (package-inputs package))))
+ (native-inputs ,(format-inputs (map car (package-native-inputs package))))
+ (propagated-inputs ,(format-inputs (map car (package-propagated-inputs package))))))
+
+;;; golang.scm ends here
--
2.21.0
M
M
Maxim Cournoyer wrote on 6 Aug 2021 05:40
Re: bug#35627: [PATCH] Add golang-importer
(name . nly)(address . nly@disroot.org)(address . 35627-done@debbugs.gnu.org)
87o8abkzq3.fsf@gmail.com
Hello Amar,

nly <nly@disroot.org> writes:

Toggle quote (18 lines)
> Hi. I've attached some patches for a golang importer
>
> To use:
> 1. (load "golang.scm")
> 2. (define gopkg (make-go-package go-name*))
> 3. (package-sexp gopkg)
> ;; go-name looks like "golang.org/auth/pkg"
>
> ;; TODO
> 1. reliant on github urls
> 2. extract license
> 3. Recursive?
>
> Suggestions welcome
>
> Thanks,
> Amar

Thank you for your efforts. Perhaps others have built on it, as we now
have a relatively featureful Go importer in Guix, as you probably saw!

Closing,

Maxim
Closed
?