(address . guix-patches@gnu.org)(name . Mathieu Othacehe)(address . othacehe@gnu.org)
20200629142434.21308-1-othacehe@gnu.org
This commit enables GZIP compression for linux-libre kernel modules, reducing
the size of linux-libre by 63% (165MB). The initrd modules are kept
uncompressed as the initrd is already compressed as a whole.
The linux-libre kernel also supports XZ compression, but as Guix does not have
any available bindings for now, and the compression time is far more
significant, GZIP seems to be a better option.
* gnu/packages/aux-files/linux-libre/5.4-arm.conf: Enable GZ compression.
* gnu/packages/aux-files/linux-libre/5.4-arm64.conf: Ditto.
* gnu/packages/aux-files/linux-libre/5.4-i686.conf: Ditto.
* gnu/packages/aux-files/linux-libre/5.4-x86_64.conf: Ditto.
* gnu/build/linux-modules.scm (modinfo-section-contents): Use
'call-with-gzip-input-port' to read from a module file using '.gz' extension,
(strip-extension): new procedure,
(dot-ko): adapt to support compression,
(ensure-dot-ko): ditto,
(file-name->module-name): ditto,
(find-module-file): ditto,
(load-linux-module*): ditto,
(module-name->file-name/guess): ditto,
(module-name-lookup): ditto,
(write-module-name-database): ditto,
(write-module-alias-database): ditto,
(write-module-device-database): ditto.
* gnu/system/linux-initrd.scm (flat-linux-module-directory): Make sure that
zlib bindings are available because they may be used in
'write-module-device-database'. Also make sure that the initrd only contains
uncompressed module files.
---
Hello,
I think the commit message pretty much describes this change. Just wanted to
add that it passes the loadable-kernel-modules-X tests.
Thanks,
Mathieu
gnu/build/linux-modules.scm | 102 ++++++++++++------
.../aux-files/linux-libre/5.4-arm.conf | 4 +-
.../aux-files/linux-libre/5.4-arm64.conf | 4 +-
.../aux-files/linux-libre/5.4-i686.conf | 4 +-
.../aux-files/linux-libre/5.4-x86_64.conf | 4 +-
gnu/system/linux-initrd.scm | 38 +++++--
6 files changed, 111 insertions(+), 45 deletions(-)
Toggle diff (341 lines)
diff --git a/gnu/build/linux-modules.scm b/gnu/build/linux-modules.scm
index aa1c7cfeae..7c6945a881 100644
--- a/gnu/build/linux-modules.scm
+++ b/gnu/build/linux-modules.scm
@@ -21,6 +21,7 @@
(define-module (gnu build linux-modules)
#:use-module (guix elf)
#:use-module (guix glob)
+ #:use-module (guix zlib)
#:use-module (guix build syscalls)
#:use-module ((guix build utils) #:select (find-files invoke))
#:use-module (guix build union)
@@ -97,7 +98,16 @@ string list."
(define (modinfo-section-contents file)
"Return the contents of the '.modinfo' section of FILE as a list of
key/value pairs.."
- (let* ((bv (call-with-input-file file get-bytevector-all))
+ (define (get-bytevector file)
+ (cond
+ ((string-contains file ".ko.gz")
+ (call-with-input-file file
+ (lambda (port)
+ (call-with-gzip-input-port port get-bytevector-all))))
+ (else
+ (call-with-input-file file get-bytevector-all))))
+
+ (let* ((bv (get-bytevector file))
(elf (parse-elf bv))
(section (elf-section-by-name elf ".modinfo"))
(modinfo (section-contents elf section)))
@@ -110,7 +120,7 @@ key/value pairs.."
(define (module-formal-name file)
"Return the module name of FILE as it appears in its info section. Usually
the module name is the same as the base name of FILE, modulo hyphens and minus
-the \".ko\" extension."
+the \".ko[.gz|.xz]\" extension."
(match (assq 'name (modinfo-section-contents file))
(('name . name) name)
(#f #f)))
@@ -171,14 +181,25 @@ modules that can be postloaded, of the soft dependencies of module FILE."
(_ #f))
(modinfo-section-contents file))))
-(define dot-ko
- (cut string-append <> ".ko"))
-
-(define (ensure-dot-ko name)
- "Return NAME with a '.ko' prefix appended, unless it already has it."
- (if (string-suffix? ".ko" name)
+(define (strip-extension filename)
+ (let ((extension (string-index filename #\.)))
+ (if extension
+ (string-take filename extension)
+ filename)))
+
+(define (dot-ko name compression)
+ (let ((suffix (match compression
+ ('xz ".ko.xz")
+ ('gzip ".ko.gz")
+ (else ".ko"))))
+ (string-append name suffix)))
+
+(define (ensure-dot-ko name compression)
+ "Return NAME with a '.ko[.gz|.xz]' suffix appended, unless it already has
+it."
+ (if (string-contains name ".ko")
name
- (dot-ko name)))
+ (dot-ko name compression)))
(define (normalize-module-name module)
"Return the \"canonical\" name for MODULE, replacing hyphens with
@@ -191,9 +212,9 @@ underscores."
module))
(define (file-name->module-name file)
- "Return the module name corresponding to FILE, stripping the trailing '.ko'
-and normalizing it."
- (normalize-module-name (basename file ".ko")))
+ "Return the module name corresponding to FILE, stripping the trailing
+'.ko[.gz|.xz]' and normalizing it."
+ (normalize-module-name (strip-extension (basename file))))
(define (find-module-file directory module)
"Lookup module NAME under DIRECTORY, and return its absolute file name.
@@ -208,19 +229,19 @@ whereas file names often, but not always, use hyphens. Examples:
;; List of possible file names. XXX: It would of course be cleaner to
;; have a database that maps module names to file names and vice versa,
;; but everyone seems to be doing hacks like this one. Oh well!
- (map ensure-dot-ko
- (delete-duplicates
- (list module
- (normalize-module-name module)
- (string-map (lambda (chr) ;converse of 'normalize-module-name'
- (case chr
- ((#\_) #\-)
- (else chr)))
- module)))))
+ (delete-duplicates
+ (list module
+ (normalize-module-name module)
+ (string-map (lambda (chr) ;converse of 'normalize-module-name'
+ (case chr
+ ((#\_) #\-)
+ (else chr)))
+ module))))
(match (find-files directory
(lambda (file stat)
- (member (basename file) names)))
+ (member (strip-extension
+ (basename file)) names)))
((file)
file)
(()
@@ -290,8 +311,8 @@ not a file name."
(recursive? #t)
(lookup-module dot-ko)
(black-list (module-black-list)))
- "Load Linux module from FILE, the name of a '.ko' file; return true on
-success, false otherwise. When RECURSIVE? is true, load its dependencies
+ "Load Linux module from FILE, the name of a '.ko[.gz|.xz]' file; return true
+on success, false otherwise. When RECURSIVE? is true, load its dependencies
first (à la 'modprobe'.) The actual files containing modules depended on are
obtained by calling LOOKUP-MODULE with the module name. Modules whose name
appears in BLACK-LIST are not loaded."
@@ -523,16 +544,27 @@ are required to access DEVICE."
;;; Module databases.
;;;
-(define (module-name->file-name/guess directory name)
+(define* (module-name->file-name/guess directory name
+ #:key compression)
"Guess the file name corresponding to NAME, a module name. That doesn't
always work because sometimes underscores in NAME map to hyphens (e.g.,
\"input-leds.ko\"), sometimes not (e.g., \"mac_hid.ko\")."
- (string-append directory "/" (ensure-dot-ko name)))
+ (string-append directory "/" (ensure-dot-ko name compression)))
(define (module-name-lookup directory)
"Return a one argument procedure that takes a module name (e.g.,
\"input_leds\") and returns its absolute file name (e.g.,
\"/.../input-leds.ko\")."
+ (define (guess-file-name name)
+ (let ((names (list
+ (module-name->file-name/guess directory name)
+ (module-name->file-name/guess directory name
+ #:compression 'xz)
+ (module-name->file-name/guess directory name
+ #:compression 'gzip))))
+ (or (find file-exists? names)
+ (first names))))
+
(catch 'system-error
(lambda ()
(define mapping
@@ -541,23 +573,23 @@ always work because sometimes underscores in NAME map to hyphens (e.g.,
(lambda (name)
(or (assoc-ref mapping name)
- (module-name->file-name/guess directory name))))
+ (guess-file-name name))))
(lambda args
(if (= ENOENT (system-error-errno args))
- (cut module-name->file-name/guess directory <>)
+ (cut guess-file-name <>)
(apply throw args)))))
(define (write-module-name-database directory)
"Write a database that maps \"module names\" as they appear in the relevant
-ELF section of '.ko' files, to actual file names. This format is
+ELF section of '.ko[.gz|.xz]' files, to actual file names. This format is
Guix-specific. It aims to deal with inconsistent naming, in particular
hyphens vs. underscores."
(define mapping
(map (lambda (file)
(match (module-formal-name file)
- (#f (cons (basename file ".ko") file))
+ (#f (cons (strip-extension (basename file)) file))
(name (cons name file))))
- (find-files directory "\\.ko$")))
+ (find-files directory "\\.ko.*$")))
(call-with-output-file (string-append directory "/modules.name")
(lambda (port)
@@ -569,12 +601,12 @@ hyphens vs. underscores."
(pretty-print mapping port))))
(define (write-module-alias-database directory)
- "Traverse the '.ko' files in DIRECTORY and create the corresponding
+ "Traverse the '.ko[.gz|.xz]' files in DIRECTORY and create the corresponding
'modules.alias' file."
(define aliases
(map (lambda (file)
(cons (file-name->module-name file) (module-aliases file)))
- (find-files directory "\\.ko$")))
+ (find-files directory "\\.ko.*$")))
(call-with-output-file (string-append directory "/modules.alias")
(lambda (port)
@@ -616,7 +648,7 @@ are found, return a tuple (DEVNAME TYPE MAJOR MINOR), otherwise return #f."
(char-set-complement (char-set #\-)))
(define (write-module-device-database directory)
- "Traverse the '.ko' files in DIRECTORY and create the corresponding
+ "Traverse the '.ko[.gz|.xz]' files in DIRECTORY and create the corresponding
'modules.devname' file. This file contains information about modules that can
be loaded on-demand, such as file system modules."
(define aliases
@@ -624,7 +656,7 @@ be loaded on-demand, such as file system modules."
(match (aliases->device-tuple (module-aliases file))
(#f #f)
(tuple (cons (file-name->module-name file) tuple))))
- (find-files directory "\\.ko$")))
+ (find-files directory "\\.ko.*$")))
(call-with-output-file (string-append directory "/modules.devname")
(lambda (port)
diff --git a/gnu/packages/aux-files/linux-libre/5.4-arm.conf b/gnu/packages/aux-files/linux-libre/5.4-arm.conf
index a54228643b..7c9ab94719 100644
--- a/gnu/packages/aux-files/linux-libre/5.4-arm.conf
+++ b/gnu/packages/aux-files/linux-libre/5.4-arm.conf
@@ -880,7 +880,9 @@ CONFIG_MODULE_FORCE_UNLOAD=y
CONFIG_MODVERSIONS=y
# CONFIG_MODULE_SRCVERSION_ALL is not set
# CONFIG_MODULE_SIG is not set
-# CONFIG_MODULE_COMPRESS is not set
+CONFIG_MODULE_COMPRESS=y
+CONFIG_MODULE_COMPRESS_GZIP=y
+# CONFIG_MODULE_COMPRESS_XZ is not set
# CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS is not set
# CONFIG_UNUSED_SYMBOLS is not set
# CONFIG_TRIM_UNUSED_KSYMS is not set
diff --git a/gnu/packages/aux-files/linux-libre/5.4-arm64.conf b/gnu/packages/aux-files/linux-libre/5.4-arm64.conf
index fabd25c6a4..6520d1ddf2 100644
--- a/gnu/packages/aux-files/linux-libre/5.4-arm64.conf
+++ b/gnu/packages/aux-files/linux-libre/5.4-arm64.conf
@@ -781,7 +781,9 @@ CONFIG_MODVERSIONS=y
CONFIG_ASM_MODVERSIONS=y
# CONFIG_MODULE_SRCVERSION_ALL is not set
# CONFIG_MODULE_SIG is not set
-# CONFIG_MODULE_COMPRESS is not set
+CONFIG_MODULE_COMPRESS=y
+CONFIG_MODULE_COMPRESS_GZIP=y
+# CONFIG_MODULE_COMPRESS_XZ is not set
# CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS is not set
# CONFIG_UNUSED_SYMBOLS is not set
# CONFIG_TRIM_UNUSED_KSYMS is not set
diff --git a/gnu/packages/aux-files/linux-libre/5.4-i686.conf b/gnu/packages/aux-files/linux-libre/5.4-i686.conf
index 50a41f6cc9..3727f9d486 100644
--- a/gnu/packages/aux-files/linux-libre/5.4-i686.conf
+++ b/gnu/packages/aux-files/linux-libre/5.4-i686.conf
@@ -850,7 +850,9 @@ CONFIG_MODVERSIONS=y
CONFIG_ASM_MODVERSIONS=y
CONFIG_MODULE_SRCVERSION_ALL=y
# CONFIG_MODULE_SIG is not set
-# CONFIG_MODULE_COMPRESS is not set
+CONFIG_MODULE_COMPRESS=y
+CONFIG_MODULE_COMPRESS_GZIP=y
+# CONFIG_MODULE_COMPRESS_XZ is not set
# CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS is not set
CONFIG_UNUSED_SYMBOLS=y
CONFIG_MODULES_TREE_LOOKUP=y
diff --git a/gnu/packages/aux-files/linux-libre/5.4-x86_64.conf b/gnu/packages/aux-files/linux-libre/5.4-x86_64.conf
index 30fdc4e86a..be7a603af1 100644
--- a/gnu/packages/aux-files/linux-libre/5.4-x86_64.conf
+++ b/gnu/packages/aux-files/linux-libre/5.4-x86_64.conf
@@ -849,7 +849,9 @@ CONFIG_MODVERSIONS=y
CONFIG_ASM_MODVERSIONS=y
CONFIG_MODULE_SRCVERSION_ALL=y
# CONFIG_MODULE_SIG is not set
-# CONFIG_MODULE_COMPRESS is not set
+CONFIG_MODULE_COMPRESS=y
+CONFIG_MODULE_COMPRESS_GZIP=y
+# CONFIG_MODULE_COMPRESS_XZ is not set
# CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS is not set
CONFIG_UNUSED_SYMBOLS=y
CONFIG_MODULES_TREE_LOOKUP=y
diff --git a/gnu/system/linux-initrd.scm b/gnu/system/linux-initrd.scm
index 0971ec29e2..99ec82246b 100644
--- a/gnu/system/linux-initrd.scm
+++ b/gnu/system/linux-initrd.scm
@@ -111,11 +111,29 @@ the derivations referenced by EXP are automatically copied to the initrd."
(define (flat-linux-module-directory linux modules)
"Return a flat directory containing the Linux kernel modules listed in
MODULES and taken from LINUX."
+ (define zlib
+ (module-ref (resolve-interface '(gnu packages compression)) 'zlib))
+
+ (define config.scm
+ (scheme-file "config.scm"
+ #~(begin
+ (define-module (guix config)
+ #:export (%libz))
+
+ (define %libz
+ #+(file-append zlib "/lib/libz")))))
+
+ (define imported-modules
+ (cons `((guix config) => ,config.scm)
+ (delete '(guix config)
+ (source-module-closure '((gnu build linux-modules)
+ (guix build utils))))))
+
(define build-exp
- (with-imported-modules (source-module-closure
- '((gnu build linux-modules)))
+ (with-imported-modules imported-modules
#~(begin
(use-modules (gnu build linux-modules)
+ (guix build utils)
(srfi srfi-1)
(srfi srfi-26))
@@ -129,12 +147,20 @@ MODULES and taken from LINUX."
(recursive-module-dependencies modules
#:lookup-module lookup))))
+ (define (maybe-uncompress file)
+ ;; If FILE is a compressed module, uncompress it, as the initrd is
+ ;; already gzipped as a whole.
+ (cond
+ ((string-contains file ".ko.gz")
+ (invoke #+(file-append gzip "/bin/gunzip") file))))
+
(mkdir #$output)
(for-each (lambda (module)
- (format #t "copying '~a'...~%" module)
- (copy-file module
- (string-append #$output "/"
- (basename module))))
+ (let ((out-module
+ (string-append #$output "/" (basename module))))
+ (format #t "copying '~a'...~%" module)
+ (copy-file module out-module)
+ (maybe-uncompress out-module)))
(delete-duplicates modules))
;; Hyphen or underscore? This database tells us.
--
2.26.2