Toggle diff (432 lines)
diff --git a/gnu/local.mk b/gnu/local.mk
index 0bb2276a2..aa262d2f3 100644
--- a/gnu/local.mk
+++ b/gnu/local.mk
@@ -353,6 +353,7 @@ GNU_SYSTEM_MODULES = \
%D%/packages/ssh.scm \
%D%/packages/stalonetray.scm \
%D%/packages/statistics.scm \
+ %D%/packages/storage.scm \
%D%/packages/suckless.scm \
%D%/packages/swig.scm \
%D%/packages/sync.scm \
@@ -508,6 +509,9 @@ dist_patch_DATA = \
%D%/packages/patches/calibre-drop-unrar.patch \
%D%/packages/patches/calibre-no-updates-dialog.patch \
%D%/packages/patches/cdparanoia-fpic.patch \
+ %D%/packages/patches/ceph-disable-cpu-optimizations.patch \
+ %D%/packages/patches/ceph-skip-collect-sys-info-test.patch \
+ %D%/packages/patches/ceph-skip-unittest_blockdev.patch \
%D%/packages/patches/chicken-CVE-2017-6949.patch \
%D%/packages/patches/chmlib-inttypes.patch \
%D%/packages/patches/clang-libc-search-path.patch \
diff --git a/gnu/packages/patches/ceph-disable-cpu-optimizations.patch b/gnu/packages/patches/ceph-disable-cpu-optimizations.patch
new file mode 100644
index 000000000..88a16dde0
--- /dev/null
+++ b/gnu/packages/patches/ceph-disable-cpu-optimizations.patch
@@ -0,0 +1,37 @@
+Disable CPU optimizations not supported by the vast majority of
+x86_64 systems. Also don't add anything for i686.
+
+--- a/cmake/modules/SIMDExt.cmake 2017-03-23 22:22:58.254071694 +0100
++++ b/cmake/modules/SIMDExt.cmake 2017-03-23 22:23:22.446848845 +0100
+@@ -6,10 +6,6 @@
+ # HAVE_INTEL_SSE
+ # HAVE_INTEL_SSE2
+ # HAVE_INTEL_SSE3
+-# HAVE_INTEL_SSSE3
+-# HAVE_INTEL_PCLMUL
+-# HAVE_INTEL_SSE4_1
+-# HAVE_INTEL_SSE4_2
+ #
+ # SIMD_COMPILE_FLAGS
+ #
+@@ -55,20 +51,4 @@
+ if(HAVE_INTEL_SSE3)
+ set(SIMD_COMPILE_FLAGS "${SIMD_COMPILE_FLAGS} -msse3")
+ endif()
+- CHECK_C_COMPILER_FLAG(-mssse3 HAVE_INTEL_SSSE3)
+- if(HAVE_INTEL_SSSE3)
+- set(SIMD_COMPILE_FLAGS "${SIMD_COMPILE_FLAGS} -mssse3")
+- endif()
+- CHECK_C_COMPILER_FLAG(-mpclmul HAVE_INTEL_PCLMUL)
+- if(HAVE_INTEL_PCLMUL)
+- set(SIMD_COMPILE_FLAGS "${SIMD_COMPILE_FLAGS} -mpclmul")
+- endif()
+- CHECK_C_COMPILER_FLAG(-msse4.1 HAVE_INTEL_SSE4_1)
+- if(HAVE_INTEL_SSE4_1)
+- set(SIMD_COMPILE_FLAGS "${SIMD_COMPILE_FLAGS} -msse4.1")
+- endif()
+- CHECK_C_COMPILER_FLAG(-msse4.2 HAVE_INTEL_SSE4_2)
+- if(HAVE_INTEL_SSE4_2)
+- set(SIMD_COMPILE_FLAGS "${SIMD_COMPILE_FLAGS} -msse4.2")
+- endif()
+ endif()
diff --git a/gnu/packages/patches/ceph-skip-collect-sys-info-test.patch b/gnu/packages/patches/ceph-skip-collect-sys-info-test.patch
new file mode 100644
index 000000000..f21079925
--- /dev/null
+++ b/gnu/packages/patches/ceph-skip-collect-sys-info-test.patch
@@ -0,0 +1,29 @@
+This test fails because /etc/os-release is not available.
+
+diff --git a/src/test/common/test_util.cc b/src/test/common/test_util.cc
+index 288dd37db1..95b6cd4b30 100644
+--- a/src/test/common/test_util.cc
++++ b/src/test/common/test_util.cc
+@@ -31,19 +31,3 @@ TEST(util, unit_to_bytesize)
+
+ ASSERT_EQ(65536ll, unit_to_bytesize(" 64K", &cerr));
+ }
+-
+-#if defined(__linux__)
+-TEST(util, collect_sys_info)
+-{
+- map<string, string> sys_info;
+-
+- CephContext *cct = (new CephContext(CEPH_ENTITY_TYPE_CLIENT))->get();
+- collect_sys_info(&sys_info, cct);
+-
+- ASSERT_TRUE(sys_info.find("distro") != sys_info.end());
+- ASSERT_TRUE(sys_info.find("distro_version") != sys_info.end());
+- ASSERT_TRUE(sys_info.find("distro_description") != sys_info.end());
+-
+- cct->put();
+-}
+-#endif
+--
+2.11.1
+
diff --git a/gnu/packages/patches/ceph-skip-unittest_blockdev.patch b/gnu/packages/patches/ceph-skip-unittest_blockdev.patch
new file mode 100644
index 000000000..43b998486
--- /dev/null
+++ b/gnu/packages/patches/ceph-skip-unittest_blockdev.patch
@@ -0,0 +1,28 @@
+This test tries to walk a sysfs path and hits a null pointer exception.
+
+/tmp/guix-build-ceph-12.0.0.drv-0/ceph-12.0.0/src/test/common/test_blkdev.cc:32: Failure
+Expected: (dir) != (nullptr), actual: NULL vs 8-byte object <00-00 00-00 00-00 00-00>
+
+diff --git a/src/test/common/CMakeLists.txt b/src/test/common/CMakeLists.txt
+index 5172663898..ac84085eaa 100644
+--- a/src/test/common/CMakeLists.txt
++++ b/src/test/common/CMakeLists.txt
+@@ -12,15 +12,6 @@ target_link_libraries(get_command_descriptions
+ ${CMAKE_DL_LIBS}
+ )
+
+-if(HAVE_BLKID)
+-# unittest_blkdev
+-add_executable(unittest_blkdev
+- test_blkdev.cc
+- )
+-add_ceph_unittest(unittest_blkdev ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/unittest_blkdev)
+-target_link_libraries(unittest_blkdev global ${BLKID_LIBRARIES})
+-endif(HAVE_BLKID)
+-
+ # unittest_bloom_filter
+ add_executable(unittest_bloom_filter
+ test_bloom_filter.cc
+--
+2.11.1
+
diff --git a/gnu/packages/storage.scm b/gnu/packages/storage.scm
new file mode 100644
index 000000000..318adb7d2
--- /dev/null
+++ b/gnu/packages/storage.scm
@@ -0,0 +1,352 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2017 Marius Bakke <mbakke@fastmail.com>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+
+(define-module (gnu packages storage)
+ #:use-module (guix download)
+ #:use-module ((guix licenses) #:prefix license:)
+ #:use-module (guix packages)
+ #:use-module (guix utils)
+ #:use-module (guix build-system cmake)
+ #:use-module (gnu packages)
+ #:use-module (gnu packages admin)
+ #:use-module (gnu packages assembly)
+ #:use-module (gnu packages bdw-gc)
+ #:use-module (gnu packages boost)
+ #:use-module (gnu packages compression)
+ #:use-module (gnu packages crypto)
+ #:use-module (gnu packages cryptsetup)
+ #:use-module (gnu packages curl)
+ #:use-module (gnu packages databases)
+ #:use-module (gnu packages disk)
+ #:use-module (gnu packages gnuzilla)
+ #:use-module (gnu packages jemalloc)
+ #:use-module (gnu packages linux)
+ #:use-module (gnu packages lua)
+ #:use-module (gnu packages openldap)
+ #:use-module (gnu packages perl)
+ #:use-module (gnu packages pkg-config)
+ #:use-module (gnu packages python)
+ #:use-module (gnu packages tls)
+ #:use-module (gnu packages web)
+ #:use-module (gnu packages xml))
+
+(define-public ceph
+ (package
+ (name "ceph")
+ (version "12.0.0")
+ (source (origin
+ (method url-fetch)
+ (uri (string-append "https://download.ceph.com/tarballs/ceph-"
+ version ".tar.gz"))
+ (sha256
+ (base32
+ "0yzvwlwg85w04q4d9ac73vfpxjnl72dl6kc857ihv5k1lqbpp5d0"))
+ (patches
+ (search-patches "ceph-skip-unittest_blockdev.patch"
+ "ceph-skip-collect-sys-info-test.patch"
+ "ceph-disable-cpu-optimizations.patch"))
+ (modules '((guix build utils)))
+ (snippet
+ '(begin
+ (for-each delete-file-recursively
+ '(;; TODO: Unbundle these:
+ ;"src/isa-l"
+ ;"src/lua"
+ ;"src/googletest"
+ ;"src/xxHash"
+ ;"src/zstd"
+ ;"src/civetweb"
+ ;"src/Beast"
+ "src/test/downloads"
+ "src/dpdk"
+ "src/spdk"
+ "src/rocksdb"
+ "src/boost"))
+ #t))))
+ (build-system cmake-build-system)
+ (arguments
+ `(#:modules ((srfi srfi-1)
+ (guix build cmake-build-system)
+ (guix build utils))
+ #:configure-flags
+ (let* ((out (assoc-ref %outputs "out"))
+ (lib (assoc-ref %outputs "lib"))
+ (libdir (string-append lib "/lib")))
+ (list (string-append "-DCMAKE_INSTALL_PREFIX=" out)
+ (string-append "-DCMAKE_INSTALL_LIBDIR=" libdir)
+ ;; We need both libdir and libdir/ceph in RUNPATH.
+ (string-append "-DCMAKE_INSTALL_RPATH="
+ libdir ";" libdir "/ceph")
+ (string-append "-DCMAKE_INSTALL_SYSCONFDIR=" out "/etc")
+ (string-append "-DCMAKE_INSTALL_MANDIR=" out "/share/man")
+ (string-append "-DCMAKE_INSTALL_DOCDIR=" out "/share/ceph/doc")
+ (string-append "-DCMAKE_INSTALL_LIBEXECDIR=" out "/libexec")
+ (string-append "-DKEYUTILS_INCLUDE_DIR="
+ (assoc-ref %build-inputs "keyutils") "/include")
+ "-DCMAKE_INSTALL_LOCALSTATEDIR=/var"
+ "-DENABLE_SHARED=ON"
+ "-DWITH_EMBEDDED=OFF"
+ "-DWITH_SYSTEM_ROCKSDB=ON"
+ "-DWITH_SYSTEM_BOOST=ON"
+ "-DWITH_PYTHON3=ON"
+ ;; TODO: Enable these when available in Guix.
+ "-DWITH_LTTNG=OFF"
+ "-DWITH_XFS=OFF"
+ "-DWITH_XIO=OFF"
+ ;; Use jemalloc instead of tcmalloc.
+ "-DALLOCATOR=jemalloc"))
+ #:phases
+ (modify-phases %standard-phases
+ (add-after 'unpack 'patch-source
+ (lambda* (#:key outputs #:allow-other-keys)
+ (let ((out (assoc-ref outputs "out"))
+ (lib (assoc-ref outputs "lib")))
+
+ ;; Make header files follow the dynamic libraries.
+ (substitute* "src/include/CMakeLists.txt"
+ (("DESTINATION include")
+ (string-append "DESTINATION " lib "/include")))
+
+ (substitute* "cmake/modules/Distutils.cmake"
+ ;; Prevent creation of Python eggs.
+ (("setup.py install")
+ "setup.py install --single-version-externally-managed --root=/"))
+
+ (substitute* (find-files "src/pybind" "^setup\\.py$")
+ ;; Here we inject an extra line to the `setup.py' of the
+ ;; Python C libraries so RUNPATH gets set up correctly.
+ (("^([[:blank:]]+)extra_compile_args=(.*)$" _ indent args)
+ (string-append indent "extra_compile_args=" args
+ indent "extra_link_args=['-Wl,-rpath="
+ lib "/lib'],\n")))
+
+ (substitute* "src/ceph-disk/tox.ini"
+ ;; Disable flake8 test since it complains
+ ;; about too long lines.
+ (("envlist = flake8,py27")
+ "envlist = py27"))
+
+ (substitute* "src/ceph-detect-init/tox.ini"
+ ;; Disable python3 tests until we at least
+ ;; get py2 working.
+ (("envlist = pep8,py27,py3")
+ "envlist = pep8,py27"))
+
+ (substitute* "src/key_value_store/kv_flat_btree_async.cc"
+ (("/usr/include/") ""))
+
+ (substitute* "src/test/test_subprocess.cc"
+ (("/bin/sh") (which "sh")))
+ (substitute* "src/test/ceph_objectstore_tool.py"
+ (("/bin/rm") (which "rm")))
+ (substitute* "src/ceph-disk/ceph_disk/main.py"
+ (("/bin/mount") "mount")
+ (("/bin/umount") "umount")
+ (("/sbin/blkid") (which "blkid"))
+ (("'sgdisk'") (string-append "'" (which "sgdisk") "'"))
+ (("'parted'") (string-append "'" (which "parted") "'"))
+ (("'udevadm'") (string-append "'" (which "udevadm") "'")))
+
+ (substitute* "src/ceph-disk-udev"
+ (("/sbin/cryptsetup") (which "cryptsetup"))
+ (("/usr/sbin/sgdisk") (which "sgdisk"))
+ (("/usr/sbin/ceph-disk")
+ (string-append out "/sbin/ceph-disk")))
+ (substitute* "udev/50-rbd.rules"
+ (("/usr/bin/ceph-rbdnamer")
+ (string-append out "/bin/ceph-rbdnamer")))
+ (substitute* "udev/60-ceph-by-parttypeuuid.rules"
+ (("/sbin/blkid") (which "blkid")))
+ (substitute* "udev/95-ceph-osd.rules"
+ (("/usr/sbin/ceph-disk")
+ (string-append out "/sbin/ceph-disk")))
+
+ (substitute* "src/test/run-cli-tests"
+ ;; Use our python-cram instead of the (un)bundled one.
+ (("CRAM_BIN=.*$")
+ (string-append "CRAM_BIN=" (which "cram") "\n")))
+
+ ;; Disable tests that are known to fail.
+ ;; TODO: The majority of these fail because
+ ;; 'qa/workunits/ceph-helpers.sh' expects to find
+ ;; /tmp/ceph-disk-virtualenv/bin/ceph-disk, but somehow
+ ;; src/ceph-disk/CMakeLists.txt fails to create it.
+ (substitute* "src/test/CMakeLists.txt"
+ ;; FIXME: "create cannot load compressor of type zlib"
+ ;; "libceph_zlib.so: undefined symbol: isal_deflate"
+ (("^add_subdirectory\\(compressor\\)") "")
+ ;; FIXME: These tests fails because `ceph-disk'
+ ;; is not available.
+ (("^add_ceph_test\\(test-ceph-helpers\\.sh.*$") "\n")
+ (("^add_ceph_test\\(test_pidfile\\.sh.*$") "\n")
+ ;; XXX Why does this fail.
+ (("^add_ceph_test\\(cephtool-test-mon\\.sh.*$") "\n")
+ ;; This fails due to missing '/etc/fstab'.
+ (("^add_ceph_test\\(cephtool-test-rados\\.sh.*$") "\n")
+ ;; `Bad messages to stderr: OSD has the store locked'
+ (("^add_ceph_test\\(ceph_objectstore_tool\\.py.*$") "\n")
+ ;; The bundled python-cram fork needs patching to work on
+ ;; guix, and the system version does not support --error-dir.
+ ;; https://bitbucket.org/brodie/cram/issues/9
+ (("^add_ceph_test\\(run-cli-tests.*$") "\n")
+ ;; FIXME: tox/virtualenv/pip does not discover the
+ ;; required packages and tries to go online.
+ (("^add_test\\(NAME run-tox-ceph-disk.*$") "\n")
+ (("^add_test\\(NAME run-tox-ceph-detect-init.*$") "\n")
+ ;; Also remove from the set_property block.
+ (("run-tox-ceph-disk") "")
+ (("run-tox-ceph-detect-init") ""))
+ ;; More 'ceph-disk' issues here.. :-(
+ (substitute* "src/test/erasure-code/CMakeLists.txt"
+ (("^add_ceph_test\\(test-erasure-code-plugins\\.sh.*$") "\n")
+ (("^add_ceph_test\\(test-erasure-code\\.sh.*$") "\n")
+ (("^add_ceph_test\\(test-erasure-eio\\.sh.*$") "\n"))
+ (substitute* "src/test/libradosstriper/CMakeLists.txt"
+ (("^add_ceph_test\\(rados-striper\\.sh.*$") "\n"))
+ (substitute* "src/test/mon/CMakeLists.txt"
+ (("^add_ceph_test\\(osd-crush\\.sh.*$") "\n")
+ (("^add_ceph_test\\(test_pool_quota\\.sh.*$") "\n")
+ (("^add_ceph_test\\(osd-pool-create\\.sh.*$") "\n"))
+ (substitute* "src/test/osd/CMakeLists.txt"
+ (("^add_ceph_test\\(osd-bench\\.sh.*$") "\n")
+ (("^add_ceph_test\\(osd-config\\.sh.*$") "\n")
+ (("^add_ceph_test\\(osd-markdown\\.sh.*$") "\n")
+ (("^add_ceph_test\\(osd-reactivate\\.sh.*$") "\n")
+ (("^add_ceph_test\\(osd-reuse-id\\.sh.*$") "\n")
+ (("^add_ceph_test\\(osd-scrub-repair\\.sh.*$") "\n")
+ (("^add_ceph_test\\(osd-scrub-snaps\\.sh.*$") "\n")
+ (("^add_ceph_test\\(osd-copy-from\\.sh.*$") "\n")
+ (("^add_ceph_test\\(osd-fast-mark-down\\.sh.*$") "\n"))
+ #t)))
+ (add-before 'check 'set-check-environment
+ (lambda _
+ ;; Run tests in parallel.
+ (setenv "CTEST_PARALLEL_LEVEL"
+ (number->string (parallel-job-count)))
+ ;; `pip' requires write access in $HOME.
+ (setenv "HOME" "/tmp")
+ #t))
+ (add-before 'install 'set-install-environment
+ (lambda* (#:key outputs #:allow-other-keys)
+ (let* ((out (assoc-ref outputs "out"))
+ (py2sitedir
+ (string-append out "/lib/python2.7/site-packages"))
+ (py3sitedir
+ (string-append out "/lib/python"
+ ,(version-major+minor
+ (package-version python))
+ "/site-packages")))
+ ;; The Python install scripts refuses to function if
+ ;; the install directory is not on PYTHONPATH.
+ (setenv "PYTHONPATH"
+ (string-append py2sitedir ":" py3sitedir ":"
+ (getenv "PYTHONPATH")))
+ #t)))
+ (add-after 'install 'wrap-python-scripts
+ (lambda* (#:key outputs #:allow-other-keys)
+ (let* ((out (assoc-ref outputs "out"))
+ (ceph-disk (string-append out "/bin/ceph-disk"))
+ (ceph-detect-init (string-append
+ out "/bin/ceph-detect-init"))
+ (PYTHONPATH (string-append
+ out "/lib/python2.7/site-packages")))
+
+ (for-each (lambda (executable)
+ (wrap-program executable
+ `("PYTHONPATH" ":" prefix (,PYTHONPATH))))
+ (list ceph-disk ceph-detect-init))
+ #t))))))
+ (outputs
+ '("out" "lib"))
+ (native-inputs
+ `(("pkg-config" ,pkg-config)
+ ("python-cython" ,python-cython)
+ ("python-sphinx" ,python-sphinx)
+ ("yasm" ,yasm)
+
+ ;; For tests.
+ ("inetutils" ,inetutils)
+ ("jq" ,jq)
+ ("perl" ,perl)
+ ("xmlstarlet" ,xmlstarlet)
+ ("python2-cram" ,python2-cram)
+ ("python2-virtualenv" ,python2-virtualenv)
+
+ ;; These dependencies are taken from test-requirements.txt
+ ;; of ceph-disk and ceph-detect-init. The latter can also
+ ;; test agains
