[PATCH] gnu: Add leela-zero.

  • Done
  • quality assurance status badge
Details
5 participants
  • Guillaume LE VAILLANT
  • Ludovic Courtès
  • Tobias Geerinckx-Rice
  • Ricardo Wurmus
  • zimoun
Owner
unassigned
Submitted by
Guillaume LE VAILLANT
Severity
normal
G
G
Guillaume LE VAILLANT wrote on 3 Jun 2019 15:06
(address . guix-patches@gnu.org)(name . Guillaume LE VAILLANT)(address . glv@posteo.net)
20190603130611.32142-1-glv@posteo.net
* gnu/packages/games.scm (leela-zero): New variable.
* gnu/packages/patches/leela-zero-gtest.patch: New file.
* gnu/local.mk (dist_patch_DATA): Add it.
---
gnu/local.mk | 2 +
gnu/packages/games.scm | 51 +++++++++++++++++++++
gnu/packages/patches/leela-zero-gtest.patch | 40 ++++++++++++++++
3 files changed, 93 insertions(+)
create mode 100644 gnu/packages/patches/leela-zero-gtest.patch

Toggle diff (141 lines)
diff --git a/gnu/local.mk b/gnu/local.mk
index 55a8fcd361..19cab48aff 100644
--- a/gnu/local.mk
+++ b/gnu/local.mk
@@ -20,6 +20,7 @@
# Copyright © 2018 Oleg Pykhalov <go.wigust@gmail.com>
# Copyright © 2018 Stefan Stefanovi? <stefanx2ovic@gmail.com>
# Copyright © 2018 Maxim Cournoyer <maxim.cournoyer@gmail.com>
+# Copyright © 2019 Guillaume Le Vaillant <glv@posteo.net>
#
# This file is part of GNU Guix.
#
@@ -969,6 +970,7 @@ dist_patch_DATA = \
%D%/packages/patches/lcms-CVE-2018-16435.patch \
%D%/packages/patches/ldc-bootstrap-disable-tests.patch \
%D%/packages/patches/ldc-disable-phobos-tests.patch \
+ %D%/packages/patches/leela-zero-gtest.patch \
%D%/packages/patches/liba52-enable-pic.patch \
%D%/packages/patches/liba52-link-with-libm.patch \
%D%/packages/patches/liba52-set-soname.patch \
diff --git a/gnu/packages/games.scm b/gnu/packages/games.scm
index 4fdc9b01e6..ee8cc69273 100644
--- a/gnu/packages/games.scm
+++ b/gnu/packages/games.scm
@@ -40,6 +40,7 @@
;;; Copyright © 2019 Oleg Pykhalov <go.wigust@gmail.com>
;;; Copyright © 2019 Pierre Langlois <pierre.langlois@gmx.com>
;;; Copyright © 2019 Julien Lepiller <julien@lepiller.eu>
+;;; Copyright © 2019 Guillaume Le Vaillant <glv@posteo.net>
;;;
;;; This file is part of GNU Guix.
;;;
@@ -128,6 +129,7 @@
#:use-module (gnu packages netpbm)
#:use-module (gnu packages networking)
#:use-module (gnu packages ocaml)
+ #:use-module (gnu packages opencl)
#:use-module (gnu packages pcre)
#:use-module (gnu packages perl)
#:use-module (gnu packages perl-check)
@@ -7336,3 +7338,52 @@ Unfortunately, Hacker is not aware of Drascula's real ambitions: DOMINATING
the World and demonstrating that he is even more evil than his brother Vlad.")
;; Drascula uses a BSD-like license.
(license (license:non-copyleft "file:///readme.txt"))))
+
+(define-public leela-zero
+ (package
+ (name "leela-zero")
+ (version "0.17")
+ (source
+ (origin
+ (method git-fetch)
+ (uri (git-reference
+ (url "https://github.com/leela-zero/leela-zero.git")
+ (commit (string-append "v" version))))
+ (file-name (git-file-name name version))
+ (sha256
+ (base32
+ "17px5iny8mql5c01bymcli7zfssswkzvb2i8gnsmjcck6i2n8srl"))
+ (patches (search-patches "leela-zero-gtest.patch"))))
+ (build-system cmake-build-system)
+ (native-inputs
+ `(("googletest" ,googletest)))
+ (inputs
+ `(("boost" ,boost)
+ ("ocl-icd" ,ocl-icd)
+ ("openblas" ,openblas)
+ ("opencl-headers" ,opencl-headers)
+ ("qtbase" ,qtbase)
+ ("zlib" ,zlib)))
+ (arguments
+ '(#:configure-flags '("-DUSE_BLAS=YES")
+ #:phases (modify-phases %standard-phases
+ (add-before 'configure 'fix-tests
+ (lambda* (#:key outputs #:allow-other-keys)
+ (let ((home (getcwd)))
+ (setenv "HOME" home)
+ (substitute* "src/tests/gtests.cpp"
+ (("\\.\\./src/tests/0k\\.txt")
+ (string-append home "/src/tests/0k.txt"))
+ (("cfg_gtp_mode = true;")
+ "cfg_gtp_mode = true; cfg_cpu_only = true;")))
+ #t))
+ (replace 'check
+ (lambda _
+ (zero? (system* "./tests")))))))
+ (home-page "https://github.com/leela-zero/leela-zero")
+ (synopsis "Program playing the game of Go")
+ (description
+ "Leela-zero is a Go engine with no human-provided knowledge, modeled
+after the AlphaGo Zero paper. The current best network weights file for
+the engine can be downloaded from https://zero.sjeng.org/best-network.")
+ (license license:gpl3+)))
diff --git a/gnu/packages/patches/leela-zero-gtest.patch b/gnu/packages/patches/leela-zero-gtest.patch
new file mode 100644
index 0000000000..7291b4ab72
--- /dev/null
+++ b/gnu/packages/patches/leela-zero-gtest.patch
@@ -0,0 +1,40 @@
+Use shared gtest instead of submodule
+
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 617e4b2..2857875 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -18,7 +18,6 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/")
+ include(GNUInstallDirs)
+
+ project(leelaz)
+-add_subdirectory(gtest EXCLUDE_FROM_ALL) # We don't want to install gtest, exclude it from `all`
+
+ # Required Packages
+ set(Boost_MIN_VERSION "1.58.0")
+@@ -27,6 +26,7 @@ find_package(Boost 1.58.0 REQUIRED program_options filesystem)
+ find_package(Threads REQUIRED)
+ find_package(ZLIB REQUIRED)
+ find_package(OpenCL REQUIRED)
++find_package(GTest REQUIRED)
+ # We need OpenBLAS for now, because we make some specific
+ # calls. Ideally we'd use OpenBLAS is possible and fall back to
+ # not doing those calls if it's not present.
+@@ -93,6 +93,7 @@ include_directories(${IncludePath})
+ include_directories(${Boost_INCLUDE_DIRS})
+ include_directories(${OpenCL_INCLUDE_DIRS})
+ include_directories(${ZLIB_INCLUDE_DIRS})
++include_directories(${GTEST_INCLUDE_DIRS})
+
+ if((UNIX AND NOT APPLE) OR WIN32)
+ include_directories(${BLAS_INCLUDE_DIRS})
+@@ -141,7 +142,8 @@ target_link_libraries(tests ${Boost_LIBRARIES})
+ target_link_libraries(tests ${BLAS_LIBRARIES})
+ target_link_libraries(tests ${OpenCL_LIBRARIES})
+ target_link_libraries(tests ${ZLIB_LIBRARIES})
+-target_link_libraries(tests gtest_main ${CMAKE_THREAD_LIBS_INIT})
++target_link_libraries(tests ${GTEST_BOTH_LIBRARIES})
++target_link_libraries(tests ${CMAKE_THREAD_LIBS_INIT})
+
+ include(GetGitRevisionDescription)
+ git_describe(VERSION --tags)
--
2.21.0
L
L
Ludovic Courtès wrote on 5 Jun 2019 17:24
(name . Guillaume LE VAILLANT)(address . glv@posteo.net)(address . 36071@debbugs.gnu.org)
87v9xkyrzz.fsf@gnu.org
Hi,

Guillaume LE VAILLANT <glv@posteo.net> skribis:

Toggle quote (4 lines)
> * gnu/packages/games.scm (leela-zero): New variable.
> * gnu/packages/patches/leela-zero-gtest.patch: New file.
> * gnu/local.mk (dist_patch_DATA): Add it.

[...]

Toggle quote (4 lines)
> + (replace 'check
> + (lambda _
> + (zero? (system* "./tests")))))))

Please use ‘invoke’ instead of (zero? (system* …)).

Toggle quote (3 lines)
> + "Leela-zero is a Go engine with no human-provided knowledge, modeled
> +after the AlphaGo Zero paper. The current best network weights file for
> +the engine can be downloaded from https://zero.sjeng.org/best-network.")
^~~~
Please use @url for URLs.

Does the game run at all if we don’t download additional network weight
data?

There’s a debate as to whether trained neural network parameters can be
considered “source”: https://lwn.net/Articles/760142/.

Thanks,
Ludo’.
G
G
Guillaume LE VAILLANT wrote on 5 Jun 2019 19:26
(name . Ludovic Courtès)(address . ludo@gnu.org)(address . 36071@debbugs.gnu.org)
20190605192625.3c567ce7@yamatai
Hi,

Ludovic Courtès <ludo@gnu.org> a écrit :

Toggle quote (6 lines)
> Does the game run at all if we don’t download additional network
> weight data?
>
> There’s a debate as to whether trained neural network parameters can
> be considered “source”: <https://lwn.net/Articles/760142/>.

The game needs neural network weights to run. The user must download
a file with the weights (usually from
https://zero.sjeng.org/best-network, which is updated daily I think)
and put it in '$HOME/.local/share/leela-zero/'.

Do you think some weigths (e.g. taken from
https://leela.online-go.com/networks, for reproducibility) should be
embedded in the package?
Attachment: 0001-gnu-Add-leela-zero.patch
L
L
Ludovic Courtès wrote on 5 Jun 2019 22:47
(name . Guillaume LE VAILLANT)(address . glv@posteo.net)(address . 36071@debbugs.gnu.org)
87tvd3wyhx.fsf@gnu.org
Hi Guillaume,

Guillaume LE VAILLANT <glv@posteo.net> skribis:

Toggle quote (17 lines)
> Ludovic Courtès <ludo@gnu.org> a écrit :
>
>> Does the game run at all if we don’t download additional network
>> weight data?
>>
>> There’s a debate as to whether trained neural network parameters can
>> be considered “source”: <https://lwn.net/Articles/760142/>.
>
> The game needs neural network weights to run. The user must download
> a file with the weights (usually from
> <https://zero.sjeng.org/best-network>, which is updated daily I think)
> and put it in '$HOME/.local/share/leela-zero/'.
>
> Do you think some weigths (e.g. taken from
> <https://leela.online-go.com/networks>, for reproducibility) should be
> embedded in the package?

My point is about whether these trained neural network data are
something that we could distribute per the FSDG.

The LWN article above mentions ‘leela-zero’. In Debian (which has a
different policy), ‘leela-zero’ has apparently been accepted, but
without the data, and with a link to said data (as in your original
patch):


Perhaps we could do the same, but I’d like to hear what others think.

Thank you,
Ludo’.
Z
Z
zimoun wrote on 6 Jun 2019 12:38
(name . Ludovic Courtès)(address . ludo@gnu.org)
CAJ3okZ3VS9viHPhJbFu3JzYw9XRO+EW-YT9Bmx4FO-pbS5+inw@mail.gmail.com
Hi,

Thank you for the package.

On Wed, 5 Jun 2019 at 23:03, Ludovic Courtès <ludo@gnu.org> wrote:

Toggle quote (9 lines)
> > Ludovic Courtès <ludo@gnu.org> a écrit :
> >
> >> Does the game run at all if we don’t download additional network
> >> weight data?
> >>
> >> There’s a debate as to whether trained neural network parameters can
> >> be considered “source”: <https://lwn.net/Articles/760142/>.
> >

[...]

Toggle quote (3 lines)
> My point is about whether these trained neural network data are
> something that we could distribute per the FSDG.

[...]

Toggle quote (2 lines)
In addition, the thread on debian-devel exposes arguments (summarized by LWN).
See there:


Toggle quote (2 lines)
> Perhaps we could do the same, but I’d like to hear what others think.

I need myself to digest the issue.
Hum? FSF has never risen the issue---or I am not aware of.

Aside the practical discussion about this package---thank you for the
work!---, there is a real concern about Reproducibility and Freedom
with the weights of statistical models. However, this is perhaps not
so far from free client / proprietary database situation.

Well, it should be better to discuss this concern on
guix-devel---risking bikeshedding. :-)
At least to collect contradictory arguments.


All the best,
simon
R
R
Ricardo Wurmus wrote on 11 Jun 2019 22:55
(name . Ludovic Courtès)(address . ludo@gnu.org)
87lfy7alke.fsf@elephly.net
Ludovic Courtès <ludo@gnu.org> writes:

Toggle quote (29 lines)
>> Ludovic Courtès <ludo@gnu.org> a écrit :
>>
>>> Does the game run at all if we don’t download additional network
>>> weight data?
>>>
>>> There’s a debate as to whether trained neural network parameters can
>>> be considered “source”: <https://lwn.net/Articles/760142/>.
>>
>> The game needs neural network weights to run. The user must download
>> a file with the weights (usually from
>> <https://zero.sjeng.org/best-network>, which is updated daily I think)
>> and put it in '$HOME/.local/share/leela-zero/'.
>>
>> Do you think some weigths (e.g. taken from
>> <https://leela.online-go.com/networks>, for reproducibility) should be
>> embedded in the package?
>
> My point is about whether these trained neural network data are
> something that we could distribute per the FSDG.
>
> The LWN article above mentions ‘leela-zero’. In Debian (which has a
> different policy), ‘leela-zero’ has apparently been accepted, but
> without the data, and with a link to said data (as in your original
> patch):
>
> https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=903634
>
> Perhaps we could do the same, but I’d like to hear what others think.

We need to ensure that the software necessary to train the networks is
included. Is this the case?

Distributing the training data is not an option, as I see it, because
the files are very large.

I don’t know about the networks that are offered for download at
different sizes, and it is neither clear what license applies nor how
they were arrived at. They are zipped up magic numbers.

Arguably a similar statement could be made about impulse response files
for audio filters (such as convolution reverbs). While we offer
convolution filters (such as the “ir” package), we don’t include data
files. Had I not made the link to network weights for ML models I would
have considered impulse response files no different from e.g. game
assets. Now I’m not so sure any more.

I don’t really know what it really means for data to be
“non-functional”.

Back to this patch: I think it’s fine to accept it as long as the
software necessary for training is included.

--
Ricardo
T
T
Tobias Geerinckx-Rice wrote on 12 Jun 2019 00:54
(name . Ricardo Wurmus)(address . rekado@elephly.net)
87h88vens4.fsf@nckx
Ricardo Wurmus wrote:
Toggle quote (4 lines)
> Back to this patch: I think it’s fine to accept it as long as
> the
> software necessary for training is included.

I can't assess its accuracy, but this comment[0] from the linked
LWN article looks relevant. Particularly:

Toggle snippet (6 lines)
So this is basically a distributed computing client as well as a
Go
engine that runs with the results of that distributed computing
effort.

If that's true, there is no separate ‘training software’ to worry
about.

The whole link is worth a click since there seems to be a ‘server
component’ involved as well.

Kind regards,

T G-R

-----BEGIN PGP SIGNATURE-----

iHUEARYKAB0WIQT12iAyS4c9C3o4dnINsP+IT1VteQUCXQAxGwAKCRANsP+IT1Vt
efdwAP0Yywgm/Af9Sy671i1pW6uM5AmRE/HyS6EE0yzzqZxTYwD/WSnbsyxVO7QE
kRsiE8gT6CeYLNitCCfo/okRuCGQnQE=
=dBAh
-----END PGP SIGNATURE-----

L
L
Ludovic Courtès wrote on 25 Jun 2019 17:12
(name . Ricardo Wurmus)(address . rekado@elephly.net)
87lfxplmwu.fsf@gnu.org
Hello!

Ricardo Wurmus <rekado@elephly.net> skribis:

Toggle quote (3 lines)
> Back to this patch: I think it’s fine to accept it as long as the
> software necessary for training is included.

Tobias Geerinckx-Rice <me@tobias.gr> skribis:

Toggle quote (14 lines)
> I can't assess its accuracy, but this comment[0] from the linked
> LWN article looks relevant. Particularly:
>
> So this is basically a distributed computing client as well as a
> Go
> engine that runs with the results of that distributed computing
> effort.
>
> If that's true, there is no separate ‘training software’ to worry
> about.
>
> The whole link is worth a click since there seems to be a ‘server
> component’ involved as well.

Based on the info we have so far, including the fact that Debian
includes it (see https://packages.debian.org/leela-zero), I went ahead
and applied the patch.

We can obviously revisit this issue later should someone have concerns
and/or additional info.

Thanks everyone!

Ludo’.
Closed
?
Your comment

This issue is archived.

To comment on this conversation send an email to 36071@debbugs.gnu.org

To respond to this issue using the mumi CLI, first switch to it
mumi current 36071
Then, you may apply the latest patchset in this issue (with sign off)
mumi am -- -s
Or, compose a reply to this issue
mumi compose
Or, send patches to this issue
mumi send-email *.patch