path: root/posts/2015-08-30-ruby-on-guix.skr

(post
 :title "Ruby on Guix"
 :date (make-date* 2015 08 30)
 :tags '("gnu" "guix" "scheme" "guile" "ruby" "wsu")
 :summary "How to use Guix + some elbow grease to replace RVM and
Bundler on GNU/Linux"

 (p [I’ve been working with Ruby professionally for over 3 years now
and I’ve grown frustrated with two of its most popular development
tools: RVM and Bundler.  For those that may not know, RVM is the Ruby
version manager and it allows unprivileged users to download, compile,
install, and manage many versions of Ruby instead of being stuck with
the one that is installed globally by your distro’s package manager.
Bundler is the tool that allows developers to keep a version
controlled “Gemfile” that specifies all of the project’s dependencies
and provides utilities to install and update those gems.  These tools
are crucial because Ruby developers often work with many applications
that use different versions of Ruby and/or different versions of gems
such as Rails.  Traditional GNU/Linux distributions install packages
to the global ,(code [/usr]) directory, limiting users to a single
version of Ruby and associated gems, if they are packaged at all.
Traditional package management fails to meet the needs of a lot of
users, so many niche package managers have been developed to
supplement them.])

 (source-code
  (scheme-source
   ";;; guile-syntax-highlight --- General-purpose syntax highlighter
;;; Copyright © 2015 David Thompson <davet@gnu.org>
;;;
;;; Guile-syntax-highlight is free software; you can redistribute it
;;; and/or modify it under the terms of the GNU Lesser General Public
;;; License as published by the Free Software Foundation; either
;;; version 3 of the License, or (at your option) any later version.
;;;
;;; Guile-syntax-highlight is distributed in the hope that it will be
;;; useful, but WITHOUT ANY WARRANTY; without even the implied
;;; warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
;;; See the GNU Lesser General Public License for more details.
;;;
;;; You should have received a copy of the GNU Lesser General Public
;;; License along with guile-syntax-highlight.  If not, see
;;; <http://www.gnu.org/licenses/>.

;;; Commentary:
;;
;; Lexing utilities.
;;
;;; Code:

(define-module (syntax-highlight lexers)
  #:use-module (ice-9 match)
  #:use-module (ice-9 regex)
  #:use-module (srfi srfi-1)
  #:use-module (srfi srfi-9)
  #:use-module (srfi srfi-11)
  #:use-module (srfi srfi-26)
  #:export (make-cursor
            cursor?
            cursor-text
            cursor-position
            cursor-end?
            move-cursor
            move-cursor-by

            lex-fail
            lex-bind
            lex-return
            lex-lift
            lex-map
            lex-filter
            lex-any*
            lex-any
            lex-all*
            lex-all
            lex-consume
            lex-regexp
            lex-string
            lex-char-set
            lex-delimited
            lex-tag))

(define (string-prefix?* s1 s2 start-s2)
  (string-prefix? s1 s2 0 (string-length s1) start-s2))


;;;
;;; Cursor
;;;

(define-record-type <cursor>
  (make-cursor text position)
  cursor?
  (text cursor-text)
  (position cursor-position))

(define (cursor-end? cursor)
  \"Return #t if the cursor is at the end of the text.\"
  (>= (cursor-position cursor) (string-length (cursor-text cursor))))

(define (move-cursor cursor position)
  \"Move CURSOR to the character at POSITION.\"
  (make-cursor (cursor-text cursor) position))

(define (move-cursor-by cursor offset)
  \"Move CURSOR by OFFSET characters relative to its current
position.\"
  (move-cursor cursor (+ (cursor-position cursor) offset)))


;;;
;;; Lexers
;;;

(define (lex-fail cursor)
  \"Always fail to lex STR without consuming any of it.\"
  (values #f cursor))

(define (lex-bind proc lexer)
  \"Return a lexer that applies the result of LEXER to PROC, a
procedure that returns a lexer, and then applies that new lexer.\"
  (lambda (cursor)
    (let-values (((result remainder) (lexer cursor)))
      (if result
          ((proc result) remainder)
          (lex-fail cursor)))))

(define (lex-return x)
  \"Return a lexer that always yields X as the lex result.\"
  (lambda (cursor)
    (values x cursor)))

(define (lex-lift proc)
  \"Return a procedure that wraps the result of PROC in a lexer.\"
  (lambda args
    (lex-return (apply proc args))))

(define (lex-map proc lexer)
  \"Return a lexer that applies PROC to result of LEXER.\"
  (lex-bind (lex-lift proc) lexer))

(define (lex-any* lexers)
  \"Return a lexer that succeeds with the result of the first
successful lexer in LEXERS or fails if all lexers fail.\"
  (define (either a b)
    (lambda (cursor)
      (let-values (((result remainder) (a cursor)))
        (if result
            (values result remainder)
            (b cursor)))))

  (fold-right either lex-fail lexers))

(define (lex-any . lexers)
  \"Return a lexer that succeeds with the result of the first
successful lexer in LEXERS or fails if all lexers fail.\"
  (lex-any* lexers))

(define (lex-all* lexers)
  \"Return a lexer that succeeds with the results of all LEXERS in
order, or fails if any lexer fails.\"
  (define (both a b)
    (lambda (cursor)
      (let-values (((result-a remainder-a) (a cursor)))
        (if result-a
            (let-values (((result-b remainder-b) (b remainder-a)))
              (if result-b
                  (values (cons result-a result-b) remainder-b)
                  (lex-fail cursor)))
            (lex-fail cursor)))))

  (fold-right both (lex-return '()) lexers))

(define (lex-all . lexers)
  \"Return a lexer that succeeds with the results of all LEXERS in
order, or fails if any lexer fails.\"
  (lex-all* lexers))

(define (lex-consume lexer)
  \"Return a lexer that always succeeds with a list of as many
consecutive successful applications of LEXER as possible, consuming
the entire input text.  Sections of text that could not be lexed are
returned as plain strings.\"
  (define (substring* cursor start)
    (substring (cursor-text cursor) start (cursor-position cursor)))

  (lambda (cursor)
    (let loop ((cursor cursor)
               (memo '())
               (fail-start #f))
      (if (cursor-end? cursor)
          (values (reverse memo) cursor)
          (let-values (((result remainder) (lexer cursor)))
            (cond
             ;; Regular successful result.
             ((and result (not fail-start))
              (loop remainder (cons result memo) #f))
             ;; Successful result after some amount of unmatched
             ;; characters.
             ((and result fail-start)
              (loop remainder
                    (cons* result (substring* cursor fail-start) memo)
                    #f))
             ;; Consecutive failure.
             (fail-start
              (loop (move-cursor-by cursor 1)
                    memo
                    fail-start))
             ;; First failure.
             (else
              (loop (move-cursor-by cursor 1)
                    memo
                    (cursor-position cursor)))))))))

(define (lex-regexp pattern)
  \"Return a lexer that succeeds with the matched substring when the
input matches the string PATTERN.\"
  (let ((rx (make-regexp (string-append \"^\" pattern))))
    (lambda (cursor)
      (if (cursor-end? cursor)
          (lex-fail cursor)
          (let ((result (regexp-exec rx (cursor-text cursor)
                                     (cursor-position cursor))))
            (if result
                (let ((str (match:substring result 0)))
                  (values str (move-cursor-by cursor (string-length str))))
                (lex-fail cursor)))))))

(define (lex-string str)
  \"Return a lexer that succeeds with STR when the input starts with
STR.\"
  (lambda (cursor)
    (if (string-prefix?* str (cursor-text cursor) (cursor-position cursor))
        (values str (move-cursor-by cursor (string-length str)))
        (lex-fail cursor))))

(define (lex-char-set char-set)
  \"Return a lexer that succeeds with the nonempty input prefix that
matches CHAR-SET, or fails if the first input character does not
belong to CHAR-SET.\"
  (define (char-set-substring str start)
    (let ((len (string-length str)))
      (let loop ((index start))
        (cond
         ((>= index len)
          (substring str start len))
         ((char-set-contains? char-set (string-ref str index))
          (loop (1+ index)))
         (else
          (substring str start index))))))

  (lambda (cursor)
    (match (char-set-substring (cursor-text cursor) (cursor-position cursor))
      (\"\" (lex-fail cursor))
      (str (values str (move-cursor-by cursor (string-length str)))))))

(define* (lex-delimited open #:key (until open) (escape #\\) nested?)
  \"Return a lexer that succeeds with the string delimited by the
opening string OPEN and the closing string UNTIL.  Characters within
the delimited expression may be escaped with the character ESCAPE.  If
NESTED?, allow for delimited expressions to be arbitrarily nested
within.\"
  (define (delimit str start)
    (let ((len (string-length str)))
      (let loop ((index start))
        (cond
         ;; Out of bounds.
         ((>= index len)
          len)
         ;; Escape character.
         ((eqv? escape (string-ref str index))
          (loop (+ index 2)))
         ;; Closing delimiter.
         ((string-prefix?* until str index)
          (+ index (string-length until)))
         ;; Nested delimited string.
         ((and nested? (string-prefix?* open str index))
          (loop (delimit str (+ index (string-length open)))))
         (else
          (loop (1+ index)))))))

  (lambda (cursor)
    (let ((str (cursor-text cursor))
          (pos (cursor-position cursor)))
      (if (string-prefix?* open str pos)
          (let ((end (delimit str (+ pos (string-length open)))))
            (values (substring str pos end) (move-cursor cursor end)))
          (lex-fail cursor)))))

(define (lex-tag tag lexer)
  \"Wrap the results of LEXER in a two-element tuple whose head is
TAG.\"
  (lex-map (cut list tag <>) lexer))
"))

 (p [Taking a step back, it becomes apparent that dependency isolation
is a general problem that isn’t confined to software written in Ruby:
Node has npm and nvm, Python has pip and virtualenv, and so on.  A big
limitation of all these language-specific package managers is that
they cannot control what is outside of their language domain.  In
order to use RVM to successfully compile a version of Ruby, you need
to make sure you have the GCC toolchain, OpenSSL, readline, libffi,
etc. installed using the system package manager (note: I’ve seen RVM
try to build prerequisites like OpenSSL before, which I then disabled
to avoid duplication and security issues and I recommend you do the
same.)  In order to use Bundler to install Nokogiri, you need to make
sure libxml2 has been installed using the system package manager.  If
you work with more than a single language, the number of different
package management tools needed to get work done is staggering.  For
web applications, it’s not uncommon to use RVM, Bundler, NPM, Bower,
and the system package manager simultaneously to get all of the
necessary programs and libraries.  Large web applications are
notoriously difficult to deploy, and companies hire a bunch of
operations folk like me to try to wrangle it all.])

 (p [Anyway, let’s forget about Node, Python, etc. and just focus on
Ruby.  Have you or someone you work with encountered hard to debug
issues and Git merge conflicts due to a problem with
,(code [Gemfile.lock])?  Bundler’s fast and loose versioning in the
,(code [Gemfile]) (e.g. ,(code [rails >= 4.0])) causes headaches when
different users update different gems at different times and check the
resulting auto-generated ,(code [Gemfile.lock]) into the repository.
Have you ever been frustrated that it’s difficult to deduplicate gems
that are shared between multiple bundled gem sets?  Have you looked at
the ,(anchor [RVM home page] "https://rvm.io") and been frustrated
that they recommend you to ,(code [curl bash]) to install their
software?  Have you been annoyed by RVM’s strange system of overriding
shell built-ins in order to work its magic?  I’m not sure how you
feel, dear reader, but my Ruby environments feel like one giant,
brittle hack, and I’m often enough involved in fixing issues with them
on my own workstation, that of my colleagues, and on production
servers.])

 (p [So, if you’re still with me, what do we do about this?  How can we
work to improve upon the status quo?  Just use Docker?  Docker is
helpful, and certainly much better than no isolation at all, but it
hides the flaws of package management inside an opaque disk image and
restricts the environments in which your application is built to
function.  The general problem of dependency isolation is orthogonal
to the runtime environment, be it container, virtual machine, or “bare
metal.”  Enter functional package management.  What does it mean for a
package manager to be functional?  GNU Guix, the functional package
manager that I contribute to and recommend, has this to say:])

 (blockquote
    (p [GNU Guix is a functional package management tool for the GNU
system.  Package management consists of all activities that relate
to building packages from sources, honoring their build-time and
run-time dependencies, installing packages in user environments,
upgrading installed packages to new versions or rolling back to a
previous set, removing unused software packages, etc.])
    (p [The term functional refers to a specific package management
discipline.  In Guix, the package build and installation process
is seen as a function, in the mathematical sense.  That function
takes inputs, such as build scripts, a compiler, and libraries,
and returns an installed package.]))

 (p [Guix has a rich set of features, some of which you may find in
other package managers, but not all of them (unless you use another
functional package manager such as Nix.)  Gem/Bundler can do
unprivileged gem installation, but it cannot do transactional upgrades
and rollbacks or install non-Ruby dependencies.  Dpkg/yum/pacman can
install all build-time and runtime dependencies, but it cannot do
unprivileged package installation to isolated user environments.  And
none of them can precisely describe the ,(em [full]) dependency
graph (all the way down to the C compiler’s compiler) but ,(em [Guix
can]).])

 (p [Guix is written in Guile, an implementation of the Scheme
programming language.  The upcoming release of Guix will feature a
Ruby build system that captures the process of installing gems from
,(code [.gem]) archives and a RubyGems import utility to make it
easier to write Guix packages by using the metadata available on
,(anchor "RubyGems.org" "https://rubygems.org").  Ruby developers
interested in functional package management are encouraged to try
packaging their gems (and dependencies) for Guix.])

 (p [Now, how exactly can Guix replace RVM and Bundler?  Guix uses an
abstraction called a “profile” that represents a user-defined set of
packages that should work together.  Think of it as having many
,(code [/usr]) file system trees that can be used in isolation from the
others (without invoking virtualization technologies such as virtual
machines or containers.)  To install multiple versions of Ruby and
various gems, the user need only create a separate profile for them:])

 (source-code
  "$ guix package --profile=project-1 --install ruby-2.2 ruby-rspec-3
# Hypothetical packages:
$ guix package --profile=project-2 --install ruby-1.9 ruby-rspec-2")

 (p [A profile is a “symlink forest” that is the union of all the
packages it includes, and files are deduplicated among all of them.
To actually use the profile, the relevant environment variables must
be configured.  Guix is aware of such variables, and can tell you what
to set by running the following:])

 (source-code
  "$ guix package --search-paths --profile=project-1")

 (p [Additionally, you can also create ad-hoc development environments
with the ,(code [guix environment]) tool.  This tool will spawn a
sub-shell (or another program of your choice) in an environment in
which a set of specified packages are available.  This is my preferred
method as it automagically sets all of the environment variables for
me and Guix is free to garbage collect the packages when I close the
sub-shell:])

 (source-code
  "# Launch a Ruby REPL with ActiveSupport available.
$ guix environment --ad-hoc ruby ruby-activesupport -E irb")

 (p [In order to make this environment reproducible for others, I
recommend keeping a ,(code [package.scm]) file in version control that
describes the complete dependency graph for your project, as well as
metadata such as the license, version, and description:])

 (source-code
  (scheme-source
   "(use-modules (guix packages)
             (guix licenses)
             (guix build-system ruby)
             (gnu packages)
             (gnu packages version-control)
             (gnu packages ssh)
             (gnu packages ruby))

(package
  (name \"cool-ruby-project\")
  (version \"1.0\")
  (source #f) ; not needed just to create dev environment
  (build-system ruby-build-system)
  ;; These correspond roughly to \"development\" dependencies.
  (native-inputs
   `((\"git\" ,git)
     (\"openssh\" ,openssh)
     (\"ruby-rspec\" ,ruby-rspec)))
  (propagated-inputs
   `((\"ruby-pg\" ,ruby-pg)
     (\"ruby-nokogiri\" ,ruby-nokogiri)
     (\"ruby-i18n\" ,ruby-i18n)
     (\"ruby-rails\" ,ruby-rails)))
  (synopsis \"A cool Ruby project\")
  (description \"This software does some cool stuff, trust me.\")
  (home-page \"https://example.com\")
  (license expat))"))

 (p [With this package file, it is simple to an instantiate a
development environment:])

 (pre (code [$ guix environment -l package.scm]))

  (p [I’m not covering it in this post, but properly filling out the
blank ,(code [source]) field above would allow for building
development snapshots, including running the test suite, in an
isolated build container using the ,(code [guix build]) utility.  This
is very useful when composed with a continuous integration system.
Guix itself uses ,(anchor "Hydra" "https://nixos.org/hydra/") as its
CI system to perform all package builds.])

  (p [As mentioned earlier, one of the big advantages of writing Guix
package recipes is that the full dependency graph can be captured,
including non-Ruby components.  The pg gem provides a good example:])

  (source-code
   (scheme-source
"(define-public ruby-pg
  (package
    (name \"ruby-pg\")
    (version \"0.18.2\")
    (source
     (origin
       (method url-fetch)
       (uri (rubygems-uri \"pg\" version))
       (sha256
        (base32
         \"1axxbf6ij1iqi3i1r3asvjc80b0py5bz0m2wy5kdi5xkrpr82kpf\"))))
    (build-system ruby-build-system)
    (arguments
     '(#:test-target \"spec\"))
    ;; Native inputs are used only at build and test time.
    (native-inputs
     `((\"ruby-rake-compiler\" ,ruby-rake-compiler)
       (\"ruby-hoe\" ,ruby-hoe)
       (\"ruby-rspec\" ,ruby-rspec)))
    ;; Native extension links against PostgreSQL shared library.
    (inputs
     `((\"postgresql\" ,postgresql)))
    (synopsis \"Ruby interface to PostgreSQL\")
    (description \"Pg is the Ruby interface to the PostgreSQL RDBMS.  It works
  with PostgreSQL 8.4 and later.\")
     (home-page \"https://bitbucket.org/ged/ruby-pg\")
     (license license:ruby)))"))

  (p [Note how the recipe specifies the PostgreSQL dependency.  Below
is the dependency graph for ruby-pg as produced by ,(code [guix
graph]), excluding the GCC compiler toolchain and other low-level
tools for brevity.  Pretty neat, eh?])

  (image/caption "/images/ruby-pg-graph.png"
                 "Abbreviated dependency graph for the pg gem")

  (p [Given that Guix doesn’t yet have many gems packaged (help
wanted), it can still be advantageous to use it for getting more
up-to-date packages than many distros provide, but in conjuction with
Bundler for fetching Ruby gems.  This gets RVM out of your hair whilst
creating a migration path away from Bundler at a later time once the
required gems have been packaged:])

  (source-code
   "$ cd my-project/
$ guix environment --ad-hoc ruby bundler libxml2 libxslt # etc.
# A small bash script can be used to make these gem sets.
$ mkdir .gems
$ export GEM_HOME=$PWD/.gems
$ export GEM_PATH=$GEM_HOME:$GEM_PATH
$ export PATH=$GEM_HOME/bin:$PATH
$ bundle install")

  (p [As you’ve seen in the above package snippets, Guix package
definitions are typically very short and rather easy to write
yourself.  The ,(code [guix import gem]) tool was made to lower ,(code
                                                                    [foo bar]) the barrier even more by generating most of the boilerplate
code.  For example:])

  (source-code "$ guix import gem pry")

  (p [Produces this Scheme code:])

(source-code
 (scheme-source
  [;;; hello there
(+ 1 2 3)]))

  (source-code
   (scheme-source
     "(package
  (name \"ruby-pry\")
  (version \"0.10.1\")
  (source
   (origin
     (method url-fetch)
     (uri (rubygems-uri \"pry\" version))
     (sha256
      (base32
       \"1j0r5fm0wvdwzbh6d6apnp7c0n150hpm9zxpm5xvcgfqr36jaj8z\"))))
  (build-system ruby-build-system)
  (propagated-inputs
   `((\"ruby-coderay\" ,ruby-coderay)
     (\"ruby-method-source\" ,ruby-method-source)
     (\"ruby-slop\" ,ruby-slop)))
  (synopsis
   \"An IRB alternative and runtime developer console\")
  (description
   \"An IRB alternative and runtime developer console\")
  (home-page \"http://pryrepl.org\")
  (license expat))"))

  (p [One still has to package the propagated inputs if they aren’t
yet available, add the necessary inputs for building native extensions
if needed, and fiddle with the native inputs needed to run the test
suite, but for most pure Ruby gems this gets you close to a working
package quickly.])

  (p [In conclusion, while support for Ruby in Guix is still in its
early days, I hope that you have seen the benefits that using a
general-purpose, functional package manager can bring to your Ruby
environments (and all other environments, too.)  For more information
about Guix concepts, installation instructions, programming interface,
and tools, please refer to the
,(anchor [official manual] "https://gnu.org/software/guix/manual").

Check out the ,(anchor [help] "https://gnu.org/software/guix/help")
page for ways to contact the development team for help or to report
bugs.  If you are interested in getting your hands dirty, please
,(anchor [contribute] "https://gnu.org/software/guix/contribute").
Besides contributions of code, art, and docs, we also need ,(anchor
[hardware donations] "https://gnu.org/software/guix/donate") to grow
our build farm to meet the needs of all our users.  Happy hacking!]))