[svnbook] r3982 committed - Revamp the nightly build process in light of the recent directory...

svnbook at googlecode.com svnbook at googlecode.com
Thu Aug 11 10:57:37 CDT 2011


Revision: 3982
Author:   cmpilato at gmail.com
Date:     Thu Aug 11 08:56:42 2011
Log:      Revamp the nightly build process in light of the recent directory
reorganization.

* trunk/tools/build-nightlies
   Move and complete rework this...

* www/bin/build-and-deploy.py
   ...into this...

* www/bin/build-and-deploy.conf
   ...and using this as the configuration file.

* www/bin
   (Which necessitated the creation of this.)
http://code.google.com/p/svnbook/source/detail?r=3982

Added:
  /www/bin
  /www/bin/build-and-deploy.conf
  /www/bin/build-and-deploy.py
Deleted:
  /trunk/tools/build-nightlies

=======================================
--- /dev/null
+++ /www/bin/build-and-deploy.conf	Thu Aug 11 08:56:42 2011
@@ -0,0 +1,47 @@
+# Configuration file for svnbook's build-and-deploy.py
+
+[general]
+report_target = svnbook-dev at red-bean.com
+report_sender = svnbook-build-daemon at red-bean.com
+report_sender_name = Svnbook Build Daemon
+
+[en-trunk]
+locale = en
+version = 1.7
+src_path = trunk/en
+adsense = 1
+
+[en-1.6]
+locale = en
+version = 1.6
+adsense = 1
+
+[es]
+locale = es
+version = 1.0
+
+[it]
+locale = it
+version = 1.2
+
+[nb]
+locale = nb
+version = 1.4
+
+[ru]
+locale = ru
+version = 1.4
+formats = html, html-chunk
+
+[de]
+locale = de
+version = 1.5
+
+[fr]
+locale = fr
+version = 1.5
+
+[zh]
+locale = zh
+version = 1.6
+formats = html, html-chunk
=======================================
--- /dev/null
+++ /www/bin/build-and-deploy.py	Thu Aug 11 08:56:42 2011
@@ -0,0 +1,331 @@
+#!/usr/bin/env python
+# vim:sw=4
+
+# ----------------------------------------------------------------------
+# build-and-deploy.py:  Subversion book (nightly) build script.
+#
+# Read a configuration file (INI-format) for information about scheduled
+# builds of the Subversion book sources -- which versions, which output
+# formats, where to drop the results, etc. -- and perform the requisite
+# builds.
+# ----------------------------------------------------------------------
+
+import sys
+import os
+import getopt
+import shutil
+import time
+import subprocess
+import traceback
+import ConfigParser
+
+# ----------------------------------------------------------------------
+# The configuration file has a [general] section, then a set of build
+# definition sections, each of which describes a build to attempt:
+#
+#    [general]
+#
+#    # Destination email address for build reports.
+#    report_target =
+#
+#    # Sender email address for build reports.
+#    report_sender =
+#
+#    # Sender name for build reports.
+#    report_sender_name =
+#
+#
+#    [BUILD-NAME1]
+#
+#    # Locale code ("en", "de", ...).  Required.
+#    locale =
+#
+#    # Book version number ("1.5", "1.2", ...).  Required.
+#    version = BOOK-VERSION
+#
+#    # Path to the locale version book source directory.  Defaults to
+#    # "branches/<version>/<locale>".
+#    src_path =
+#
+#    # Path in which the build deliverables should be dropped.  Defaults
+#    # to "www/<locale>/<version>".
+#    dst_path = PUBLISH-PATH
+#
+#    # Comma-delimited list of output formats to use.  Defaults to
+#    # "html, html-chunk, pdf".
+#    formats = FORMAT1[, FORMAT2 ...]
+#
+#    # Set to "1" if AdSense code substitution should be applied.  Defaults
+#    # to "0".
+#    adsense = "0" | "1"
+#
+#    [BUILD-NAME2]
+#    ...
+#
+# Here's an example of such a file.  Note that the trunk English
+# version requires a "src_path", that both English version add AdSense
+# stuff, and that the German 1.5 version doesn't generate PDF output.
+#
+#    [general]
+#    report_target = svnbook-dev at red-bean.com
+#    report_sender = svnbook-build-daemon at red-bean.com
+#    report_sender_name = Svnbook Build Daemon
+#
+#    [en-trunk]
+#    locale = en
+#    version = 1.7
+#    src_path = trunk/en
+#    adsense = 1
+#
+#    [en-1.6]
+#    locale = en
+#    version = 1.6
+#    adsense = 1
+#
+#    [de-1.5]
+#    locale = de
+#    version = 1.5
+#    formats = html, html-chunk
+# ----------------------------------------------------------------------
+
+
+def format_duration(seconds):
+    """Return a string describing SECONDS as a more pretty-printed
+    string describing hours, minutes, and seconds."""
+
+    seconds = int(seconds)
+    hours = seconds / 3600
+    minutes = seconds % 3600 / 60
+    seconds = seconds % 60
+    return ((hours and "%dh " % hours or "")
+            + (minutes and "%dm " % minutes or "")
+            + "%ds" % seconds)
+
+
+def sendmail(mail_from, mail_from_name, mail_to, subject, body):
+    """Open a pipe to the 'sendmail' binary, and use it to transmit an
+    email from MAIL_FROM (with name MAIL_FROM_NAME) to MAIL_TO, with
+    SUBJECT and BODY."""
+
+    assert mail_from and mail_to and mail_from_name
+    try:
+        p = subprocess.Popen(['/usr/sbin/sendmail', '-f', mail_from,  
mail_to],
+                             stdin=subprocess.PIPE,
+                             stdout=subprocess.PIPE,
+                             stderr=subprocess.STDOUT)
+        p.stdin.write("Subject: %s\n" % subject)
+        p.stdin.write("To: %s\n" % mail_to)
+        p.stdin.write("From: %s <%s>\n" % (mail_from_name, mail_from))
+        p.stdin.write("\n")
+        p.stdin.write(body)
+        p.stdin.close()
+        output = p.stdout.read()
+        status = p.wait()
+        if len(output) or status:
+            sys.stderr.write("MTA output when sending email (%s):\n" %  
subject)
+            sys.stderr.write(output)
+            sys.stderr.write("Exit status %d\n" % status)
+    except IOError:
+        etype, value, tb = sys.exc_info()
+        sys.stderr.write("Failed sending email (%s):\n" % subject)
+        traceback.print_exception(etype, value, tb)
+        sys.stderr.write("\n")
+        sys.stderr.write("Email body:\n")
+        sys.stderr.write(body)
+
+
+def do_build(locale, version, src_path, dst_path, formats,
+             dry_run=False, verbose=False):
+
+    temp_dir = os.path.join(src_path, '__TEMPINSTALL__')
+
+    if verbose:
+        print "Build requested:"
+        print "   Locale = %s" % (locale)
+        print "   Version = %s" % (version)
+        print "   Source Path = %s" % (src_path)
+        print "   Destination Path = %s" % (dst_path)
+        print "   Formats = %s" % (str(formats))
+        print "   Temporary Directory = %s" % (temp_dir)
+
+    # Extend FORMATS to include relevant archived formats, too.
+    if 'html' in formats:
+        formats.append('html-arch')
+    if 'html-chunk' in formats:
+        formats.append('html-chunk-arch')
+
+    # Remove the temporary directory if it already exists.
+    if os.path.isdir(temp_dir):
+        if verbose:
+            print "Erase: %s" % (temp_dir)
+        if not dry_run:
+            shutil.rmtree(temp_dir)
+
+    # Create the destination directory (and parents) as necessary.
+    if not os.path.isdir(dst_path):
+        if verbose:
+            print "Create (with parents): %s" % (dst_path)
+        if not dry_run:
+            os.makedirs(dst_path)
+
+    make_cmd =  
(['make', 'INSTALL_SUBDIR=__TEMPINSTALL__', 'clean', 'valid']
+                + map(lambda x: 'install-%s' % x, formats))
+    build_log = os.path.join(dst_path, 'nightly-build.log')
+    log_fp = open(build_log, 'w', 1)
+
+    # Change to the source directory and fire off 'make'.
+    cwd = os.getcwd()
+    os.chdir(src_path)
+    try:
+        if verbose:
+            print "Change directory: %s" % (src_path)
+        if verbose:
+            print "Building: %s" % (' '.join(make_cmd))
+        if not dry_run:
+            p = subprocess.Popen(make_cmd,
+                                 stdout=subprocess.PIPE,
+                                 stderr=subprocess.STDOUT)
+            while 1:
+                data = p.stdout.readline()
+                if not data:
+                    break
+                if verbose:
+                    print data.rstrip("\n")
+                log_fp.write(data)
+            exitcode = p.wait()
+            if exitcode:
+                raise RuntimeError("make exited with error %d" % exitcode)
+    # Regardless of how the build went, change back toour
+    # original directory.
+    finally:
+        log_fp.close()
+        if verbose:
+            print "Change directory: %s" % (cwd)
+        os.chdir(cwd)
+
+    # Move stuff into place, deleting old stuff first.
+    build_log = os.path.join(dst_path, 'nightly-build.log')
+    temp_build_log = os.path.join(temp_dir, 'nightly-build.log')
+    if verbose:
+        print "Rename: %s -> %s" % (build_log, temp_build_log)
+    if not dry_run:
+        os.rename(build_log, temp_build_log)
+    if os.path.isdir(dst_path):
+        if verbose:
+            print "Erase: %s" % (dst_path)
+        if not dry_run:
+            shutil.rmtree(dst_path)
+    if verbose:
+        print "Move into place: %s -> %s" % (temp_dir, dst_path)
+    if not dry_run:
+        os.rename(temp_dir, dst_path)
+
+
+def get_option(cfg, section, option, default_value=None):
+    """Return the value of OPTION in SECTION from CFG.  If there is no
+    such option, or its value is empty, return DEFAULT_VALUE
+    instead."""
+
+    if not cfg.has_option(section, option):
+        return default_value
+    return cfg.get(section, option) or default_value
+
+
+def main(conf_file, dry_run, verbose):
+    """Read CONF_FILE and attempt the builds described there.  If
+    DRY_RUN is set, don't really do anything.  If VERBOSE is set, be
+    verbose."""
+
+    cfg = ConfigParser.RawConfigParser()
+    cfg.read(conf_file)
+    build_sections = []
+    mail_from = get_option(cfg, 'general', 'report_sender')
+    mail_to = get_option(cfg, 'general', 'report_target')
+    mail_from = get_option(cfg, 'general', 'report_sender',
+                           'SvnBook Build Daemon')
+    for section in cfg.sections():
+        if section == 'general':
+            continue
+        build_sections.append(section)
+
+    # Update the working copy
+    if verbose:
+        print "SVN Update: ."
+    if not dry_run:
+        if verbose:
+            os.system('svn up')
+        else:
+            os.system('svn up -q')
+
+    # Do the build(s)
+    for section in build_sections:
+        locale = cfg.get(section, 'locale')
+        version = cfg.get(section, 'version')
+        src_path = get_option(cfg, section, 'src_path',
+                              'branches/%s/%s' % (version, locale))
+        dst_path = get_option(cfg, section, 'dst_path',
+                              'www/%s/%s' % (locale, version))
+        formats = map(lambda x: x.strip(),
+                      get_option(cfg, section, 'formats',
+                                 'html, html-chunk, pdf').split(','))
+        build_log = os.path.join(dst_path, 'nightly-build.log')
+        try:
+            build_begin_time = time.time()
+            do_build(locale, version, src_path, dst_path, formats,
+                     dry_run, verbose)
+            build_end_time = time.time()
+        except Exception, e:
+            if dry_run:
+                print "Send failure email: %s" % (locale)
+            else:
+                body = "The svnbook build for the '%s' locale has  
failed.\n" \
+                       "Please investigate.\n\n%s\n" \
+                       "-- Svnbook Build Daemon.\n" % (locale, build_log)
+                if mail_to and mail_from:
+                    sendmail(mail_from, mail_from_name, mail_to,
+                             "Build Failure Alert: '%s'" % locale, body)
+                else:
+                    sys.stderr.write(body)
+                    sys.stderr.write(str(e) + "\n")
+
+
+def usage_and_exit(errmsg=None):
+    stream = errmsg and sys.stderr or sys.stdout
+    stream.write("""\
+Usage: %s [OPTIONS] BUILD-CONF-FILE
+
+Options:
+
+   --help (-h)      Show this usage message and exit.
+   --dry-run (-n)   Don't really do anything.  (Implies --verbose.)
+   --verbose (-v)   Be verbose about what's going on.
+
+Read Subversion Book build definitions from BUILD-CONF-FILE and build the
+specified distributions.
+""" % (os.path.basename(sys.argv[0])))
+    if errmsg:
+        stream.write("ERROR: %s\n" % (errmsg))
+    sys.exit(errmsg and 1 or 0)
+
+
+if __name__ == "__main__":
+    dry_run = False
+    verbose = False
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "hnv",
+                                   ["help", "dry-run", "verbose"])
+    except getopt.GetoptError, e:
+        usage_and_exit(str(e))
+    for option, value in opts:
+        if option in ["-h", "--help"]:
+            usage_and_exit()
+        elif option in ["-n", "--dry-run"]:
+            dry_run = True
+        elif option in ["-v", "--verbose"]:
+            verbose = True
+    if not args:
+        usage_and_exit("Not enough arguments.")
+    conf_file = args[0]
+    if dry_run:
+        verbose = True
+    main(conf_file, dry_run, verbose)
=======================================
--- /trunk/tools/build-nightlies	Fri Jul  2 05:23:07 2010
+++ /dev/null
@@ -1,212 +0,0 @@
-#!/usr/bin/python
-# vim:sw=4
-
-import sys
-import os
-import shutil
-import time
-import subprocess
-import traceback
-
-SKIP_LOCALES = ()
-SKIP_PDF_LOCALES = ('ru', 'zh')
-MAIL_DESTINATION = "svnbook-dev at red-bean.com"
-MAIL_SENDER = "svnbook-build-daemon at red-bean.com"
-MAIL_SENDER_NAME = "Svnbook Build Daemon"
-DROPSPOT_URL = "http://svnbook.red-bean.com/nightly"
-
-
-def format_duration(seconds):
-    seconds = int(seconds)
-    hours = seconds / 3600
-    minutes = seconds % 3600 / 60
-    seconds = seconds % 60
-    return ((hours and "%dh " % hours or "")
-            + (minutes and "%dm " % minutes or "")
-            + "%ds" % seconds)
-
-
-def sendmail(subject, body):
-    try:
-        p = subprocess.Popen(
-                ['/usr/sbin/sendmail', '-f', MAIL_SENDER,  
MAIL_DESTINATION],
-                stdin=subprocess.PIPE, stdout=subprocess.PIPE,  
stderr=subprocess.STDOUT)
-
-        p.stdin.write("Subject: %s\n" % subject)
-        p.stdin.write("To: %s\n" % MAIL_DESTINATION)
-        p.stdin.write("From: %s <%s>\n" % (MAIL_SENDER_NAME, MAIL_SENDER))
-        p.stdin.write("\n")
-        p.stdin.write(body)
-        p.stdin.close()
-        output = p.stdout.read()
-        status = p.wait()
-        if len(output) or status:
-            sys.stderr.write("MTA output when sending email (%s):\n" %  
subject)
-            sys.stderr.write(output)
-            sys.stderr.write("Exit status %d" % status)
-    except IOError:
-        etype, value, tb = sys.exc_info()
-        sys.stderr.write("Failed sending email (%s):\n" % subject)
-        traceback.print_exception(etype, value, tb)
-        sys.stderr.write("\n")
-        sys.stderr.write("Email body:\n")
-        sys.stderr.write(body)
-
-
-if len(sys.argv) < 3:
-    sys.stderr.write("""Usage: %s SRC-DIR TGT-DIR [--dryrun]
-
-Crawl SRC-DIR looking for book translations, building distributions of
-them, and exploding those distributions into TGT-DIR.
-""" % (os.path.basename(sys.argv[0])))
-    sys.exit(1)
-
-BOOKSRC = os.path.abspath(sys.argv[1])
-DROPSPOT = os.path.abspath(sys.argv[2])
-DRYRUN = (len(sys.argv) > 3)
-
-# Update the working copy
-if DRYRUN:
-    print "SVN-Update: %s" % BOOKSRC
-else:
-    os.system('svn up -q ' + BOOKSRC)
-
-# Timestamp
-build_begin_time = time.time()
-
-# Find translations
-locales = []
-built_locales = []
-kids = os.listdir(BOOKSRC)
-for kid in kids:
-    full_path = os.path.join(BOOKSRC, kid)
-    if os.path.isfile(full_path):
-        continue
-    if os.path.exists(os.path.join(full_path, 'Makefile')):
-        locales.append(kid)
-
-# Build the locales
-for i in SKIP_LOCALES:
-    try:
-        locales.remove(i)
-    except ValueError:
-        pass
-locales.sort()
-cwd = os.getcwd()
-for locale in locales:
-    # Calculate some paths
-    locale_dir = os.path.join(BOOKSRC, locale)
-    temp_dir = os.path.join(locale_dir, '__TEMPINSTALL__')
-    build_log = os.path.join(DROPSPOT, 'nightly-build.%s.log' % (locale))
-    dropspot_locale_path = os.path.join(DROPSPOT, locale)
-
-    # Figger out which book formats to build
-    book_formats = ['html',
-                    'html-chunk',
-                    'html-arch',
-                    'html-chunk-arch',
-                    'pdf',
-                    ]
-    if locale in SKIP_PDF_LOCALES:
-        book_formats.remove('pdf')
-
-    try:
-        # Build
-        make_cmd =  
(['make', 'INSTALL_SUBDIR=__TEMPINSTALL__', 'clean', 'valid']
-                    + map(lambda x: 'install-%s' % x, book_formats))
-        if os.path.isdir(temp_dir):
-            if DRYRUN:
-                print "Erase: %s" % (temp_dir)
-            else:
-                shutil.rmtree(temp_dir)
-        os.chdir(locale_dir)
-        try:
-            if DRYRUN:
-                print "Run: %s" % (make_cmd)
-            else:
-                p = subprocess.Popen(make_cmd, stdout=subprocess.PIPE,
-                        stderr=subprocess.STDOUT)
-                logfp = open(build_log, 'w', 1)
-                while 1:
-                    data = p.stdout.readline()
-                    if not data: break
-                    logfp.write(data)
-                exitcode = p.wait()
-                if exitcode:
-                    raise RuntimeError("make exited with error %d" %  
exitcode)
-        finally:
-            os.chdir(cwd)
-
-        # Move stuff into place.
-        if os.path.isdir(dropspot_locale_path):
-            if DRYRUN:
-                print "Erase: %s" % (dropspot_locale_path)
-            else:
-                shutil.rmtree(dropspot_locale_path)
-        if DRYRUN:
-            print "Move into place: %s -> %s" % (temp_dir,  
dropspot_locale_path)
-        else:
-            os.rename(temp_dir, dropspot_locale_path)
-        built_locales.append(locale)
-    except:
-        if DRYRUN:
-            print "Send failure email: %s" % (locale)
-        else:
-            sendmail("Nightly Build Failure Alert: '%s'" % locale,
-                    "The nightly svnbook build for the '%s' locale\n"
-                    "has failed.  Please investigate.\n"
-                    "\n"
-                    "%s/nightly-build.%s.log\n"
-                    "\n"
-                    "-- The Svnbook Build Daemon.\n"
-                    % (locale, DROPSPOT_URL, locale))
-
-# Timestamp
-build_end_time = time.time()
-
-# Write out index.html
-if DRYRUN:
-    print "Write index.html:"
-    fp = sys.stdout
-else:
-    fp = open(os.path.join(DROPSPOT, 'index.html'), 'w')
-
-fp.write("""<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
-"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
-<html>
-<head>
-<title>Version Control with Subversion - Nightly Builds</title>
-<style type="text/css">
-h1, h2, h3 { margin: 0 }
-dl { margin-left: 3em }
-</style>
-</head>
-<body>
-<h1><i>Version Control with Subversion</i></h1>
-<h2>Automated Nightly Book Builds</h2>
-<h3>Last Build Completed At: %s GMT</h3>
-<h3>Last Build Duration: %s</h3>
-<dl>
-""" % (time.asctime(time.gmtime(build_end_time)),
-    format_duration(build_end_time - build_begin_time)))
-for locale in locales:
-    fp.write("<dt>%s</dt>" % (locale.upper()))
-    if locale in built_locales:
-        fp.write("""
-<dd>[<a href="%s/svn-book.html">single-page HTML (read online)</a>]</dd>
-<dd>[<a href="%s/index.html">multi-page HTML (read online)</a>]</dd>
-<dd>[<a href="%s/svn-book-html.tar.bz2">single-page HTML  
(in .tar.bz2)</a>]</dd>
-<dd>[<a href="%s/svn-book-html-chunk.tar.bz2">multi-page HTML  
(in .tar.bz2)</a>]</dd>
-""" % (locale, locale, locale, locale))
-        if locale not in SKIP_PDF_LOCALES:
-            fp.write('<dd>[<a href="%s/svn-book.pdf">PDF</a>]</dd>\n' %  
locale)
-    else:
-        fp.write("""
-<dd><em>Uh-oh!  No nightly build for this locale.
-        (See <a href="nightly-build.%s.log">build log</a>.)</em></dd>
-""" % (locale))
-fp.write("""
-</dl>
-</body>
-</html>
-""")




More information about the svnbook-dev mailing list