[svnbook] r3982 committed - Revamp the nightly build process in light of the recent directory...
svnbook at googlecode.com
svnbook at googlecode.com
Thu Aug 11 10:57:37 CDT 2011
Revision: 3982
Author: cmpilato at gmail.com
Date: Thu Aug 11 08:56:42 2011
Log: Revamp the nightly build process in light of the recent directory
reorganization.
* trunk/tools/build-nightlies
Move and complete rework this...
* www/bin/build-and-deploy.py
...into this...
* www/bin/build-and-deploy.conf
...and using this as the configuration file.
* www/bin
(Which necessitated the creation of this.)
http://code.google.com/p/svnbook/source/detail?r=3982
Added:
/www/bin
/www/bin/build-and-deploy.conf
/www/bin/build-and-deploy.py
Deleted:
/trunk/tools/build-nightlies
=======================================
--- /dev/null
+++ /www/bin/build-and-deploy.conf Thu Aug 11 08:56:42 2011
@@ -0,0 +1,47 @@
+# Configuration file for svnbook's build-and-deploy.py
+
+[general]
+report_target = svnbook-dev at red-bean.com
+report_sender = svnbook-build-daemon at red-bean.com
+report_sender_name = Svnbook Build Daemon
+
+[en-trunk]
+locale = en
+version = 1.7
+src_path = trunk/en
+adsense = 1
+
+[en-1.6]
+locale = en
+version = 1.6
+adsense = 1
+
+[es]
+locale = es
+version = 1.0
+
+[it]
+locale = it
+version = 1.2
+
+[nb]
+locale = nb
+version = 1.4
+
+[ru]
+locale = ru
+version = 1.4
+formats = html, html-chunk
+
+[de]
+locale = de
+version = 1.5
+
+[fr]
+locale = fr
+version = 1.5
+
+[zh]
+locale = zh
+version = 1.6
+formats = html, html-chunk
=======================================
--- /dev/null
+++ /www/bin/build-and-deploy.py Thu Aug 11 08:56:42 2011
@@ -0,0 +1,331 @@
+#!/usr/bin/env python
+# vim:sw=4
+
+# ----------------------------------------------------------------------
+# build-and-deploy.py: Subversion book (nightly) build script.
+#
+# Read a configuration file (INI-format) for information about scheduled
+# builds of the Subversion book sources -- which versions, which output
+# formats, where to drop the results, etc. -- and perform the requisite
+# builds.
+# ----------------------------------------------------------------------
+
+import sys
+import os
+import getopt
+import shutil
+import time
+import subprocess
+import traceback
+import ConfigParser
+
+# ----------------------------------------------------------------------
+# The configuration file has a [general] section, then a set of build
+# definition sections, each of which describes a build to attempt:
+#
+# [general]
+#
+# # Destination email address for build reports.
+# report_target =
+#
+# # Sender email address for build reports.
+# report_sender =
+#
+# # Sender name for build reports.
+# report_sender_name =
+#
+#
+# [BUILD-NAME1]
+#
+# # Locale code ("en", "de", ...). Required.
+# locale =
+#
+# # Book version number ("1.5", "1.2", ...). Required.
+# version = BOOK-VERSION
+#
+# # Path to the locale version book source directory. Defaults to
+# # "branches/<version>/<locale>".
+# src_path =
+#
+# # Path in which the build deliverables should be dropped. Defaults
+# # to "www/<locale>/<version>".
+# dst_path = PUBLISH-PATH
+#
+# # Comma-delimited list of output formats to use. Defaults to
+# # "html, html-chunk, pdf".
+# formats = FORMAT1[, FORMAT2 ...]
+#
+# # Set to "1" if AdSense code substitution should be applied. Defaults
+# # to "0".
+# adsense = "0" | "1"
+#
+# [BUILD-NAME2]
+# ...
+#
+# Here's an example of such a file. Note that the trunk English
+# version requires a "src_path", that both English version add AdSense
+# stuff, and that the German 1.5 version doesn't generate PDF output.
+#
+# [general]
+# report_target = svnbook-dev at red-bean.com
+# report_sender = svnbook-build-daemon at red-bean.com
+# report_sender_name = Svnbook Build Daemon
+#
+# [en-trunk]
+# locale = en
+# version = 1.7
+# src_path = trunk/en
+# adsense = 1
+#
+# [en-1.6]
+# locale = en
+# version = 1.6
+# adsense = 1
+#
+# [de-1.5]
+# locale = de
+# version = 1.5
+# formats = html, html-chunk
+# ----------------------------------------------------------------------
+
+
+def format_duration(seconds):
+ """Return a string describing SECONDS as a more pretty-printed
+ string describing hours, minutes, and seconds."""
+
+ seconds = int(seconds)
+ hours = seconds / 3600
+ minutes = seconds % 3600 / 60
+ seconds = seconds % 60
+ return ((hours and "%dh " % hours or "")
+ + (minutes and "%dm " % minutes or "")
+ + "%ds" % seconds)
+
+
+def sendmail(mail_from, mail_from_name, mail_to, subject, body):
+ """Open a pipe to the 'sendmail' binary, and use it to transmit an
+ email from MAIL_FROM (with name MAIL_FROM_NAME) to MAIL_TO, with
+ SUBJECT and BODY."""
+
+ assert mail_from and mail_to and mail_from_name
+ try:
+ p = subprocess.Popen(['/usr/sbin/sendmail', '-f', mail_from,
mail_to],
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
+ p.stdin.write("Subject: %s\n" % subject)
+ p.stdin.write("To: %s\n" % mail_to)
+ p.stdin.write("From: %s <%s>\n" % (mail_from_name, mail_from))
+ p.stdin.write("\n")
+ p.stdin.write(body)
+ p.stdin.close()
+ output = p.stdout.read()
+ status = p.wait()
+ if len(output) or status:
+ sys.stderr.write("MTA output when sending email (%s):\n" %
subject)
+ sys.stderr.write(output)
+ sys.stderr.write("Exit status %d\n" % status)
+ except IOError:
+ etype, value, tb = sys.exc_info()
+ sys.stderr.write("Failed sending email (%s):\n" % subject)
+ traceback.print_exception(etype, value, tb)
+ sys.stderr.write("\n")
+ sys.stderr.write("Email body:\n")
+ sys.stderr.write(body)
+
+
+def do_build(locale, version, src_path, dst_path, formats,
+ dry_run=False, verbose=False):
+
+ temp_dir = os.path.join(src_path, '__TEMPINSTALL__')
+
+ if verbose:
+ print "Build requested:"
+ print " Locale = %s" % (locale)
+ print " Version = %s" % (version)
+ print " Source Path = %s" % (src_path)
+ print " Destination Path = %s" % (dst_path)
+ print " Formats = %s" % (str(formats))
+ print " Temporary Directory = %s" % (temp_dir)
+
+ # Extend FORMATS to include relevant archived formats, too.
+ if 'html' in formats:
+ formats.append('html-arch')
+ if 'html-chunk' in formats:
+ formats.append('html-chunk-arch')
+
+ # Remove the temporary directory if it already exists.
+ if os.path.isdir(temp_dir):
+ if verbose:
+ print "Erase: %s" % (temp_dir)
+ if not dry_run:
+ shutil.rmtree(temp_dir)
+
+ # Create the destination directory (and parents) as necessary.
+ if not os.path.isdir(dst_path):
+ if verbose:
+ print "Create (with parents): %s" % (dst_path)
+ if not dry_run:
+ os.makedirs(dst_path)
+
+ make_cmd =
(['make', 'INSTALL_SUBDIR=__TEMPINSTALL__', 'clean', 'valid']
+ + map(lambda x: 'install-%s' % x, formats))
+ build_log = os.path.join(dst_path, 'nightly-build.log')
+ log_fp = open(build_log, 'w', 1)
+
+ # Change to the source directory and fire off 'make'.
+ cwd = os.getcwd()
+ os.chdir(src_path)
+ try:
+ if verbose:
+ print "Change directory: %s" % (src_path)
+ if verbose:
+ print "Building: %s" % (' '.join(make_cmd))
+ if not dry_run:
+ p = subprocess.Popen(make_cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
+ while 1:
+ data = p.stdout.readline()
+ if not data:
+ break
+ if verbose:
+ print data.rstrip("\n")
+ log_fp.write(data)
+ exitcode = p.wait()
+ if exitcode:
+ raise RuntimeError("make exited with error %d" % exitcode)
+ # Regardless of how the build went, change back toour
+ # original directory.
+ finally:
+ log_fp.close()
+ if verbose:
+ print "Change directory: %s" % (cwd)
+ os.chdir(cwd)
+
+ # Move stuff into place, deleting old stuff first.
+ build_log = os.path.join(dst_path, 'nightly-build.log')
+ temp_build_log = os.path.join(temp_dir, 'nightly-build.log')
+ if verbose:
+ print "Rename: %s -> %s" % (build_log, temp_build_log)
+ if not dry_run:
+ os.rename(build_log, temp_build_log)
+ if os.path.isdir(dst_path):
+ if verbose:
+ print "Erase: %s" % (dst_path)
+ if not dry_run:
+ shutil.rmtree(dst_path)
+ if verbose:
+ print "Move into place: %s -> %s" % (temp_dir, dst_path)
+ if not dry_run:
+ os.rename(temp_dir, dst_path)
+
+
+def get_option(cfg, section, option, default_value=None):
+ """Return the value of OPTION in SECTION from CFG. If there is no
+ such option, or its value is empty, return DEFAULT_VALUE
+ instead."""
+
+ if not cfg.has_option(section, option):
+ return default_value
+ return cfg.get(section, option) or default_value
+
+
+def main(conf_file, dry_run, verbose):
+ """Read CONF_FILE and attempt the builds described there. If
+ DRY_RUN is set, don't really do anything. If VERBOSE is set, be
+ verbose."""
+
+ cfg = ConfigParser.RawConfigParser()
+ cfg.read(conf_file)
+ build_sections = []
+ mail_from = get_option(cfg, 'general', 'report_sender')
+ mail_to = get_option(cfg, 'general', 'report_target')
+ mail_from = get_option(cfg, 'general', 'report_sender',
+ 'SvnBook Build Daemon')
+ for section in cfg.sections():
+ if section == 'general':
+ continue
+ build_sections.append(section)
+
+ # Update the working copy
+ if verbose:
+ print "SVN Update: ."
+ if not dry_run:
+ if verbose:
+ os.system('svn up')
+ else:
+ os.system('svn up -q')
+
+ # Do the build(s)
+ for section in build_sections:
+ locale = cfg.get(section, 'locale')
+ version = cfg.get(section, 'version')
+ src_path = get_option(cfg, section, 'src_path',
+ 'branches/%s/%s' % (version, locale))
+ dst_path = get_option(cfg, section, 'dst_path',
+ 'www/%s/%s' % (locale, version))
+ formats = map(lambda x: x.strip(),
+ get_option(cfg, section, 'formats',
+ 'html, html-chunk, pdf').split(','))
+ build_log = os.path.join(dst_path, 'nightly-build.log')
+ try:
+ build_begin_time = time.time()
+ do_build(locale, version, src_path, dst_path, formats,
+ dry_run, verbose)
+ build_end_time = time.time()
+ except Exception, e:
+ if dry_run:
+ print "Send failure email: %s" % (locale)
+ else:
+ body = "The svnbook build for the '%s' locale has
failed.\n" \
+ "Please investigate.\n\n%s\n" \
+ "-- Svnbook Build Daemon.\n" % (locale, build_log)
+ if mail_to and mail_from:
+ sendmail(mail_from, mail_from_name, mail_to,
+ "Build Failure Alert: '%s'" % locale, body)
+ else:
+ sys.stderr.write(body)
+ sys.stderr.write(str(e) + "\n")
+
+
+def usage_and_exit(errmsg=None):
+ stream = errmsg and sys.stderr or sys.stdout
+ stream.write("""\
+Usage: %s [OPTIONS] BUILD-CONF-FILE
+
+Options:
+
+ --help (-h) Show this usage message and exit.
+ --dry-run (-n) Don't really do anything. (Implies --verbose.)
+ --verbose (-v) Be verbose about what's going on.
+
+Read Subversion Book build definitions from BUILD-CONF-FILE and build the
+specified distributions.
+""" % (os.path.basename(sys.argv[0])))
+ if errmsg:
+ stream.write("ERROR: %s\n" % (errmsg))
+ sys.exit(errmsg and 1 or 0)
+
+
+if __name__ == "__main__":
+ dry_run = False
+ verbose = False
+ try:
+ opts, args = getopt.getopt(sys.argv[1:], "hnv",
+ ["help", "dry-run", "verbose"])
+ except getopt.GetoptError, e:
+ usage_and_exit(str(e))
+ for option, value in opts:
+ if option in ["-h", "--help"]:
+ usage_and_exit()
+ elif option in ["-n", "--dry-run"]:
+ dry_run = True
+ elif option in ["-v", "--verbose"]:
+ verbose = True
+ if not args:
+ usage_and_exit("Not enough arguments.")
+ conf_file = args[0]
+ if dry_run:
+ verbose = True
+ main(conf_file, dry_run, verbose)
=======================================
--- /trunk/tools/build-nightlies Fri Jul 2 05:23:07 2010
+++ /dev/null
@@ -1,212 +0,0 @@
-#!/usr/bin/python
-# vim:sw=4
-
-import sys
-import os
-import shutil
-import time
-import subprocess
-import traceback
-
-SKIP_LOCALES = ()
-SKIP_PDF_LOCALES = ('ru', 'zh')
-MAIL_DESTINATION = "svnbook-dev at red-bean.com"
-MAIL_SENDER = "svnbook-build-daemon at red-bean.com"
-MAIL_SENDER_NAME = "Svnbook Build Daemon"
-DROPSPOT_URL = "http://svnbook.red-bean.com/nightly"
-
-
-def format_duration(seconds):
- seconds = int(seconds)
- hours = seconds / 3600
- minutes = seconds % 3600 / 60
- seconds = seconds % 60
- return ((hours and "%dh " % hours or "")
- + (minutes and "%dm " % minutes or "")
- + "%ds" % seconds)
-
-
-def sendmail(subject, body):
- try:
- p = subprocess.Popen(
- ['/usr/sbin/sendmail', '-f', MAIL_SENDER,
MAIL_DESTINATION],
- stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
-
- p.stdin.write("Subject: %s\n" % subject)
- p.stdin.write("To: %s\n" % MAIL_DESTINATION)
- p.stdin.write("From: %s <%s>\n" % (MAIL_SENDER_NAME, MAIL_SENDER))
- p.stdin.write("\n")
- p.stdin.write(body)
- p.stdin.close()
- output = p.stdout.read()
- status = p.wait()
- if len(output) or status:
- sys.stderr.write("MTA output when sending email (%s):\n" %
subject)
- sys.stderr.write(output)
- sys.stderr.write("Exit status %d" % status)
- except IOError:
- etype, value, tb = sys.exc_info()
- sys.stderr.write("Failed sending email (%s):\n" % subject)
- traceback.print_exception(etype, value, tb)
- sys.stderr.write("\n")
- sys.stderr.write("Email body:\n")
- sys.stderr.write(body)
-
-
-if len(sys.argv) < 3:
- sys.stderr.write("""Usage: %s SRC-DIR TGT-DIR [--dryrun]
-
-Crawl SRC-DIR looking for book translations, building distributions of
-them, and exploding those distributions into TGT-DIR.
-""" % (os.path.basename(sys.argv[0])))
- sys.exit(1)
-
-BOOKSRC = os.path.abspath(sys.argv[1])
-DROPSPOT = os.path.abspath(sys.argv[2])
-DRYRUN = (len(sys.argv) > 3)
-
-# Update the working copy
-if DRYRUN:
- print "SVN-Update: %s" % BOOKSRC
-else:
- os.system('svn up -q ' + BOOKSRC)
-
-# Timestamp
-build_begin_time = time.time()
-
-# Find translations
-locales = []
-built_locales = []
-kids = os.listdir(BOOKSRC)
-for kid in kids:
- full_path = os.path.join(BOOKSRC, kid)
- if os.path.isfile(full_path):
- continue
- if os.path.exists(os.path.join(full_path, 'Makefile')):
- locales.append(kid)
-
-# Build the locales
-for i in SKIP_LOCALES:
- try:
- locales.remove(i)
- except ValueError:
- pass
-locales.sort()
-cwd = os.getcwd()
-for locale in locales:
- # Calculate some paths
- locale_dir = os.path.join(BOOKSRC, locale)
- temp_dir = os.path.join(locale_dir, '__TEMPINSTALL__')
- build_log = os.path.join(DROPSPOT, 'nightly-build.%s.log' % (locale))
- dropspot_locale_path = os.path.join(DROPSPOT, locale)
-
- # Figger out which book formats to build
- book_formats = ['html',
- 'html-chunk',
- 'html-arch',
- 'html-chunk-arch',
- 'pdf',
- ]
- if locale in SKIP_PDF_LOCALES:
- book_formats.remove('pdf')
-
- try:
- # Build
- make_cmd =
(['make', 'INSTALL_SUBDIR=__TEMPINSTALL__', 'clean', 'valid']
- + map(lambda x: 'install-%s' % x, book_formats))
- if os.path.isdir(temp_dir):
- if DRYRUN:
- print "Erase: %s" % (temp_dir)
- else:
- shutil.rmtree(temp_dir)
- os.chdir(locale_dir)
- try:
- if DRYRUN:
- print "Run: %s" % (make_cmd)
- else:
- p = subprocess.Popen(make_cmd, stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT)
- logfp = open(build_log, 'w', 1)
- while 1:
- data = p.stdout.readline()
- if not data: break
- logfp.write(data)
- exitcode = p.wait()
- if exitcode:
- raise RuntimeError("make exited with error %d" %
exitcode)
- finally:
- os.chdir(cwd)
-
- # Move stuff into place.
- if os.path.isdir(dropspot_locale_path):
- if DRYRUN:
- print "Erase: %s" % (dropspot_locale_path)
- else:
- shutil.rmtree(dropspot_locale_path)
- if DRYRUN:
- print "Move into place: %s -> %s" % (temp_dir,
dropspot_locale_path)
- else:
- os.rename(temp_dir, dropspot_locale_path)
- built_locales.append(locale)
- except:
- if DRYRUN:
- print "Send failure email: %s" % (locale)
- else:
- sendmail("Nightly Build Failure Alert: '%s'" % locale,
- "The nightly svnbook build for the '%s' locale\n"
- "has failed. Please investigate.\n"
- "\n"
- "%s/nightly-build.%s.log\n"
- "\n"
- "-- The Svnbook Build Daemon.\n"
- % (locale, DROPSPOT_URL, locale))
-
-# Timestamp
-build_end_time = time.time()
-
-# Write out index.html
-if DRYRUN:
- print "Write index.html:"
- fp = sys.stdout
-else:
- fp = open(os.path.join(DROPSPOT, 'index.html'), 'w')
-
-fp.write("""<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
-"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
-<html>
-<head>
-<title>Version Control with Subversion - Nightly Builds</title>
-<style type="text/css">
-h1, h2, h3 { margin: 0 }
-dl { margin-left: 3em }
-</style>
-</head>
-<body>
-<h1><i>Version Control with Subversion</i></h1>
-<h2>Automated Nightly Book Builds</h2>
-<h3>Last Build Completed At: %s GMT</h3>
-<h3>Last Build Duration: %s</h3>
-<dl>
-""" % (time.asctime(time.gmtime(build_end_time)),
- format_duration(build_end_time - build_begin_time)))
-for locale in locales:
- fp.write("<dt>%s</dt>" % (locale.upper()))
- if locale in built_locales:
- fp.write("""
-<dd>[<a href="%s/svn-book.html">single-page HTML (read online)</a>]</dd>
-<dd>[<a href="%s/index.html">multi-page HTML (read online)</a>]</dd>
-<dd>[<a href="%s/svn-book-html.tar.bz2">single-page HTML
(in .tar.bz2)</a>]</dd>
-<dd>[<a href="%s/svn-book-html-chunk.tar.bz2">multi-page HTML
(in .tar.bz2)</a>]</dd>
-""" % (locale, locale, locale, locale))
- if locale not in SKIP_PDF_LOCALES:
- fp.write('<dd>[<a href="%s/svn-book.pdf">PDF</a>]</dd>\n' %
locale)
- else:
- fp.write("""
-<dd><em>Uh-oh! No nightly build for this locale.
- (See <a href="nightly-build.%s.log">build log</a>.)</em></dd>
-""" % (locale))
-fp.write("""
-</dl>
-</body>
-</html>
-""")
More information about the svnbook-dev
mailing list