Move from sandbox.

[SVN r37651]

tools/buildbot/src/boost/buildbot/__init__.py

@@ -0,0 +1,9 @@
+
+# Copyright Redshift Software, Inc. 2005-2007
+#
+# Distributed under the Boost Software License, Version 1.0. 
+# (See accompanying file LICENSE_1_0.txt or copy at 
+# http://www.boost.org/LICENSE_1_0.txt)
+
+modified = '$Date$'
+revision = '$Revision$'

tools/buildbot/src/boost/buildbot/char_translation_table.py

@@ -0,0 +1,19 @@
+
+# Copyright Redshift Software, Inc. 2005-2007
+#
+# Distributed under the Boost Software License, Version 1.0. 
+# (See accompanying file LICENSE_1_0.txt or copy at 
+# http://www.boost.org/LICENSE_1_0.txt)
+
+import string
+
+def chr_or_question_mark( c ):
+    if chr(c) in string.printable and c < 128 and c not in ( 0x09, 0x0b, 0x0c ):
+        return chr(c)
+    else:
+        return '?'
+
+char_translation_table = string.maketrans( 
+      ''.join( map( chr, range(0, 256) ) )
+    , ''.join( map( chr_or_question_mark, range(0, 256) ) )
+    )

tools/buildbot/src/boost/buildbot/factory.py

@@ -0,0 +1,281 @@
+
+# Copyright Redshift Software, Inc. 2005-2007
+#
+# Distributed under the Boost Software License, Version 1.0. 
+# (See accompanying file LICENSE_1_0.txt or copy at 
+# http://www.boost.org/LICENSE_1_0.txt)
+
+import boost.buildbot.step
+import buildbot
+import buildbot.process.base
+import buildbot.process.factory
+import buildbot.process.step
+import os.path
+import re
+import string
+import time
+import twisted.python
+import types
+import urllib
+
+from buildbot.process.factory import s
+
+def action(_action,*_args,**_kwargs):
+    _args = _args or []
+    _kwargs = _kwargs or {}
+    return (_action,_args,_kwargs)
+
+def defaults(_defaults = {},**_kwargs):
+    _defaults.update({
+        'haltOnFailure': _kwargs.get('haltOnFailure',False),
+        'flunkOnWarnings': _kwargs.get('flunkOnWarnings',False),
+        'flunkOnFailure': _kwargs.get('flunkOnFailure',True),
+        'warnOnWarnings': _kwargs.get('warnOnWarnings',False),
+        'warnOnFailure': _kwargs.get('warnOnFailure',False),
+        'timeout': _kwargs.get('timeout',30*60)
+        })
+    return _defaults
+
+class Boost_BuildFactory(buildbot.process.factory.BuildFactory):
+    
+    def __init__(self, *actions, **args):
+        buildbot.process.factory.BuildFactory.__init__(self)
+        self.actions = actions or []
+        self.options = args or {}
+        #~ --
+        self.steps = []
+        self.treeStableTimer = 5*60
+        self.buildClass = Boost_Build
+    
+    def newBuild(self):
+        b = buildbot.process.factory.BuildFactory.newBuild(self)
+        b.setOptions(self.options)
+        steps = []
+        files = []
+        for (_action,_args,_kwargs) in self.actions:
+            action_call = getattr(self,'action_%s' % _action,None)
+            if callable(action_call):
+                for k in _kwargs.keys():
+                    if _kwargs[k] == None: del _kwargs[k]
+                _kwargs.update(self.options)
+                (action_steps,action_files) = action_call(b,*_args,**_kwargs)
+                steps = steps + action_steps
+                files = files + action_files
+        b.important_files = files
+        b.setSteps(steps)
+        return b
+    
+    def action_cvs(self,b,*args,**kwargs):
+        opt = {
+            'cvsmodule'         : kwargs.get('module',"boost"),
+            'global_options'    : ["-z9"],
+            'mode'              : kwargs.get('mode',"copy"),
+            'branch'            : kwargs.get('branch','HEAD'),
+            'cvsroot'           : kwargs.get('root')
+            }
+        if kwargs.has_key('passwd'):
+            opt['login'] = kwargs['passwd'] or ""
+        opt.update(defaults(**kwargs))
+        return (
+            [ s(buildbot.process.step.CVS,**opt) ],
+            kwargs.get('files',[".*"]) )
+
+    def action_tarball(self,b,*args,**kwargs):
+        return (
+            [ s( boost.buildbot.step.Tarball
+                ,description = kwargs.get('description')
+                ,archive = kwargs.get('archive',b.workdir)
+                ,publishdir = kwargs['publishdir']
+                ,branch = kwargs.get('branch','HEAD')
+                ,**defaults(**kwargs)
+                ) ],
+            kwargs.get('files',[]) )
+
+    def action_selfupdate(self,b,*args,**kwargs):
+        return (
+            [ s( boost.buildbot.step.SelfUpdate
+                ,description = kwargs.get('description')
+                ,**defaults(**kwargs)
+                ) ],
+            kwargs.get('files',[]) )
+    
+    def action_bjam_build(self,b,*args,**kwargs):
+        return (
+            [ s( boost.buildbot.step.Boost_Jam_Build
+                ,description = kwargs.get('description')
+                ,workdir = b.workdir
+                ,jam_src = kwargs.get('jam_src','tools/build/jam_src')
+                ,toolset = kwargs.get('toolset',None)
+                ,**defaults(**kwargs)
+                ) ],
+            kwargs.get('files',[]) )
+    
+    def action_bjam(self,b,*args,**kwargs):
+        return (
+            [ s( boost.buildbot.step.Boost_Jam
+                ,description = kwargs.get('description')
+                ,workdir = b.workdir
+                ,bjam = kwargs.get('bjam','tools/build/jam_src/bin/bjam')
+                ,project = kwargs.get('project','.')
+                ,options = kwargs.get('options',[])
+                ,target = kwargs.get('target','all')
+                ,locate = kwargs.get('locate','build')
+                ,env = kwargs.get('env',{})
+                ,logfile = kwargs.get('logfile',False)
+                ,**defaults(**kwargs)
+                ) ],
+            kwargs.get('files',[]) )
+    
+    def action_test_tools_build(self,b,*args,**kwargs):
+        return self.action_bjam( b
+            ,description = kwargs.get('description',['test tools','build'])
+            ,project = 'tools/regression/build'
+            ,options = [
+                '-sBUILD=release',
+                '-sTOOLS=%s' % kwargs['toolset']
+                ] + kwargs.get('options',[])
+            ,target = 'run'
+            ,locate = kwargs.get('locate','build')
+            ,env = kwargs.get('env',{})
+            ,**defaults(**kwargs)
+            )
+
+    def action_btest(self,b,*args,**kwargs):
+        return (
+            [ s( boost.buildbot.step.Boost_Test
+                ,description = kwargs.get('description')
+                ,workdir = b.workdir
+                ,tests = kwargs.get('tests',['.*'])
+                ,bjam = kwargs.get('bjam','tools/build/jam_src/bin/bjam')
+                ,project = kwargs.get('project','status')
+                ,options = kwargs.get('options',[
+                    '--dump-tests',
+                    '--dump-test-targets',
+                    '-sBUILD=%s' % kwargs.get('build','debug'),
+                    '-sTOOLS=%s' % kwargs['toolset']
+                    ] + kwargs.get('options',[]))
+                ,target = 'nothing'
+                ,locate = kwargs.get('locate','build')
+                ,env = kwargs.get('env',{})
+                ,logfile = kwargs.get('logfile','bjam.log')
+                ,**defaults(**kwargs)
+                ) ],
+            kwargs.get('files',[]) )
+
+    def action_btest_all(self,b,*args,**kwargs):
+        return self.action_bjam( b
+            ,description = kwargs.get('description',['btest','all'])
+            ,project = kwargs.get('project','status')
+            ,options = [
+                '--dump-tests',
+                '--dump-test-targets',
+                '-sBUILD=%s' % kwargs.get('build','debug'),
+                '-sTOOLS=%s' % kwargs['toolset']
+                ] + kwargs.get('options',[])
+            ,target = 'test'
+            ,locate = kwargs.get('locate','build')
+            ,env = kwargs.get('env',{})
+            ,logfile = kwargs.get('logfile','bjam.log')
+            ,files = kwargs.get('files',['boost.*','libs.*','status.*'])
+            ,**defaults(**kwargs)
+            )
+
+    def action_process_jam_log(self,b,*args,**kwargs):
+        return (
+            [ s( boost.buildbot.step.Boost_Process_Jam_Log
+                ,description = kwargs.get('description',['process log'])
+                ,workdir = b.workdir
+                ,projcess_jam_log = kwargs.get('projcess_jam_log','tools/regression/build/run/process_jam_log')
+                ,locate = kwargs.get('locate','build')
+                ,logfile = kwargs.get('logfile','bjam.log')
+                ,**defaults(**kwargs)
+                ) ],
+            kwargs.get('files',[]) )
+    
+    def action_collect_results(self,b,*args,**kwargs):
+        return (
+            [ s( boost.buildbot.step.Boost_Collect_Results
+                ,description = kwargs.get('description')
+                ,workdir = b.workdir
+                ,locate = kwargs.get('locate',b.options.get('locate','build'))
+                ,runner = kwargs['runner']
+                ,branch = kwargs['branch']
+                ,source_type = kwargs['source_type']
+                ,**defaults(**kwargs)
+                ) ],
+            kwargs.get('files',[]) )
+    
+    def action_publish_results(self,b,*args,**kwargs):
+        return (
+            [ s( boost.buildbot.step.Boost_Publish_Results
+                ,description = kwargs.get('description')
+                ,workdir = b.workdir
+                ,locate = kwargs.get('locate',b.options.get('locate','build'))
+                ,runner = kwargs['runner']
+                ,branch = kwargs['branch']
+                ,source_type = kwargs['source_type']
+                ,publish_location = kwargs['publish_location']
+                ,proxy = kwargs.get('proxy')
+                ,**defaults(**kwargs)
+                ) ],
+            kwargs.get('files',[]) )
+
+class Boost_Build(buildbot.process.base.Build):
+    
+    def __init__(self):
+        buildbot.process.base.Build.__init__(self)
+        self.important_files = []
+        self.important_re = None
+    
+    def isFileImportant(self, filename):
+        if self.important_re == None:
+            self.important_re = []
+            for file in self.important_files:
+                self.important_re.append(re.compile(file))
+        for file_re in self.important_re:
+            if file_re.search(filename):
+                return 1;
+        return 0
+    
+    def setOptions(self,options = {}):
+        self.options = options or {}
+        self.workdir = self.options.get('workdir','build')
+
+    def setupBuild(self, expectations):
+        #~ Hack the stamp as an allowed arg for steps.
+        if 'stamp' not in buildbot.process.step.BuildStep.parms:
+            buildbot.process.step.BuildStep.parms.append('stamp')
+        
+        return buildbot.process.base.Build.setupBuild(self,expectations)
+    
+    def getNextStep(self):
+        s = buildbot.process.base.Build.getNextStep(self)
+        if s:
+            #~ Add a stamp arg for the steps to use as needed.
+            stamp = self._get_stamp()
+            s.stamp = stamp
+            if hasattr(s,'cmd'):
+                if hasattr(s.cmd,'args'):
+                    s.cmd.args.update( { 'stamp' : stamp } )
+        return s
+    
+    def _get_stamp(self):
+        #~ The default is to use the revision sequence as the "time".
+        #~ If not available, because of a forced build for example, we 
+        #~ use the current time.
+        stamp = time.strftime( '%Y-%m-%dT%H:%M:%S', time.gmtime() )
+        revision, patch = self.getSourceStamp()
+        if not revision:
+            changes = self.allChanges()
+            if changes:
+                last_change_time = max([c.when for c in changes])
+                last_change_revision = max([c.revision for c in changes])
+                #~ Prefer using the revision change if present. If it's not
+                #~ it's likely a CVS like time sequence, so use the time sequence
+                #~ int that case (adjusted with the tree timer).
+                if last_change_revision:
+                    stamp = last_change_revision
+                else:
+                    stamp = time.strftime( '%Y-%m-%dT%H:%M:%S',
+                        time.gmtime(last_change_time + self.treeStableTimer / 2) )
+        return stamp

tools/buildbot/src/boost/buildbot/remote.py

@@ -0,0 +1,520 @@
+
+# Copyright Redshift Software, Inc. 2005
+#
+# Distributed under the Boost Software License, Version 1.0. 
+# (See accompanying file LICENSE_1_0.txt or copy at 
+# http://www.boost.org/LICENSE_1_0.txt)
+
+import boost.buildbot.char_translation_table
+import ftplib
+import platform
+import re
+import os
+import os.path
+import shutil
+import string
+import sys
+import tarfile
+import urlparse
+import xml.sax.saxutils
+import zipfile
+
+from buildbot.slave.commands import Command, AbandonChain, ShellCommand
+from buildbot.slave.registry import registerSlaveCommand
+from twisted.internet import reactor, defer
+from twisted.python import failure, log, runtime
+
+cvs_ver = '$Revision$'[1+len("Revision: "):-2]
+
+class LoggedShellCommand(ShellCommand):
+    
+    def __init__(self, builder, command, workdir, **kwargs):
+        ShellCommand.__init__(self,builder,command,workdir
+            ,environ = kwargs.get('environ',{})
+            ,sendStdout = kwargs.get('sendStdout',True)
+            ,sendStderr = kwargs.get('sendStderr',True)
+            ,sendRC = kwargs.get('sendRC',True)
+            ,timeout = kwargs.get('timeout',None)
+            ,stdin = kwargs.get('stdin',None)
+            ,keepStdout = kwargs.get('keepStdout',False)
+            )
+        self.logfile = None
+        logfile = kwargs.get('logfile')
+        if logfile:
+            logdir = os.path.dirname(logfile)
+            if not os.path.exists(logdir):
+                os.makedirs(logdir)
+            if kwargs.get('appendToLog',False) and os.path.exists(logfile):
+                self.logfile = file(logfile,"a")
+            else:
+                self.logfile = file(logfile,"w")
+    
+    def addStdout(self, data):
+        ShellCommand.addStdout(self,data)
+        if self.logfile: self.logfile.write(data)
+    
+    def addStdout(self, data):
+        ShellCommand.addStdout(self,data)
+        if self.logfile: self.logfile.write(data)
+    
+    def finished(self, sig, rc):
+        if self.logfile: self.logfile.close()
+        ShellCommand.finished(self,sig,rc)
+
+def c(callback, *args, **kwargs):
+    args = args or []
+    kwargs = kwargs or {}
+    return (callback,args,kwargs)
+
+class NoOpCommand(Command):
+
+    def start(self):
+        return self._start("noop",c(self.doNoOp))
+
+    def doNoOp(self):
+        self.stdout("do noop")
+        return 0
+
+    def stdout(self, message):
+        self.sendStatus({'stdout': message+"\n"})
+
+    def interrupt(self):
+        self.interrupted = True
+
+    def _start(self, name, *callbacks):
+        d = defer.Deferred()
+        self.stdout("starting %s operation" % name)
+        self.name = name
+        self.command = None
+        for call,args,kwargs in callbacks:
+            d.addCallbacks(self._do_call,None,[call]+args,kwargs)
+            d.addCallback(self._result_check)
+        d.addCallbacks(self._success,self._failure)
+        reactor.callLater(2,d.callback,0)
+        return d
+
+    def _do_call(self, rc, call, *args, **kwargs):
+        return call(*args,**kwargs)
+    
+    def _result_check(self, rc):
+        if self.interrupted:
+            raise AbandonChain(-1)
+        if rc != 0:
+            raise AbandonChain(rc)
+        return 0
+
+    def _success(self, rc):
+        self.sendStatus({'rc': 0})
+        return None
+
+    def _failure(self, fail):
+        fail.trap(AbandonChain)
+        self.sendStatus({'rc': fail.value.args[0]})
+        return None
+
+registerSlaveCommand("noop", NoOpCommand, cvs_ver)
+
+class SelfUpdateCommand(NoOpCommand):
+
+    def start(self):
+        return self._start("selfupdate",c(self.doUpdateCommandRegistry))
+    
+    def doUpdateCommandRegistry(self):
+        import buildbot.slave.registry
+        import buildbot.slave.commands
+        import boost.buildbot.remote
+
+        self.stdout("updating command registry")
+        reload(buildbot.slave.registry)
+        self.stdout("reloading standard commands")
+        reload(buildbot.slave.commands)
+        self.stdout("reloading boost commands")
+        reload(boost.buildbot.remote)
+        self.stdout("command registry update complete")
+
+        self.stdout("commands:")
+        for name, (factory, version) in buildbot.slave.registry.commandRegistry.items():
+            self.stdout("  %s (%s)" % (name,version))
+
+        return 0
+
+registerSlaveCommand("selfupdate", SelfUpdateCommand, cvs_ver)
+
+class TarballCommand(NoOpCommand):
+
+    def start(self):
+        stamp = self.args.get('stamp','')
+        stamp = stamp.replace(' ','-')
+        stamp = stamp.replace(':','_')
+        archive_stamped = os.path.normpath(os.path.join(self.builder.basedir,
+            "%s-%s-%s" % (self.args['archive'],self.args.get('branch','X'),stamp)))
+        return self._start( "tarball",
+            c( self.doCleanRepository,
+                repository = os.path.normpath(os.path.join(self.builder.basedir, self.args['workdir'])) ),
+            c( self.doArchive,
+                source = os.path.normpath(os.path.join(self.builder.basedir, self.args['workdir'])),
+                archive = archive_stamped ),
+            c( self.doPublish,
+                archive = archive_stamped,
+                publishdir = os.path.normpath(self.args['publishdir']) ) )
+    
+    def doCleanRepository(self,*args,**kwargs):
+        
+        self.stdout("cleaning repository at %s..." % kwargs['repository'])
+        
+        self._clean_r(kwargs['repository'])
+        return 0
+    
+    def doArchive(self,*args,**kwargs):
+        source_path = kwargs['source']
+        archive_path = "%s.tar.bz2" % kwargs['archive']
+        archive_dir = os.path.basename( kwargs['archive'] )
+        
+        self.stdout("creating archive %s for %s" % ( archive_path, source_path ))
+        
+        previous_umask = os.umask(0022)
+        tar = tarfile.open(archive_path, 'w:bz2')
+        #~ Disabling posix allows for longer names and hence deeper directories. 
+        tar.Posix = False
+        tar.add(source_path, archive_dir)
+        tar.close()
+        os.umask(previous_umask)
+        
+        return 0
+    
+    def doPublish(self,*args,**kwargs):
+        archive_path = "%s.tar.bz2" % kwargs['archive']
+        
+        self.stdout("publishing archive %s to %s" % ( archive_path, kwargs['publishdir'] ))
+        
+        previous_umask = os.umask(0022)
+        try:
+            os.makedirs(kwargs['publishdir'],0755)
+        except:
+            pass
+        #~ shutil.move is available on py2.3, consider copy/rename implementation to
+        #~ support py2.2. Or possibly do an external async "mv" command.
+        shutil.move(archive_path,kwargs['publishdir'])
+        self._clean_archives( kwargs['publishdir'], '[^\.]+\.tar\.bz2',
+            ( os.path.basename(archive_path) ) )
+        os.umask(previous_umask)
+        return 0
+    
+    def _clean_r(self,dir):
+        names = os.listdir(dir)
+        names.sort()
+        for name in names:
+            entry = os.path.join(dir,name)
+            if name == 'CVS':
+                self.stdout("[REMOVE] %s" % entry)
+                shutil.rmtree( entry )
+            elif os.path.isdir(entry):
+                self._clean_r(entry)
+    
+    def _clean_archives(self,dir,m,exclude):
+        m_re = re.compile(m)
+        names = os.listdir(dir)
+        names.sort()
+        for name in names:
+            if m_re.search(name) and name not in exclude:
+                entry = os.path.join(dir,name)
+                self.stdout("[REMOVE] %s" % entry)
+                os.remove( entry )
+
+registerSlaveCommand("tarball", TarballCommand, cvs_ver)
+
+class Command_Boost_Jam_Build(NoOpCommand):
+
+    def start(self):
+        return self._start( "boost.bjam.build",
+            c( self.doBJamBuild,
+                jam_src = os.path.normpath(os.path.join(
+                    self.builder.basedir, self.args['workdir'], self.args['jam_src'])),
+                toolset = self.args.get('toolset',None),
+                timeout = self.args.get('timeout',60*5))
+            )
+    
+    def doBJamBuild(self,*args,**kwargs):
+        self.stdout("building bjam at %s..." % kwargs['jam_src'])
+        if runtime.platformType != 'posix':
+            command = [ '.\build.bat' ]
+        else:
+            command = [ 'sh', './build.sh' ]
+        if kwargs['toolset']:
+            command.append(kwargs['toolset'])
+        self.command = ShellCommand(self.builder, command,
+            kwargs['jam_src'], { 'LOCATE_TARGET' : 'bin' },
+            sendRC = False, timeout = kwargs['timeout'] )
+        return self.command.start()
+
+registerSlaveCommand("boost.jam.build", Command_Boost_Jam_Build, cvs_ver)
+
+class Command_Boost_Jam(NoOpCommand):
+
+    def start(self):
+        _env = self.args.get('env',{})
+        _env.update({
+            'ALL_LOCATE_TARGET': os.path.normpath(os.path.join(
+                self.builder.basedir,self.args.get('locate','build'))),
+            'BOOST_BUILD_PATH': "%s:%s:%s" % (
+                os.path.normpath(self.builder.basedir),
+                os.path.normpath(os.path.join(self.builder.basedir,'..')),
+                _env.get('BOOST_BUILD_PATH','.') )
+            })
+        _logfile = False
+        if self.args.get('logfile'):
+            _logfile = os.path.normpath(os.path.join(
+                _env['ALL_LOCATE_TARGET'],self.args['logfile']))
+        return self._start( "boost.bjam",
+            c( self.doBJam
+                ,bjam = os.path.normpath(os.path.join(self.builder.basedir,
+                    self.args['workdir'], self.args['bjam']))
+                ,project = os.path.normpath(os.path.join(self.builder.basedir, 
+                    self.args['workdir'], self.args.get('project','.')))
+                ,options = self.args.get('options',[])
+                ,target = self.args.get('target','all')
+                ,env = _env
+                ,logfile = _logfile
+                ,appendToLog = self.args.get('appendToLog',False)
+                ,timeout = self.args.get('timeout',60*5)
+                )
+            )
+    
+    def doBJam(self,*args,**kwargs):
+        self.stdout("bjam %s..." % kwargs['target'])
+        self.stdout("  env:")
+        env = os.environ.copy()
+        env.update(kwargs['env'])
+        for item in env.items():
+            self.stdout("    %s = '%s'" % item)
+        
+        command = [ kwargs['bjam'] ] + kwargs['options'] + [ kwargs['target'] ]
+        self.command = LoggedShellCommand(self.builder
+            ,command
+            ,kwargs['project']
+            ,environ = kwargs['env']
+            ,sendRC = False
+            ,timeout = kwargs['timeout']
+            ,logfile = kwargs['logfile']
+            ,appendToLog = kwargs['appendToLog']
+            )
+        return self.command.start()
+
+registerSlaveCommand("boost.jam", Command_Boost_Jam, cvs_ver)
+
+class Command_Boost_ProcessJamLog(NoOpCommand):
+    
+    def start(self):
+        return self._start( "boost.process_jam_log"
+            ,c( self.doProcessJamLog
+                ,process_jam_log = os.path.normpath(os.path.join(
+                    self.builder.basedir,self.args.get('locate','build'),
+                    self.args.get('process_jam_log','tools/regression/build/run/process_jam_log')))
+                ,boostroot = os.path.normpath(os.path.join(
+                    self.builder.basedir,self.args.get('boostroot',self.args.get('workdir','.'))))
+                ,logfile = os.path.normpath(os.path.join(
+                    self.builder.basedir,self.args.get('locate','build'),
+                    self.args.get('logfile','bjam.log')))
+                ,locate = os.path.normpath(os.path.join(
+                    self.builder.basedir,self.args.get('locate','build')))
+                ,timeout = self.args.get('timeout',60*15)
+                )
+            )
+    
+    def doProcessJamLog(self,*args,**kwargs):
+        self.stdout("processing the regression test results...")
+        if runtime.platformType != 'posix':
+            command = 'type "%s" | "%s" "%s"' % (kwargs['logfile'], kwargs['process_jam_log'], kwargs['locate'])
+        else:
+            command = 'cat "%s" | "%s" "%s"' % (kwargs['logfile'], kwargs['process_jam_log'], kwargs['locate'])
+        self.command = ShellCommand(self.builder
+            ,command
+            ,kwargs['boostroot']
+            ,timeout = kwargs['timeout']
+            )
+        return self.command.start()
+
+registerSlaveCommand("boost.process_jam_log", Command_Boost_ProcessJamLog, cvs_ver)
+
+class Command_Boost_CollectResults(NoOpCommand):
+
+    def start(self):
+        return self._start( "boost.collect_results",
+            c( self.doCollectResults
+                ,results = os.path.normpath(os.path.join(
+                    self.builder.basedir,self.args.get('locate','build'),
+                    '%s.xml' % self.args['runner']))
+                ,locate = os.path.normpath(os.path.join(
+                    self.builder.basedir,self.args.get('locate','build')))
+                ,runner = self.args['runner']
+                ,timestamp = string.replace(self.args['stamp'],'T',' ')
+                ,tag = '%s-%s' % (self.args['source_type'],self.args['branch'])
+                ,source = self.args['source_type']
+                ,comments = self.args.get('comments',
+                    os.path.normpath(os.path.join(self.builder.basedir,'..','comments.html')))
+                ,platform = self.args.get('platform',platform.system())
+                ,timeout = self.args.get('timeout',60*15)
+                ),
+            c( self.doZipArchive
+                ,source = os.path.normpath(os.path.join(
+                    self.builder.basedir,self.args.get('locate','build'),
+                    '%s.xml' % self.args['runner']))
+                ,archive = os.path.normpath(os.path.join(
+                    self.builder.basedir,self.args.get('locate','build'),
+                    '%s.zip' % self.args['runner']))
+                ,timeout = self.args.get('timeout',60*15)
+                )
+            )
+    
+    def doCollectResults(self,*args,**kwargs):
+        self.stdout("collecting the regression test results...")
+        result = 0
+        previous_umask = os.umask(0022)
+        results_writer = open( kwargs['results'], 'w' )
+        self.stdout( 'Collecting test logs into "%s"...' % kwargs['results'] )
+        
+        results_xml = xml.sax.saxutils.XMLGenerator( results_writer )
+        results_xml.startDocument()
+        results_xml.startElement( 'test-run' ,{
+            'tag': kwargs['tag']
+            ,'platform': kwargs['platform']
+            ,'runner': kwargs['runner']
+            ,'timestamp': kwargs['timestamp']
+            ,'source': kwargs['source']
+            ,'run-type': 'incremental'
+            })
+        
+        self._copy_comments( results_xml, kwargs['comments'] )
+        self._collect_test_logs( [ kwargs['locate'] ], results_writer )
+    
+        results_xml.endElement( "test-run" )
+        results_xml.endDocument()
+        results_writer.close()
+        self.stdout( 'Done writing "%s".' % kwargs['results'] )
+        os.umask(previous_umask)
+        return result
+
+    def _copy_comments(self,results_xml,comment_file):
+        results_xml.startElement( 'comment', {} )
+    
+        if os.path.exists( comment_file ):
+            self.stdout( 'Reading comments file "%s"...' % comment_file )
+            f = open( comment_file, 'r' )
+            try:
+                results_xml.characters( f.read() )
+            finally:
+                f.close()    
+        else:
+            self.stdout( 'Warning: comment file "%s" is not found.' % comment_file )
+     
+        results_xml.endElement( 'comment' )
+
+    def _collect_test_logs(self,input_dirs,test_results_writer):
+        self.stdout( 'Collecting test logs ...' )
+        for input_dir in input_dirs:
+            self.stdout( 'Walking directory "%s" ...' % input_dir )
+            os.path.walk( input_dir, self._process_test_log_files, test_results_writer )
+
+    def _process_test_log_files(self,output_file,dir,names):
+        for file in names:
+            if os.path.basename( file ) == 'test_log.xml':
+                self._process_xml_file( os.path.join( dir, file ), output_file )
+
+    def _process_xml_file(self,input_file,output_file):
+        self.stdout( 'Processing test log "%s"' % input_file )
+        
+        f = open( input_file, 'r' )
+        xml = f.readlines()
+        f.close()
+        
+        for i in range( 0, len(xml)):
+            xml[i] = string.translate( xml[i], boost.buildbot.char_translation_table.char_translation_table )
+    
+        output_file.writelines( xml )
+    
+    def doZipArchive(self,*args,**kwargs):
+        source_path = kwargs['source']
+        archive_path = kwargs['archive']
+        self.stdout("creating archive %s for %s" % ( archive_path, source_path ))
+        result = 0
+        previous_umask = os.umask(0022)
+        try:
+            z = zipfile.ZipFile( archive_path, 'w', zipfile.ZIP_DEFLATED )
+            z.write( source_path, os.path.basename( source_path ) )
+            z.close()
+            self.stdout( 'Done writing "%s".'% archive_path )
+        except Exception, msg:
+            self.stdout( 'Warning: Compressing failed (%s)' % msg )
+            self.stdout( '         Trying to compress using a platform-specific tool...' )
+            try: import zip_cmd
+            except ImportError:
+                script_dir = os.path.dirname( os.path.abspath( sys.argv[0] ) )
+                self.stdout( 'Could not find \'zip_cmd\' module in the script directory (%s).' % script_dir )
+                result = -1
+            else:
+                if os.path.exists( archive_path ):
+                    os.unlink( archive_path )
+                    self.stdout( 'Removing stale "%s".' % archive_path )
+                    
+                zip_cmd.main( source_path, archive_path )
+                self.stdout( 'Done compressing "%s".' % archive_path )
+        os.umask(previous_umask)
+        return result
+
+registerSlaveCommand("boost.collect_results", Command_Boost_CollectResults, cvs_ver)
+
+class Command_Boost_PublishResults(NoOpCommand):
+
+    def start(self):
+        return self._start( "boost.publish_results",
+            c( self.doPublish
+                ,source = os.path.normpath(os.path.join(
+                    self.builder.basedir,self.args.get('locate','build'),
+                    '%s.zip' % self.args['runner']))
+                ,target = '%s/%s-%s' % (self.args['publish_location'],self.args['source_type'],self.args['branch'])
+                ,proxy = self.args.get('proxy')
+                ,timeout = self.args.get('timeout',60*15)
+                )
+            )
+    
+    def doPublish(self,*args,**kwargs):
+        self.stdout("publishing the regression test results...")
+        result = 0
+        
+        (scheme,site,path,query,fragment) = urlparse.urlsplit(kwargs['target'])
+        publish_call = getattr(self,'_publish_%s' % scheme,None)
+        if callable(publish_call):
+            result = publish_call(scheme,site,path,query,fragment,**kwargs)
+        else:
+            self.stdout('unknown publish method "%s"' % scheme)
+            result = -1
+        
+        return result
+
+    def _publish_ftp(self,scheme,site,path,query,fragment,**kwargs):
+        self.stdout( 'Uploading log archive "%s" to %s' % ( kwargs['source'], kwargs['target'] ) )
+        
+        if not kwargs['proxy']:
+            ftp = ftplib.FTP( site )
+            ftp.set_debuglevel( 1 )
+            ftp.login()
+        else:
+            utils.log( '    Connecting through FTP proxy server "%s"' % kwargs['proxy'] )
+            ftp = ftplib.FTP( kwargs['proxy'] )
+            ftp.set_debuglevel( 1 )
+            ftp.set_pasv (0) # turn off PASV mode
+            ftp.login( 'anonymous@%s' % site, 'anonymous@' )
+        
+        ftp.cwd( os.path.dirname(path) )
+        try:
+            ftp.cwd( os.path.basename(path) )
+        except ftplib.error_perm:
+            ftp.mkd( os.path.basename(path) )
+            ftp.cwd( os.path.basename(path) )
+    
+        f = open( kwargs['source'], 'rb' )
+        ftp.storbinary( 'STOR %s' % os.path.basename( kwargs['source'] ), f )
+        ftp.quit()
+        return 0
+
+registerSlaveCommand("boost.publish_results", Command_Boost_PublishResults, cvs_ver)

tools/buildbot/src/boost/buildbot/script.py

@@ -0,0 +1,185 @@
+
+# Copyright Redshift Software, Inc. 2005-2007
+#
+# Distributed under the Boost Software License, Version 1.0. 
+# (See accompanying file LICENSE_1_0.txt or copy at 
+# http://www.boost.org/LICENSE_1_0.txt)
+
+import os
+import os.path
+import sys
+import getopt
+import re
+import boost
+
+
+def show_revision( **unused ):
+    re_keyword_value = re.compile( r'^\$\w+:\s+(.*)\s+\$$' )
+    print '\n\tResivion: %s' % re_keyword_value.match( boost.buildbot.revision ).group( 1 )
+    print '\tLast modified on: %s\n' % re_keyword_value.match( boost.buildbot.modified ).group( 1 )
+    sys.exit(0)
+
+
+def create_tester( root, server, runner, passwd, debug_level, **unused ):
+    import twisted.scripts.mktap
+
+    root = os.path.abspath(root)
+    if os.path.exists(root):
+        print "Testing root location %s exists." % root
+        print "Skipping to prevent corruption of existing setup."
+        sys.exit(1)
+    if not os.path.exists(root):
+        if debug_level > 0: print "mkdir", root
+        os.mkdir(root)
+    if debug_level > 0: print "chdir", root
+    os.chdir(root)
+    sys.argv = [
+        'mktap', 'buildbot', 'slave',
+        '--basedir', root,
+        '--master', server,
+        '--name', runner,
+        '--passwd', passwd
+        ]
+    if debug_level > 0: print ' '.join( sys.argv )
+    twisted.scripts.mktap.run()
+    if debug_level > 0: print "Tester configured in %s." % root
+    sys.exit(0)
+
+def create_server( root, debug_level, **unused ):
+    import twisted.scripts.mktap
+
+    root = os.path.abspath(root)
+    if os.path.exists(root):
+        print "Testing root location %s exists." % root
+        print "Skipping to prevent corruption of existing setup."
+        sys.exit(1)
+    if not os.path.exists(root):
+        if debug_level > 0: print "mkdir", root
+        os.mkdir(root)
+    if debug_level > 0: print "chdir", root
+    os.chdir(root)
+    sys.argv = [
+        'mktap', 'buildbot', 'master',
+        '--basedir', root
+        ]
+    if debug_level > 0: print ' '.join( sys.argv )
+    twisted.scripts.mktap.run()
+    if debug_level > 0: print "Server configured in %s." % root
+    sys.exit(0)
+
+def start_daemon( root, debug_level, **unused ):
+    import twisted.python.runtime
+    
+    # import the various built in slave commands so that we can add our own
+    import buildbot.slave.registry
+    import buildbot.slave.commands
+    import boost.buildbot.remote
+    
+    root = os.path.abspath(root)
+    if debug_level > 0: print "chdir", root
+    os.chdir(root)
+    sys.argv = [
+        'twistd',
+        '--no_save',
+        '--file=buildbot.tap'
+        ]
+    if sys.platform == "win32":
+        sys.arvg.append("--reactor=win32")
+    if debug_level > 0: print ' '.join( sys.argv )
+    if twisted.python.runtime.platformType == "Win32":
+        import twisted.scripts.twistw
+        twisted.scripts.twistw.run()
+    else:
+        import twisted.scripts.twistd
+        twisted.scripts.twistd.run()
+    sys.exit(0)
+
+def stop_daemon( root, debug_level, **unused ):
+    import signal
+    
+    twistd_pid_file = os.path.join(root,'twistd.pid')
+    if os.path.isfile(twistd_pid_file):
+        twistd_pid = file(twistd_pid_file,'r').read()
+        os.kill(int(re.search(r'^(\d+)',twistd_pid).group(1)),signal.SIGTERM);
+        sys.exit(0)
+    else:
+        sys.exit(1)
+
+def accept_args( args ):
+    args_spec = [
+        'root=',
+        'server=',
+        'runner=',
+        'passwd=',
+        ##
+        'debug-level=',
+        'help'
+        ]
+    
+    options = {
+        '--root'            : None,
+        '--server'          : None,
+        '--runner'          : None,
+        '--passwd'          : None,
+        ##
+        '--debug-level'     : 0
+        }
+    
+    ( option_pairs, other_args ) = getopt.getopt( args, '', args_spec )
+    map( lambda x: options.__setitem__( x[0], x[1] ), option_pairs )
+
+    if options.has_key( '--help' ):
+        usage()
+        sys.exit( 1 )
+
+    return {
+        'root'              : options[ '--root' ],
+        'server'            : options[ '--server' ],
+        'runner'            : options[ '--runner' ],
+        'passwd'            : options[ '--passwd' ],
+        ##
+        'debug_level'       : int(options[ '--debug-level' ]),
+        'args'              : other_args
+        }
+
+
+commands = {
+    'show-revision'         : show_revision,
+    'create-tester'         : create_tester,
+    'create-server'         : create_server,
+    'start'                 : start_daemon,
+    'stop'                  : stop_daemon
+    }
+
+def lp( l ):
+    print l;
+
+def usage():
+    lp('Usage:')
+    lp('')
+    lp('python %s [command] options' % os.path.basename( sys.argv[0] ))
+    lp('')
+    lp('Commands:')
+    lp('')
+    lp('\n'.join( commands.keys() ))
+    lp('')
+    lp('Options:')
+    lp('')
+    lp('--root          Directory of server or runner.')
+    lp('--server        The server address for the runner to connect to')
+    lp('                in the for of DNSNAME:PORT.')
+    lp('--runner        The name of the runner.')
+    lp('--passwd        The password for the runner to connect ro the server.')
+    lp('--debug-level   Debugging level; controls the amount of debugging')
+    lp('                output printed; 0 by default (no debug output).')
+    lp('')
+
+def run():
+    if len(sys.argv) > 1 and sys.argv[1] in commands:
+        command = sys.argv[1]
+        args = sys.argv[ 2: ]
+    else:
+        command = 'show-revision'
+        args = sys.argv[ 1: ]
+        
+    commands[ command ]( **accept_args( args ) )

tools/buildbot/src/boost/buildbot/server.py

@@ -0,0 +1,415 @@
+
+# Copyright Redshift Software, Inc. 2005-2007
+#
+# Distributed under the Boost Software License, Version 1.0. 
+# (See accompanying file LICENSE_1_0.txt or copy at 
+# http://www.boost.org/LICENSE_1_0.txt)
+
+import buildbot
+import buildbot.changes.changes
+import buildbot.changes.mail
+import buildbot.status.builder
+import buildbot.status.html
+import buildbot.util
+import email.Utils
+import os.path
+import re
+import rfc822
+import string
+import time
+import types
+import twisted.python
+import twisted.python.components
+import twisted.web.static
+import urllib
+
+waterfall_content_html = '''<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+  <title>BuildBot: %(project_name)s</title>
+  <link href="buildbot.css" rel="stylesheet" type="text/css" />
+</head>
+
+<body>
+%(heading)s
+%(body)s
+%(footer)s
+</body>
+</html>
+'''
+
+waterfall_body_html = '''
+  <table id="waterfall">
+    <tr id="builders">
+      <td colspan="2" class="project">
+        <a href="%(project_url)s">%(project_name)s</a>
+      </td>
+      
+      %(builders)s
+    </tr>
+
+    <tr id="current-activity">
+      <td colspan="2" class="heading">
+        CURRENT&nbsp;ACTIVITY
+      </td>
+      
+      %(current_activity)s
+    </tr>
+
+    <tr id="last-activity">
+      <td class="heading">
+        TIME %(timezone)+02.0f
+      </td>
+
+      <td class="heading">
+        <a href="changes">CHANGES</a>
+      </td>
+      
+      %(last_activity)s
+    </tr>
+    
+    %(waterfall)s
+  </table>
+'''
+
+waterfall_footer_html = '''
+  <div id="footer">
+    <p><a href="http://buildbot.sourceforge.net/">Buildbot</a>-%(version)s working
+    for the <a href="%(project_url)s">%(project_name)s</a> project.</p>
+
+    <p>Page built: %(page_time)s</p>
+  </div>
+'''
+
+class Boost_WaterfallStatusResource(buildbot.status.html.WaterfallStatusResource):
+    
+    def __init__(self, status, changemaster, categories, css=None):
+        buildbot.status.html.WaterfallStatusResource.__init__(self,status,changemaster,categories,css)
+        
+    def content(self, request):
+        self.page_time = time.strftime("%a %d %b %Y %H:%M:%S",time.localtime(buildbot.util.now()))
+        return waterfall_content_html % {
+            "project_name"      : self.status.getProjectName(),
+            "project_url"       : self.status.getProjectURL(),
+            "page_time"         : self.page_time,
+            "heading"           : self.heading(request),
+            "body"              : self.body(request),
+            "footer"            : self.footer(request) }
+
+    def heading(self, request):
+        return ""
+
+    def body(self, request):
+        "This method builds the main waterfall display."
+        phase = request.args.get("phase",["2"])
+        phase = int(phase[0])
+
+        showBuilders = request.args.get("show", None)
+        allBuilders = self.status.getBuilderNames(categories=self.categories)
+        if showBuilders:
+            builderNames = []
+            for b in showBuilders:
+                if b not in allBuilders:
+                    continue
+                if b in builderNames:
+                    continue
+                builderNames.append(b)
+        else:
+            builderNames = allBuilders
+        builders = map(
+            lambda name: self.status.getBuilder(name),
+            builderNames)
+
+        if phase == -1:
+            return self.body0(request, builders)
+        
+        (changeNames, builderNames, timestamps, eventGrid, sourceEvents) = self.buildGrid(request, builders)
+        if phase == 0:
+            return self.phase0(request, changeNames, timestamps, eventGrid)
+        
+        last_activity_html = "";
+        for b in builders:
+            box = buildbot.status.html.ITopBox(b).getBox()
+            last_activity_html += box.td()
+        
+        current_activity_html = "";
+        for b in builders:
+            box = buildbot.status.html.ICurrentBox(b).getBox()
+            current_activity_html += box.td()
+        
+        builders_html = "";
+        for name in builderNames:
+            builders_html += "<td class=\"builder\"><a href=\"%s\">%s</a></td>" % (
+                urllib.quote(name),
+                string.join(string.split(name,'-'),'<br />') )
+        
+        if phase == 1:
+            f = self.phase1
+        else:
+            f = self.phase2
+        waterfall_html = f(request, changeNames+builderNames, timestamps, eventGrid, sourceEvents)
+        
+        return waterfall_body_html % {
+            "project_name"      : self.status.getProjectName(),
+            "project_url"       : self.status.getProjectURL(),
+            "last_activity"     : last_activity_html,
+            "current_activity"  : current_activity_html,
+            "builders"          : builders_html,
+            "waterfall"         : waterfall_html,
+            "version"           : buildbot.version,
+            "page_time"         : self.page_time,
+            "timezone"          : time.timezone/60
+            }
+
+    def footer(self, request):
+        return waterfall_footer_html % {
+            "project_name"      : self.status.getProjectName(),
+            "project_url"       : self.status.getProjectURL(),
+            "version"           : buildbot.version,
+            "page_time"         : self.page_time
+            }
+
+## Override some of the display elements to make them CSS friendly.
+
+def td(text="", parms={}, **props):
+    props.update(parms)
+    
+    td_props_html = "";
+    for prop in ("colspan", "rowspan", "class", "style"):
+        p = props.get(prop, None)
+        if p != None:
+            td_props_html += " %s=\"%s\"" % (prop, p)
+    
+    if type(text) == types.ListType:
+        td_text_html = "<div>%s</div>" % string.join(text, "</div><div>")
+    else:
+        td_text_html = "<div>%s</div>" % text
+    
+    return "<td%s>%s</td>\n" % (td_props_html,td_text_html)
+
+color_map = {
+    '#c000c0' : 'purple'
+    }
+def c(a_color):
+    if a_color == None:
+        return 'none'
+    elif color_map.has_key(a_color):
+        return color_map[a_color]
+    else:
+        return a_color
+
+class Boost_Box:
+
+    spacer = False
+    
+    def __init__(self, other_box, props={}):
+        self.text = other_box.text
+        self.urlbase = other_box.urlbase
+        self.show_idle = other_box.show_idle
+        self.parms = other_box.parms
+        self.parms.update(props)
+    
+    def td(self, **props):
+        props.update(self.parms)
+        text = self.text
+        if not text and self.show_idle:
+            text = ["[idle]"]
+        return td(text, props)
+
+class Boost_CurrentBox(buildbot.status.html.CurrentBox):
+
+    def getBox(self):
+        state, ETA, build = self.original.getState()
+        return Boost_Box( buildbot.status.html.CurrentBox.getBox(self),
+            { 'class': "activity-%s" % state } )
+
+twisted.python.components.theAdapterRegistry.adapterRegistry[
+    (buildbot.status.builder.BuilderStatus, buildbot.status.html.ICurrentBox)] = Boost_CurrentBox
+
+class Boost_ChangeBox(buildbot.status.html.ChangeBox):
+
+    def getBox(self):
+        return Boost_Box( buildbot.status.html.ChangeBox.getBox(self),
+            { 'class': "commit" } )
+
+twisted.python.components.theAdapterRegistry.adapterRegistry[
+    (buildbot.changes.changes.Change, buildbot.status.html.IBox)] = Boost_ChangeBox
+
+class Boost_BuildBox(buildbot.status.html.BuildBox):
+
+    def getBox(self):
+        return Boost_Box( buildbot.status.html.BuildBox.getBox(self),
+            { 'class': "build" } )
+
+twisted.python.components.theAdapterRegistry.adapterRegistry[
+    (buildbot.status.builder.BuildStatus, buildbot.status.html.IBox)] = Boost_BuildBox
+
+class Boost_StepBox(buildbot.status.html.StepBox):
+
+    def getBox(self):
+        return Boost_Box( buildbot.status.html.StepBox.getBox(self),
+            { 'class': "step-%s" % c(self.original.getColor()) } )
+
+twisted.python.components.theAdapterRegistry.adapterRegistry[
+    (buildbot.status.builder.BuildStepStatus, buildbot.status.html.IBox)] = Boost_StepBox
+
+class Boost_EventBox(buildbot.status.html.EventBox):
+
+    def getBox(self):
+        return Boost_Box( buildbot.status.html.EventBox.getBox(self),
+            { 'class': "event-%s" % c(self.original.getColor()) } )
+
+twisted.python.components.theAdapterRegistry.adapterRegistry[
+    (buildbot.status.builder.Event, buildbot.status.html.IBox)] = Boost_EventBox
+
+class Boost_BuildTopBox(buildbot.status.html.BuildTopBox):
+
+    def getBox(self):
+        box = buildbot.status.html.BuildTopBox.getBox(self)
+        return Boost_Box( box,
+            { 'class': "build-%s" % c(box.color) } )
+
+twisted.python.components.theAdapterRegistry.adapterRegistry[
+    (buildbot.status.builder.BuilderStatus, buildbot.status.html.ITopBox)] = Boost_BuildTopBox
+
+##
+
+class Boost_StatusResource(buildbot.status.html.StatusResource):
+    
+    def __init__(self, status, control, changemaster, categories, root):
+        buildbot.status.html.StatusResource.__init__(self,
+            status, control, changemaster, categories,
+            twisted.web.static.File(os.path.join(root,"buildbot.css")))
+        self.putChild("",
+            Boost_WaterfallStatusResource(self.status, self.changemaster,
+                self.categories, self.css))
+        self.putChild("buildbot.css",
+            twisted.web.static.File(os.path.join(root,"buildbot.css")))
+
+class Boost_Waterfall(buildbot.status.html.Waterfall):
+
+    root = None
+
+    def __init__(self, http_port=None, distrib_port=None, allowForce=True, root=None):
+        buildbot.status.html.Waterfall.__init__(self,http_port,distrib_port,allowForce)
+        self.root = root
+    
+    def setup(self):
+        buildbot.status.html.Waterfall.setup(self)
+        self.site.resource = Boost_StatusResource(
+            self.site.resource.status,
+            self.site.resource.control,
+            self.site.resource.changemaster,
+            self.site.resource.categories,
+            self.root)
+
+def Boost_parseSyncmail(self, fd, prefix=None, sep="/"):
+    m = rfc822.Message(fd)
+    
+    # The mail is sent from the person doing the checkin. Assume that the
+    # local username is enough to identify them (this assumes a one-server
+    # cvs-over-rsh environment rather than the server-dirs-shared-over-NFS
+    # model)
+    name, addr = m.getaddr("from")
+    if not addr:
+        return None # no From means this message isn't from FreshCVS
+    at = addr.find("@")
+    if at == -1:
+        who = addr # might still be useful
+    else:
+        who = addr[:at]
+    
+    # take the date of the email as the time of checkin, but fall back to
+    # delivery time
+    when = buildbot.util.now()
+    email_time = m.getheader("date")
+    if email_time:
+        email_time = email.Utils.parsedate_tz(email_time)
+        if email_time:
+            when = email.Utils.mktime_tz(email_time)
+    
+    # syncmail puts the repository-relative directory in the subject:
+    # "CVS: %(dir)s %(file)s,%(oldversion)s,%(newversion)s"
+    # this is the only reasonable way to determine the directory name
+    subject = m.getheader("subject")
+    bits = subject.split(" ")
+    while bits:
+        bit = bits.pop(0)
+        if bit == "CVS:":
+            break;
+    directory = bits.pop(0)
+    
+    files = []
+    comments = ""
+    isdir = 0
+    lines = m.fp.readlines()
+    while lines:
+        line = lines.pop(0)
+        if (line.find("Modified Files:") == 0 or
+            line.find("Added Files:") == 0 or
+            line.find("Removed Files:") == 0):
+            break
+    while lines:
+        line = lines.pop(0)
+        if line == "\n" or line == "\r" or line == "\r\n" or line == "\n\r":
+            break
+        if line.find("Log Message:") == 0:
+            lines.insert(0, line)
+            break
+        if (line.find("Modified Files:") == 0 or
+            line.find("Added Files:") == 0 or
+            line.find("Removed Files:") == 0):
+            continue
+        line = line.lstrip()
+        line = line.rstrip()
+        # note: syncmail will send one email per directory involved in a
+        # commit, with multiple files if they were in the same directory.
+        # Unlike freshCVS, it makes no attempt to collect all related
+        # commits into a single message.
+        thesefiles = line.split(" ")
+        for file in thesefiles:
+            file = sep.join([directory, file])
+            file = file.replace("\\",sep)
+            file = file.replace("/",sep)
+            if prefix:
+                # insist that the file start with the prefix: we may get
+                # changes we don't care about too
+                bits = file.split(sep)
+                if bits[0] == prefix:
+                    file = sep.join(bits[1:])
+                else:
+                    break
+            # TODO: figure out how new directories are described, set .isdir
+            files.append(file)
+    
+    if not files:
+        return None
+    
+    while lines:
+        line = lines.pop(0)
+        if line.find("Log Message:") == 0:
+            break
+    # message is terminated by "Index:..." (patch) or "--- NEW FILE.."
+    # or "--- filename DELETED ---". Sigh.
+    while lines:
+        line = lines.pop(0)
+        if line.find("Index: ") == 0:
+            break
+        if re.search(r"^--- NEW FILE", line):
+            break
+        if re.search(r" DELETED ---$", line):
+            break
+        comments += line
+    comments = comments.rstrip() + "\n"
+    
+    change = buildbot.changes.changes.Change(who, files, comments, isdir, when=when)
+    
+    return change
+
+class Boost_SyncmailMaildirSource(buildbot.changes.mail.SyncmailMaildirSource):
+    parser = Boost_parseSyncmail
+    def messageReceived(self, filename):
+        twisted.python.log.msg("Boost_SyncmailMaildirSource.messageReceived: "+filename)
+        buildbot.changes.mail.SyncmailMaildirSource.messageReceived(self,filename)

tools/buildbot/src/boost/buildbot/step.py

@@ -0,0 +1,132 @@
+
+# Copyright Redshift Software, Inc. 2005-2007
+#
+# Distributed under the Boost Software License, Version 1.0. 
+# (See accompanying file LICENSE_1_0.txt or copy at 
+# http://www.boost.org/LICENSE_1_0.txt)
+
+#~ import buildbot
+#~ import buildbot.process.factory
+import buildbot.process.step
+#~ import os.path
+import re
+import string
+#~ import time
+import twisted.python
+#~ import types
+#~ import urllib
+
+from buildbot.process.factory import s
+
+class command_base(buildbot.process.step.ShellCommand):
+    def __init__(self, _name, _description, **kwargs):
+        if kwargs.get('name'): _name = kwargs.get('name')
+        if kwargs.get('description'): _description = kwargs.get('description')
+        
+        buildbot.process.step.ShellCommand.__init__(self,**kwargs)
+        
+        if kwargs.has_key('name'): del kwargs['name']
+        if kwargs.has_key('description'): del kwargs['description']
+        if kwargs.has_key('build'): del kwargs['build']
+        
+        self.name = _name
+        self.description = _description
+        self.cmd = buildbot.process.step.LoggedRemoteCommand(_name,kwargs)
+
+class SelfUpdate(command_base):
+    def __init__(self, **kwargs):
+        command_base.__init__(self, 'selfupdate', ["self","update"], **kwargs)
+
+class Tarball(command_base):
+    def __init__(self, **kwargs):
+        command_base.__init__(self, 'tarball', ["tarball"], **kwargs)
+
+class Boost_Jam_Build(command_base):
+    def __init__(self, **kwargs):
+        command_base.__init__(self, 'boost.jam.build', ["bjam","build"], **kwargs)
+
+class Boost_Jam(command_base):
+    def __init__(self, **kwargs):
+        command_base.__init__(self, 'boost.jam', ["bjam"], **kwargs)
+
+class Boost_Test(command_base):
+    def __init__(self, **kwargs):
+        self.tests = kwargs.get('tests');
+        if kwargs.has_key('tests'): del kwargs['tests']
+        
+        self._kwargs = kwargs
+        
+        command_base.__init__(self, 'boost.jam', ["btest"], **kwargs)
+    
+    def commandComplete(self, cmd):
+        
+        def test_match(t,r):
+            return t or r.match(parts[1])
+        
+        #~ Get the log so we can parse it to find all the targets
+        #~ we can test.
+        out = cmd.log.getText()
+        lines = string.split(out,"\n")
+        test_targets = {}
+        test_re = []
+        for test in self.tests:
+            test_re.append(re.compile(test))
+        for line in lines:
+            parts = re.split('(?:" ")|(?:" ")|(?: ")|(?:" )|(?: [[]")|(?:"[]] )|(?:")',line)
+            if not parts: continue
+            if parts[0] != 'boost-test(TARGET)': continue
+            if not reduce(test_match,test_re,False): continue
+            try:
+                target_i = parts.index(':')+1
+            except:
+                continue
+            twisted.python.log.msg("Boost_Test.commandComplete: TEST = %s -- TARGETS = %s" %
+                (parts[1],string.join(parts[target_i:-1],' ')) )
+            for t in parts[target_i:-1]:
+                test_targets[t] = True
+        test_targets = test_targets.keys()
+        test_targets.sort()
+        
+        #~ Construct new steps for each of the targets we want to test. It would be much
+        #~ better to tell bjam all targets to test in groups instead of one per invocation.
+        #~ But there's no "easy" way to do that. Passing in args can blow the command line
+        #~ limits. Setting an env can also blow that limit, but this may be a higher limit
+        #~ and we could do them piecemeal.
+        kwargs = self._kwargs.copy()
+        kwargs.update({
+            'flunkOnFailure': False,
+            'appendToLog': True
+            })
+        kwargs['options'].remove('--dump-tests')
+        kwargs['options'].remove('--dump-test-targets')
+        count = 0
+        for test_target in test_targets:
+            kwargs['target'] = test_target
+            step = Boost_Jam(**kwargs)
+            count += 1
+            step.name = "%s.%d" % (step.name,count)
+            #~ The steps up to our point have been eaten away already. So we
+            #~ can add to the front so that the additional steps get executed
+            #~ before the rest.
+            self.build.steps.insert(count-1,step)
+            self.build.build_status.addStep(step)
+        #~ Rearrange the steps on the build_status to match the order in the
+        #~ actual build.
+        existing_count = len(self.build.steps)-count
+        new_count = count
+        a = self.build.build_status.steps[0:-new_count-existing_count]
+        c = self.build.build_status.steps[-new_count-existing_count:-new_count]
+        b = self.build.build_status.steps[-new_count:]
+        self.build.build_status.steps = a+b+c
+
+class Boost_Process_Jam_Log(command_base):
+    def __init__(self, **kwargs):
+        command_base.__init__(self, 'boost.process_jam_log', ["process log"], **kwargs)
+
+class Boost_Collect_Results(command_base):
+    def __init__(self, **kwargs):
+        command_base.__init__(self, 'boost.collect_results', ["collect results"], **kwargs)
+
+class Boost_Publish_Results(command_base):
+    def __init__(self, **kwargs):
+        command_base.__init__(self, 'boost.publish_results', ["publish results"], **kwargs)