various improvements

[SVN r23974]

tools/regression/xsl_reports/merger/merge_logs.py

@@ -8,22 +8,46 @@
 import xml.sax.saxutils
 import xml.dom.minidom
 import zipfile
+import ftplib
 import glob
 import os.path
 import sys
 
 
-def download_test_runs( destination, tag, user ):
-    utils.log( 'Downloading test runs for tag "%s" [connecting as %s]...' % ( tag, user ) )
+def download_from_ftp( destination_dir, tag ):
+    ftp_site = 'fx.meta-comm.com'
+    site_path = '/boost-regression/%s' % tag
+    utils.log( 'Downloading test runs from  ftp://%s%s' % ( ftp_site, site_path ) )
 
-    destination_dir = os.path.join( destination, tag )
+    ftp = ftplib.FTP( ftp_site )
+    ftp.login()
+    ftp.cwd( site_path )
+    
+    files = ftp.nlst()
+    for f in files:
+        utils.log( '  Downloading %s into "%s" directory...' % ( f, destination_dir ) )
+        result = open( os.path.join( destination_dir, f ), 'wb' )
+        ftp.retrbinary( 'RETR %s' % f, result.write )
+
+    ftp.quit()
+
+
+def download_test_runs( incoming_dir, tag, user ):
+    utils.log( 'Downloading test runs for tag "%s"...' % tag )
+
+    destination_dir = os.path.join( incoming_dir, tag )
     utils.makedirs( destination_dir )
+    
+    if user is not None:
+        utils.log( 'Downloading test runs from SourceForge [connecting as %s]...' % user )
+        utils.sourceforge.download( 
+              'regression-logs/incoming/%s/' % tag
+            , destination_dir
+            , user
+            )
+    
+    download_from_ftp( destination_dir, tag )
 
-    utils.sourceforge.download( 
-          'regression-logs/incoming/%s/' % tag
-        , destination_dir
-        , user
-        )
     
 def unzip( archive_path, result_dir ):
     z = zipfile.ZipFile( archive_path, 'r', zipfile.ZIP_DEFLATED ) 
@@ -46,18 +70,20 @@ def unzip_test_runs( dir ):
             os.unlink( zip_path )
         except Exception, msg:
             utils.log( '  Skipping "%s" due to errors (%s)' % ( test_run, msg ) )
-    
-def merge_test_runs( incoming_dir, tag, writer ):
+
+
+def merge_test_runs( incoming_dir, tag, writer, dont_collect_logs ):
     test_runs_dir = os.path.join( incoming_dir, tag )
     
-    utils.log( 'Removing stale XMLs in "%s"...' % test_runs_dir )
-    files = glob.glob( os.path.join( test_runs_dir, '*.xml' ) )
-    for f in files:  
-       	utils.log( '  Removing "%s" ...' % f )
-        os.unlink( f )
-
-    utils.log( 'Unzipping new test runs...' )
-    unzip_test_runs( test_runs_dir )
+    if not dont_collect_logs:
+        utils.log( 'Removing stale XMLs in "%s"...' % test_runs_dir )
+        files = glob.glob( os.path.join( test_runs_dir, '*.xml' ) )
+        for f in files:  
+       	    utils.log( '  Removing "%s" ...' % f )
+            os.unlink( f )
+
+        utils.log( 'Unzipping new test runs...' )
+        unzip_test_runs( test_runs_dir )
     
     all_runs_xml = xml.sax.saxutils.XMLGenerator( writer )
     all_runs_xml.startDocument()
@@ -87,7 +113,8 @@ def merge_logs(
         , dont_collect_logs
         ):
     
-    utils.log( 'Merging test runs into "%s"...' % results_xml )
+    results_xml_path = os.path.join( results_dir, results_xml )
+    utils.log( 'Merging test runs into "%s"...' % results_xml_path )
     
     incoming_dir = os.path.join( results_dir, 'incoming/' )
 
@@ -95,10 +122,10 @@ def merge_logs(
     if not dont_collect_logs:
         download_test_runs( incoming_dir, tag, user )
     
-    writer = open( results_xml, 'w' )
-    merge_test_runs( incoming_dir, tag, writer )
+    writer = open( results_xml_path, 'w' )
+    merge_test_runs( incoming_dir, tag, writer, dont_collect_logs )
 
-    utils.log( 'Done writing "%s"' % results_xml )
+    utils.log( 'Done writing "%s"' % results_xml_path )
 
 
 def accept_args( args ):
@@ -111,7 +138,11 @@ def accept_args( args ):
         , 'help'
         ]
 
-    options = { '--results-xml' : 'all-runs.xml' }
+    options = { 
+          '--results-xml' : 'all-runs.xml'
+        , '--user' :        None
+        , '--tag' :         'CVS-HEAD'
+        }
     utils.accept_args( args_spec, args, options, usage )
 
     return (
@@ -126,11 +157,12 @@ def accept_args( args ):
 def usage():
     print 'Usage: %s [options]' % os.path.basename( sys.argv[0] )
     print    '''
-\t--tag                 the tag for the results (e.g. 'CVS-HEAD')
-\t--user                SourceForge user name for a shell account
-\t--results-dir         directory for the resulting XML
+\t--results-dir         directory for the resulting XML/collected logs
 \t--results-xml         name of the resulting XML document (default 'all-runs.xml')
-\t--dont-collect-logs   don't collect logs from SourceForge
+\t--tag                 the tag for the results ('CVS-HEAD' by default)
+\t--user                SourceForge user name for a shell account (optional,
+\t                      if not provided, logs are collected from FTP only)
+\t--dont-collect-logs   don't collect logs from SourceForge/FTP
 '''
     
 def main():

tools/regression/xsl_reports/report.py

@@ -39,7 +39,7 @@ def make_result_pages(
           test_results_file
         , expected_results_file
         , failures_markup_file
-        , source
+        , tag
         , run_date
         , comment_file
         , results_dir
@@ -88,7 +88,7 @@ def make_result_pages(
             , xsl_path( 'links_page.xsl', v2 )
             , links
             , {
-                  'source':                 source
+                  'source':                 tag
                 , 'run_date':               run_date 
                 , 'comment_file':           comment_file
                 , 'explicit_markup_file':   failures_markup_file
@@ -105,7 +105,7 @@ def make_result_pages(
             , xsl_path( 'issues_page.xsl', v2 )
             , issues
             , {
-                  'source':                 source
+                  'source':                 tag
                 , 'run_date':               run_date
                 , 'comment_file':           comment_file
                 , 'explicit_markup_file':   failures_markup_file
@@ -123,7 +123,7 @@ def make_result_pages(
                 , { 
                       'links_file':             'links.html'
                     , 'mode':                   mode
-                    , 'source':                 source
+                    , 'source':                 tag
                     , 'run_date':               run_date 
                     , 'comment_file':           comment_file
                     , 'expected_results_file':  expected_results_file
@@ -141,7 +141,7 @@ def make_result_pages(
                 , os.path.join( output_dir, mode, 'summary.html' )
                 , { 
                       'mode' :                  mode 
-                    , 'source':                 source
+                    , 'source':                 tag
                     , 'run_date':               run_date 
                     , 'comment_file':           comment_file
                     , 'explicit_markup_file' :  failures_markup_file
@@ -174,7 +174,7 @@ def make_result_pages(
 
 def build_xsl_reports( 
           locate_root_dir
-        , source
+        , tag
         , expected_results_file
         , failures_markup_file
         , comment_file
@@ -195,7 +195,7 @@ def build_xsl_reports(
     if v2:
         import merger
         merger.merge_logs(
-              source
+              tag
             , user
             , results_dir
             , test_results_file
@@ -214,7 +214,7 @@ def build_xsl_reports(
           test_results_file
         , expected_results_file
         , failures_markup_file
-        , source
+        , tag
         , run_date
         , comment_file
         , results_dir

tools/regression/xsl_reports/runner/collect_and_upload_logs.py

@@ -47,22 +47,23 @@ def upload_to_sourceforge( user, tag, results_file ):
     
     utils.sourceforge.upload( results_file, upload_dir, user )
 
-def upload_to_ftp( tag, result_file ):
+def upload_to_ftp( tag, source ):
     ftp_site = 'fx.meta-comm.com'
-    side_path = '/boost-regression'
-    utils.log( "Uploading log archive \"%s\" to ftp://%s%s/%s" % ( result_file, ftp_site, side_path, tag ) )
+    site_path = '/boost-regression'
+    utils.log( "Uploading log archive \"%s\" to ftp://%s%s/%s" % ( result_file, ftp_site, site_path, tag ) )
     
     ftp = ftplib.FTP( ftp_site )
     ftp.login()
-    ftp.cwd( side_path )
+    ftp.cwd( site_path )
     try:
         ftp.cwd( tag )
     except ftplib.error_perm:
         ftp.mkd( tag )
         ftp.cwd( tag )
 
-    result_file_reader = open( result_file, "rb" )
-    ftp.storbinary( "STOR %s" % os.path.basename( result_file ), result_file_reader )
+    f = open( source, 'rb' )
+    ftp.storbinary( 'STOR %s' % os.path.basename( result_file ), f )
+    ftp.quit()
 
 
 def copy_comments( results_xml, comment_file ):
@@ -170,9 +171,12 @@ def accept_args( args ):
         ]
     
     options = {
-          '--user' :          None
-          , '--source' :      ''
-          , '--run-type' :    ''
+          '--tag' :         'CVS-HEAD'
+        , '--platform' :    sys.platform
+        , '--user' :        None
+        , '--comment' :     None
+        , '--source' :      ''
+        , '--run-type' :    ''
         }
     
     utils.accept_args( args_spec, args, options, usage )
@@ -193,15 +197,16 @@ def accept_args( args ):
 def usage():
     print 'Usage: %s [options]' % os.path.basename( sys.argv[0] )
     print    '''
-\t--locate-root         directory to to scan for 'test_log.xml' files
-\t--runner              runner ID (e.g. 'Metacomm')
-\t--tag                 the tag for the results (e.g. 'CVS-HEAD')
-\t--platform            platform name
-\t--comment             an html comment file (will be inserted in the reports)
-\t--timestamp           timestamp of the run
-\t--user                SourceForge user name for a shell account (optional)
-\t--source              SourceForge user name for a shell account (optional)
-\t--run-type            "incremental" or "full" (optional)
+\t--locate-root   directory to to scan for 'test_log.xml' files
+\t--runner        runner ID (e.g. 'Metacomm')
+\t--timestamp     timestamp of the run
+\t--comment       an HTML comment file to be inserted in the reports
+\t                ('comment.html' by default)
+\t--tag           the tag for the results ('CVS-HEAD' by default)
+\t--user          SourceForge user name for a shell account (optional)
+\t--source        where Boost sources came from (e.g. CVS, tarball, 
+\t                anonymous CVS)
+\t--run-type      "incremental" or "full" (optional)
 '''
     
 def main():

tools/regression/xsl_reports/runner/regression.py

@@ -393,14 +393,14 @@ def collect_logs(
     else: 
         comment_path = os.path.join( regression_root, comment )
     
-    run_type = ""
-    if incremental: run_type = "incremental"
-    else:           run_type = "incremental"
-
-    source = ""
-    if user is None:          source = "tarball"
-    elif user == "anonymous": source = "cvs:pserver %s" % tag
-    else:                     source = "cvs:ext %s" % tag
+    run_type = ''
+    if incremental: run_type = 'incremental'
+    else:           run_type = 'full'
+
+    source = ''
+    if user is None:          source = 'tarball'
+    elif user == 'anonymous': source = 'anonymous CVS'
+    else:                     source = 'CVS'
    
     from runner import collect_logs
     collect_logs( 
@@ -536,8 +536,8 @@ Commands:
 Options:
 \t--runner        runner ID (e.g. 'Metacomm')
 \t--tag           the tag for the results ('CVS-HEAD' by default)
-\t--comment       an html comment file (will be inserted in the reports, 
-\t                'comment.html' by default)
+\t--comment       an HTML comment file to be inserted in the reports
+\t                ('comment.html' by default)
 \t--incremental   do incremental run (do not remove previous binaries)
 \t--user          SourceForge user name for a shell/CVS account (optional)
 \t--toolsets      comma-separated list of toolsets to test with (optional)