unreseached failures markup progress

[SVN r26008]

tools/regression/xsl_reports/test/common.py

@@ -108,24 +108,35 @@ def make_expicit_failure_markup( num_of_libs, num_of_toolsets, num_of_tests ):
             g.endElement( "mark-unusable" )
 
         for i_test in range( 0, num_of_tests ):
+
+            corner_case_test = 0
+            explicitly_marked_failure = 0
+            unresearched = 0
+
             if i_test > num_of_tests - 3:     corner_case_test = 1
-            else:                             corner_case_test = 0
             
-            if i_test % 4 == 0:      explicitly_marked_test = 1
-            else:                    explicitly_marked_test = 0
+            if i_test % 3 == 0:
+                explicitly_marked_failure = 1
+                if i_test % 2 == 0:
+                    unresearched = 1
 
-            if corner_case_test or explicitly_marked_test:
+            if corner_case_test or explicitly_marked_failure:
                 test_attrs = { "name": make_test_name( i_library, i_test ) }
                 if corner_case_test:
                     test_attrs[ "corner-case" ] = "yes"
                 g.startElement( "test", test_attrs )
-                if explicitly_marked_test:
-                    g.startElement( "mark-failure", {} )
+                if explicitly_marked_failure:
+                    failure_attrs = {}
+                    if unresearched: failure_attrs[ "reason" ] = "not-researched"
+                    
+                    g.startElement( "mark-failure", failure_attrs )
                     
                     g.startElement( "toolset", { "name": make_toolset_name( 1 ) } )
                     g.endElement( "toolset" )
                     g.startElement( "toolset", { "name": make_toolset_name( 0 ) } )
                     g.endElement( "toolset" )
+                    g.startElement( "toolset", { "name": make_toolset_name( 2 ) } )
+                    g.endElement( "toolset" )
 
                     g.startElement( "note", {  "author": "V. Annotated" } )
                     g.characters( "Some thoughtful note" )

tools/regression/xsl_reports/test/generate_test_results.py

@@ -11,6 +11,21 @@ num_of_tests = 10
 
 tag = "1_30_0"
 
+
+# Generated results follow the rules:
+#
+# * odd runners are testing on Win32, even runners are testin on Unix
+# * the third toolset has 2 variants
+#
+
+# Generated expected markup:
+#
+# * First two toolset are required
+# * Every fourth library is unusable on event toolsets
+# * Last two tests are corner-ase tests
+# * Every 4th test is explicitly marked up as expected-failure
+
+
 def library_build_failed( library_idx ):
     return library_idx % 2
 
@@ -25,6 +40,7 @@ def test_run_type( runner_idx ):
 def make_test_results():
     if not os.path.exists( tag ):
         os.makedirs( tag )
+
     for i_runner in range( 0, num_of_runners ):
         runner_id = "runner_%02d" % i_runner
         g = xml.sax.saxutils.XMLGenerator( open( os.path.join( tag, runner_id + ".xml" ), "w" ) )
@@ -83,13 +99,15 @@ def make_test_results():
                     if i_runner % 2: test_result = "success"
                     else:             test_result = "fail"
 
+                    if i_runner == 1 and i_toolset == 2 and i_test % 6 == 0:
+                        test_result = "fail"
+                        
                     if test_result == "success" and ( 0 == i_test % 5 ):
                         show_run_output = "true"
 
                     if i_toolset == 2:
                         variants = [ "static-lib", "shared-lib" ]
                     else:
-                        print i_toolset
                         variants = [ None ]
 
                     for variant in variants:

tools/regression/xsl_reports/test/run.py

@@ -27,7 +27,7 @@ report.make_result_pages(
     , comment_file = os.path.abspath( "comment.html" )
     , results_dir = "results"
     , result_prefix = ""
-    , reports = [ "dd" ]
+    , reports = [ "x", "ds", "dd" ]
     , v2 = 1
     )
 

tools/regression/xsl_reports/xsl/v2/add_expected_results.xsl

@@ -93,6 +93,10 @@ http://www.boost.org/LICENSE_1_0.txt)
 
         <xsl:variable name="unexpected_success" select="$status = 'unexpected' and $actual_result = 'success'"/>
 
+        <xsl:variable name="expected_reason">
+            <xsl:value-of select="$test_failures_markup/@reason"/>
+        </xsl:variable>
+
         <xsl:variable name="notes">
 
             <xsl:if test="$unexpected_success and $has_explicit_markup">
@@ -165,6 +169,7 @@ http://www.boost.org/LICENSE_1_0.txt)
 
         <xsl:attribute name="result"><xsl:value-of select="$actual_result"/></xsl:attribute>
         <xsl:attribute name="expected-result"><xsl:value-of select="$expected_result"/></xsl:attribute>
+        <xsl:attribute name="expected-reason"><xsl:value-of select="$expected_reason"/></xsl:attribute>
         <xsl:attribute name="status"><xsl:value-of select="$status"/></xsl:attribute>
         <xsl:attribute name="is-new"><xsl:value-of select="$is_new"/></xsl:attribute>
         <xsl:element name="notes"><xsl:copy-of select="$notes"/></xsl:element>

tools/regression/xsl_reports/xsl/v2/common.xsl

@@ -237,10 +237,16 @@ http://www.boost.org/LICENSE_1_0.txt)
                     <xsl:text>fail-unexpected-new</xsl:text>
                 </xsl:when>
 
+
+                <xsl:when test="count( $test_logs[@result='fail' and @expected-reason != '' ] )">
+                    <xsl:text>fail-expected-unresearched</xsl:text>
+                </xsl:when>
+
                 <xsl:when test="count( $test_logs[@result='fail'] )">
                     <xsl:text>fail-expected</xsl:text>
                 </xsl:when>
                 
+                
                 <xsl:when test="count( $test_logs[@result='success' and  @status='unexpected'] )">
                     <xsl:text>success-unexpected</xsl:text>
                 </xsl:when>

tools/regression/xsl_reports/xsl/v2/html/master.css

@@ -253,6 +253,7 @@ tr.summary-row td
 ,td.summary-user-fail-expected
 ,td.summary-user-success
 ,td.summary-unknown-status
+,td.summary-fail-expected-unresearched
 { 
     width: 60pt;
     text-align: center;
@@ -339,6 +340,18 @@ td.summary-unknown-status
     padding-right: 2pt;
 }
 
+td.library-fail-expected-unresearched
+{
+    width: 60pt;
+    text-align: center;
+    background-color: white;
+    color: black;
+    border: 1px solid black;
+    padding-left: 2pt;
+    padding-right: 2pt;
+}
+
+
   tr.summary-row-first  td.summary-missing
 , tr.summary-row-single td.summary-missing
 , tr.library-row-first  td.library-missing

tools/regression/xsl_reports/xsl/v2/result_page.xsl

@@ -418,13 +418,13 @@ http://www.boost.org/LICENSE_1_0.txt)
             <xsl:when test="$log_link != ''">
                 <xsl:text>&#160;&#160;</xsl:text>
                 <a href="{$log_link}" class="log-link" target="_top">
-                    <xsl:value-of select="$result"/>
+                    <xsl:copy-of select="$result"/>
                 </a>
                 <xsl:text>&#160;&#160;</xsl:text>
             </xsl:when>
             <xsl:otherwise>
                 <xsl:text>&#160;&#160;</xsl:text>
-                <xsl:value-of select="$result"/>
+                <xsl:copy-of select="$result"/>
                 <xsl:text>&#160;&#160;</xsl:text>
             </xsl:otherwise>
         </xsl:choose>
@@ -520,7 +520,16 @@ http://www.boost.org/LICENSE_1_0.txt)
  
             <xsl:when test="$test_log/@result != 'success' and $test_log/@status = 'expected'">
                 <xsl:call-template name="insert_test_result">
-                    <xsl:with-param name="result" select="'fail*'"/>
+                    <xsl:with-param name="result">
+                        <xsl:choose>
+                            <xsl:when test="$test_log/@expected-reason != ''">
+                                <xsl:text>fail?</xsl:text>
+                            </xsl:when> 
+                            <xsl:otherwise>
+                                <xsl:text>fail*</xsl:text>
+                            </xsl:otherwise>
+                        </xsl:choose>
+                    </xsl:with-param>
                     <xsl:with-param name="log_link" select="$cell_link"/>
                 </xsl:call-template>
             </xsl:when>