Patchwork [2/3] cmake: follow ptest output format

login
register
mail settings
Submitter Kang Kai
Date April 1, 2014, 9:09 a.m.
Message ID <26f22e4e2b8b1fa1eff604feac7088986d561eab.1396343261.git.kai.kang@windriver.com>
Download mbox | patch
Permalink /patch/69803/
State New
Headers show

Comments

Kang Kai - April 1, 2014, 9:09 a.m.
From: Li Wang <li.wang@windriver.com>

ptest output format is incorrect, according to yocto Development Manual
(http://www.yoctoproject.org/docs/latest/dev-manual/dev-manual.html#testing-packages-with-ptest)

The test generates output in the format used by Automake:
<result>: <testname>
where the result can be PASS, FAIL, or SKIP, and the testname can be any identifying string.

So we should change the test result format to match yocto ptest rules.

Signed-off-by: Li Wang <li.wang@windriver.com>
Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
Signed-off-by: Kai Kang <kai.kang@windriver.com>
---
 meta/recipes-devtools/cmake/cmake.inc              |   1 +
 .../cmake/cmake/follow_ptest_output_format.patch   | 118 +++++++++++++++++++++
 2 files changed, 119 insertions(+)
 create mode 100644 meta/recipes-devtools/cmake/cmake/follow_ptest_output_format.patch
Richard Purdie - April 1, 2014, 10:49 a.m.
On Tue, 2014-04-01 at 17:09 +0800, Kai Kang wrote:
> From: Li Wang <li.wang@windriver.com>
> 
> ptest output format is incorrect, according to yocto Development Manual
> (http://www.yoctoproject.org/docs/latest/dev-manual/dev-manual.html#testing-packages-with-ptest)
> 
> The test generates output in the format used by Automake:
> <result>: <testname>
> where the result can be PASS, FAIL, or SKIP, and the testname can be any identifying string.
> 
> So we should change the test result format to match yocto ptest rules.
> 
> Signed-off-by: Li Wang <li.wang@windriver.com>
> Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
> Signed-off-by: Kai Kang <kai.kang@windriver.com>
> ---
>  meta/recipes-devtools/cmake/cmake.inc              |   1 +
>  .../cmake/cmake/follow_ptest_output_format.patch   | 118 +++++++++++++++++++++
>  2 files changed, 119 insertions(+)
>  create mode 100644 meta/recipes-devtools/cmake/cmake/follow_ptest_output_format.patch
> 
> diff --git a/meta/recipes-devtools/cmake/cmake.inc b/meta/recipes-devtools/cmake/cmake.inc
> index 1d5303f..254af45 100644
> --- a/meta/recipes-devtools/cmake/cmake.inc
> +++ b/meta/recipes-devtools/cmake/cmake.inc
> @@ -17,6 +17,7 @@ SRC_URI = "http://www.cmake.org/files/v${CMAKE_MAJOR_VERSION}/cmake-${PV}.tar.gz
>             file://aarch64-kwsys.patch \
>             file://qt4-fail-silent.patch \
>             file://cmake-2.8.11.2-FindFreetype.patch \
> +           file://follow_ptest_output_format.patch \
>             "
>  
>  inherit autotools-brokensep
> diff --git a/meta/recipes-devtools/cmake/cmake/follow_ptest_output_format.patch b/meta/recipes-devtools/cmake/cmake/follow_ptest_output_format.patch
> new file mode 100644
> index 0000000..5428df2
> --- /dev/null
> +++ b/meta/recipes-devtools/cmake/cmake/follow_ptest_output_format.patch
> @@ -0,0 +1,118 @@
> +cmake: follow ptest output format
> +
> +ptest output format is incorrect, according to yocto Development Manual
> +(http://www.yoctoproject.org/docs/latest/dev-manual/dev-manual.html#testing-packages-with-ptest)
> +5.10.6. Testing Packages With ptest
> +The test generates output in the format used by Automake:
> +<result>: <testname>
> +where the result can be PASS, FAIL, or SKIP, and the testname can be any identifying string.
> +So we should change the test result format to match yocto ptest rules.
> +
> +Signed-off-by: Li Wang <li.wang@windriver.com>

No upstream status. Is there any chance of this going upstream? Might we
be better off postprocessing the output with a script to put it into the
right format?

Cheers,

Richard


> +---
> +--- a/Source/CTest/cmCTestRunTest.cxx
> ++++ b/Source/CTest/cmCTestRunTest.cxx
> +@@ -145,8 +145,8 @@
> +     this->CompressOutput();
> +     }
> + 
> +-  this->WriteLogOutputTop(completed, total);
> +   std::string reason;
> ++  std::string result;
> +   bool passed = true;
> +   int res = started ? this->TestProcess->GetProcessStatus()
> +                     : cmsysProcess_State_Error;
> +@@ -208,57 +208,58 @@
> +       || (!success && this->TestProperties->WillFail))
> +       {
> +       this->TestResult.Status = cmCTestTestHandler::COMPLETED;
> +-      cmCTestLog(this->CTest, HANDLER_OUTPUT, "   Passed  " );
> ++      result = "           ";
> +       }
> +     else
> +       {
> +       this->TestResult.Status = cmCTestTestHandler::FAILED;
> +-      cmCTestLog(this->CTest, HANDLER_OUTPUT, "***Failed  " << reason );
> ++      result = "           " + reason;
> +       outputTestErrorsToConsole = this->CTest->OutputTestOutputOnTestFailure;
> +       }
> +     }
> +   else if ( res == cmsysProcess_State_Expired )
> +     {
> +-    cmCTestLog(this->CTest, HANDLER_OUTPUT, "***Timeout ");
> ++    result = "***Timeout ";
> +     this->TestResult.Status = cmCTestTestHandler::TIMEOUT;
> +     outputTestErrorsToConsole = this->CTest->OutputTestOutputOnTestFailure;
> +     }
> +   else if ( res == cmsysProcess_State_Exception )
> +     {
> +     outputTestErrorsToConsole = this->CTest->OutputTestOutputOnTestFailure;
> +-    cmCTestLog(this->CTest, HANDLER_OUTPUT, "***Exception: ");
> ++    result = "***Exception: ";
> +     switch(this->TestProcess->GetExitException())
> +       {
> +       case cmsysProcess_Exception_Fault:
> +-        cmCTestLog(this->CTest, HANDLER_OUTPUT, "SegFault");
> ++        result += "SegFault";
> +         this->TestResult.Status = cmCTestTestHandler::SEGFAULT;
> +         break;
> +       case cmsysProcess_Exception_Illegal:
> +-        cmCTestLog(this->CTest, HANDLER_OUTPUT, "Illegal");
> ++        result += "Illegal";
> +         this->TestResult.Status = cmCTestTestHandler::ILLEGAL;
> +         break;
> +       case cmsysProcess_Exception_Interrupt:
> +-        cmCTestLog(this->CTest, HANDLER_OUTPUT, "Interrupt");
> ++        result += "Interrupt";
> +         this->TestResult.Status = cmCTestTestHandler::INTERRUPT;
> +         break;
> +       case cmsysProcess_Exception_Numerical:
> +-        cmCTestLog(this->CTest, HANDLER_OUTPUT, "Numerical");
> ++        result += "Numerical";
> +         this->TestResult.Status = cmCTestTestHandler::NUMERICAL;
> +         break;
> +       default:
> +-        cmCTestLog(this->CTest, HANDLER_OUTPUT, "Other");
> ++        result += "Other";
> +         this->TestResult.Status = cmCTestTestHandler::OTHER_FAULT;
> +       }
> +     }
> +   else //cmsysProcess_State_Error
> +     {
> +-    cmCTestLog(this->CTest, HANDLER_OUTPUT, "***Not Run ");
> ++    result = "***Not Run ";
> +     }
> + 
> +   passed = this->TestResult.Status == cmCTestTestHandler::COMPLETED;
> +   char buf[1024];
> +   sprintf(buf, "%6.2f sec", this->TestProcess->GetTotalTime());
> +-  cmCTestLog(this->CTest, HANDLER_OUTPUT, buf << "\n" );
> ++  this->WriteLogOutputTop(completed, total);
> ++  cmCTestLog(this->CTest, HANDLER_OUTPUT, result << buf << "\n" );
> + 
> +   if ( outputTestErrorsToConsole )
> +     {
> +@@ -398,6 +399,7 @@
> + // Starts the execution of a test.  Returns once it has started
> + bool cmCTestRunTest::StartTest(size_t total)
> + {
> ++  cmCTestLog(this->CTest, HANDLER_OUTPUT, "      ");
> +   cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(2*getNumWidth(total) + 8)
> +     << "Start "
> +     << std::setw(getNumWidth(this->TestHandler->GetMaxIndex()))
> +@@ -679,6 +681,15 @@
> + 
> + void cmCTestRunTest::WriteLogOutputTop(size_t completed, size_t total)
> + {
> ++  if ( this->TestResult.Status == cmCTestTestHandler::COMPLETED )
> ++    {
> ++    cmCTestLog(this->CTest, HANDLER_OUTPUT, "PASS: " );
> ++    }
> ++  else
> ++    {
> ++    cmCTestLog(this->CTest, HANDLER_OUTPUT, "FAIL: " );
> ++    }
> ++
> +   cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
> +              << completed << "/");
> +   cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
> -- 
> 1.8.1.2
>
Otavio Salvador - April 1, 2014, 10:33 p.m.
Hello,

On Tue, Apr 1, 2014 at 7:49 AM, Richard Purdie
<richard.purdie@linuxfoundation.org> wrote:
> On Tue, 2014-04-01 at 17:09 +0800, Kai Kang wrote:
>> From: Li Wang <li.wang@windriver.com>
>>
>> ptest output format is incorrect, according to yocto Development Manual
>> (http://www.yoctoproject.org/docs/latest/dev-manual/dev-manual.html#testing-packages-with-ptest)
>>
>> The test generates output in the format used by Automake:
>> <result>: <testname>
>> where the result can be PASS, FAIL, or SKIP, and the testname can be any identifying string.
>>
>> So we should change the test result format to match yocto ptest rules.
>>
>> Signed-off-by: Li Wang <li.wang@windriver.com>
>> Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
>> Signed-off-by: Kai Kang <kai.kang@windriver.com>
>> ---
>>  meta/recipes-devtools/cmake/cmake.inc              |   1 +
>>  .../cmake/cmake/follow_ptest_output_format.patch   | 118 +++++++++++++++++++++
>>  2 files changed, 119 insertions(+)
>>  create mode 100644 meta/recipes-devtools/cmake/cmake/follow_ptest_output_format.patch
>>
>> diff --git a/meta/recipes-devtools/cmake/cmake.inc b/meta/recipes-devtools/cmake/cmake.inc
>> index 1d5303f..254af45 100644
>> --- a/meta/recipes-devtools/cmake/cmake.inc
>> +++ b/meta/recipes-devtools/cmake/cmake.inc
>> @@ -17,6 +17,7 @@ SRC_URI = "http://www.cmake.org/files/v${CMAKE_MAJOR_VERSION}/cmake-${PV}.tar.gz
>>             file://aarch64-kwsys.patch \
>>             file://qt4-fail-silent.patch \
>>             file://cmake-2.8.11.2-FindFreetype.patch \
>> +           file://follow_ptest_output_format.patch \
>>             "
>>
>>  inherit autotools-brokensep
>> diff --git a/meta/recipes-devtools/cmake/cmake/follow_ptest_output_format.patch b/meta/recipes-devtools/cmake/cmake/follow_ptest_output_format.patch
>> new file mode 100644
>> index 0000000..5428df2
>> --- /dev/null
>> +++ b/meta/recipes-devtools/cmake/cmake/follow_ptest_output_format.patch
>> @@ -0,0 +1,118 @@
>> +cmake: follow ptest output format
>> +
>> +ptest output format is incorrect, according to yocto Development Manual
>> +(http://www.yoctoproject.org/docs/latest/dev-manual/dev-manual.html#testing-packages-with-ptest)
>> +5.10.6. Testing Packages With ptest
>> +The test generates output in the format used by Automake:
>> +<result>: <testname>
>> +where the result can be PASS, FAIL, or SKIP, and the testname can be any identifying string.
>> +So we should change the test result format to match yocto ptest rules.
>> +
>> +Signed-off-by: Li Wang <li.wang@windriver.com>
>
> No upstream status. Is there any chance of this going upstream? Might we
> be better off postprocessing the output with a script to put it into the
> right format?

I agree with both remarks. It'd be good to try to get it somehow
integrated upstream and for OE-Core interim solution would be better
to 'convert' using an external script. This may break external tests
tools people may be using as it changes the format without allowing to
'revert' for normal format.
Kang Kai - April 2, 2014, 1:24 a.m.
On 2014?04?02? 06:33, Otavio Salvador wrote:
> Hello,
>
> On Tue, Apr 1, 2014 at 7:49 AM, Richard Purdie
> <richard.purdie@linuxfoundation.org> wrote:
>> On Tue, 2014-04-01 at 17:09 +0800, Kai Kang wrote:
>>> From: Li Wang <li.wang@windriver.com>
>>>
>>> ptest output format is incorrect, according to yocto Development Manual
>>> (http://www.yoctoproject.org/docs/latest/dev-manual/dev-manual.html#testing-packages-with-ptest)
>>>
>>> The test generates output in the format used by Automake:
>>> <result>: <testname>
>>> where the result can be PASS, FAIL, or SKIP, and the testname can be any identifying string.
>>>
>>> So we should change the test result format to match yocto ptest rules.
>>>
>>> Signed-off-by: Li Wang <li.wang@windriver.com>
>>> Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
>>> Signed-off-by: Kai Kang <kai.kang@windriver.com>
>>> ---
>>>   meta/recipes-devtools/cmake/cmake.inc              |   1 +
>>>   .../cmake/cmake/follow_ptest_output_format.patch   | 118 +++++++++++++++++++++
>>>   2 files changed, 119 insertions(+)
>>>   create mode 100644 meta/recipes-devtools/cmake/cmake/follow_ptest_output_format.patch
>>>
>>> diff --git a/meta/recipes-devtools/cmake/cmake.inc b/meta/recipes-devtools/cmake/cmake.inc
>>> index 1d5303f..254af45 100644
>>> --- a/meta/recipes-devtools/cmake/cmake.inc
>>> +++ b/meta/recipes-devtools/cmake/cmake.inc
>>> @@ -17,6 +17,7 @@ SRC_URI = "http://www.cmake.org/files/v${CMAKE_MAJOR_VERSION}/cmake-${PV}.tar.gz
>>>              file://aarch64-kwsys.patch \
>>>              file://qt4-fail-silent.patch \
>>>              file://cmake-2.8.11.2-FindFreetype.patch \
>>> +           file://follow_ptest_output_format.patch \
>>>              "
>>>
>>>   inherit autotools-brokensep
>>> diff --git a/meta/recipes-devtools/cmake/cmake/follow_ptest_output_format.patch b/meta/recipes-devtools/cmake/cmake/follow_ptest_output_format.patch
>>> new file mode 100644
>>> index 0000000..5428df2
>>> --- /dev/null
>>> +++ b/meta/recipes-devtools/cmake/cmake/follow_ptest_output_format.patch
>>> @@ -0,0 +1,118 @@
>>> +cmake: follow ptest output format
>>> +
>>> +ptest output format is incorrect, according to yocto Development Manual
>>> +(http://www.yoctoproject.org/docs/latest/dev-manual/dev-manual.html#testing-packages-with-ptest)
>>> +5.10.6. Testing Packages With ptest
>>> +The test generates output in the format used by Automake:
>>> +<result>: <testname>
>>> +where the result can be PASS, FAIL, or SKIP, and the testname can be any identifying string.
>>> +So we should change the test result format to match yocto ptest rules.
>>> +
>>> +Signed-off-by: Li Wang <li.wang@windriver.com>
>> No upstream status. Is there any chance of this going upstream? Might we
>> be better off postprocessing the output with a script to put it into the
>> right format?
> I agree with both remarks. It'd be good to try to get it somehow
> integrated upstream and for OE-Core interim solution would be better
> to 'convert' using an external script. This may break external tests
> tools people may be using as it changes the format without allowing to
> 'revert' for normal format.
>

OK. I'll send it upstream and see what's going on.

Patch

diff --git a/meta/recipes-devtools/cmake/cmake.inc b/meta/recipes-devtools/cmake/cmake.inc
index 1d5303f..254af45 100644
--- a/meta/recipes-devtools/cmake/cmake.inc
+++ b/meta/recipes-devtools/cmake/cmake.inc
@@ -17,6 +17,7 @@  SRC_URI = "http://www.cmake.org/files/v${CMAKE_MAJOR_VERSION}/cmake-${PV}.tar.gz
            file://aarch64-kwsys.patch \
            file://qt4-fail-silent.patch \
            file://cmake-2.8.11.2-FindFreetype.patch \
+           file://follow_ptest_output_format.patch \
            "
 
 inherit autotools-brokensep
diff --git a/meta/recipes-devtools/cmake/cmake/follow_ptest_output_format.patch b/meta/recipes-devtools/cmake/cmake/follow_ptest_output_format.patch
new file mode 100644
index 0000000..5428df2
--- /dev/null
+++ b/meta/recipes-devtools/cmake/cmake/follow_ptest_output_format.patch
@@ -0,0 +1,118 @@ 
+cmake: follow ptest output format
+
+ptest output format is incorrect, according to yocto Development Manual
+(http://www.yoctoproject.org/docs/latest/dev-manual/dev-manual.html#testing-packages-with-ptest)
+5.10.6. Testing Packages With ptest
+The test generates output in the format used by Automake:
+<result>: <testname>
+where the result can be PASS, FAIL, or SKIP, and the testname can be any identifying string.
+So we should change the test result format to match yocto ptest rules.
+
+Signed-off-by: Li Wang <li.wang@windriver.com>
+---
+--- a/Source/CTest/cmCTestRunTest.cxx
++++ b/Source/CTest/cmCTestRunTest.cxx
+@@ -145,8 +145,8 @@
+     this->CompressOutput();
+     }
+ 
+-  this->WriteLogOutputTop(completed, total);
+   std::string reason;
++  std::string result;
+   bool passed = true;
+   int res = started ? this->TestProcess->GetProcessStatus()
+                     : cmsysProcess_State_Error;
+@@ -208,57 +208,58 @@
+       || (!success && this->TestProperties->WillFail))
+       {
+       this->TestResult.Status = cmCTestTestHandler::COMPLETED;
+-      cmCTestLog(this->CTest, HANDLER_OUTPUT, "   Passed  " );
++      result = "           ";
+       }
+     else
+       {
+       this->TestResult.Status = cmCTestTestHandler::FAILED;
+-      cmCTestLog(this->CTest, HANDLER_OUTPUT, "***Failed  " << reason );
++      result = "           " + reason;
+       outputTestErrorsToConsole = this->CTest->OutputTestOutputOnTestFailure;
+       }
+     }
+   else if ( res == cmsysProcess_State_Expired )
+     {
+-    cmCTestLog(this->CTest, HANDLER_OUTPUT, "***Timeout ");
++    result = "***Timeout ";
+     this->TestResult.Status = cmCTestTestHandler::TIMEOUT;
+     outputTestErrorsToConsole = this->CTest->OutputTestOutputOnTestFailure;
+     }
+   else if ( res == cmsysProcess_State_Exception )
+     {
+     outputTestErrorsToConsole = this->CTest->OutputTestOutputOnTestFailure;
+-    cmCTestLog(this->CTest, HANDLER_OUTPUT, "***Exception: ");
++    result = "***Exception: ";
+     switch(this->TestProcess->GetExitException())
+       {
+       case cmsysProcess_Exception_Fault:
+-        cmCTestLog(this->CTest, HANDLER_OUTPUT, "SegFault");
++        result += "SegFault";
+         this->TestResult.Status = cmCTestTestHandler::SEGFAULT;
+         break;
+       case cmsysProcess_Exception_Illegal:
+-        cmCTestLog(this->CTest, HANDLER_OUTPUT, "Illegal");
++        result += "Illegal";
+         this->TestResult.Status = cmCTestTestHandler::ILLEGAL;
+         break;
+       case cmsysProcess_Exception_Interrupt:
+-        cmCTestLog(this->CTest, HANDLER_OUTPUT, "Interrupt");
++        result += "Interrupt";
+         this->TestResult.Status = cmCTestTestHandler::INTERRUPT;
+         break;
+       case cmsysProcess_Exception_Numerical:
+-        cmCTestLog(this->CTest, HANDLER_OUTPUT, "Numerical");
++        result += "Numerical";
+         this->TestResult.Status = cmCTestTestHandler::NUMERICAL;
+         break;
+       default:
+-        cmCTestLog(this->CTest, HANDLER_OUTPUT, "Other");
++        result += "Other";
+         this->TestResult.Status = cmCTestTestHandler::OTHER_FAULT;
+       }
+     }
+   else //cmsysProcess_State_Error
+     {
+-    cmCTestLog(this->CTest, HANDLER_OUTPUT, "***Not Run ");
++    result = "***Not Run ";
+     }
+ 
+   passed = this->TestResult.Status == cmCTestTestHandler::COMPLETED;
+   char buf[1024];
+   sprintf(buf, "%6.2f sec", this->TestProcess->GetTotalTime());
+-  cmCTestLog(this->CTest, HANDLER_OUTPUT, buf << "\n" );
++  this->WriteLogOutputTop(completed, total);
++  cmCTestLog(this->CTest, HANDLER_OUTPUT, result << buf << "\n" );
+ 
+   if ( outputTestErrorsToConsole )
+     {
+@@ -398,6 +399,7 @@
+ // Starts the execution of a test.  Returns once it has started
+ bool cmCTestRunTest::StartTest(size_t total)
+ {
++  cmCTestLog(this->CTest, HANDLER_OUTPUT, "      ");
+   cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(2*getNumWidth(total) + 8)
+     << "Start "
+     << std::setw(getNumWidth(this->TestHandler->GetMaxIndex()))
+@@ -679,6 +681,15 @@
+ 
+ void cmCTestRunTest::WriteLogOutputTop(size_t completed, size_t total)
+ {
++  if ( this->TestResult.Status == cmCTestTestHandler::COMPLETED )
++    {
++    cmCTestLog(this->CTest, HANDLER_OUTPUT, "PASS: " );
++    }
++  else
++    {
++    cmCTestLog(this->CTest, HANDLER_OUTPUT, "FAIL: " );
++    }
++
+   cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
+              << completed << "/");
+   cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))