[v2,ptest-runner] ptest-runner: Add support to report duration of each ptest

Submitted by Aníbal Limón on Jan. 7, 2019, 9:18 p.m. | Patch ID: 157617

Details

Message ID 20190107211837.25225-1-anibal.limon@linaro.org
State New
Headers show

Commit Message

Aníbal Limón Jan. 7, 2019, 9:18 p.m.
In stdout reported as,

...
BEGIN: ptest-dir
...
DURATION: N
END: ptest-dir
...

In XML reported as,

...
<testcase classname='test1' name='run-ptest'>
	<duration>N</duration>
</testcase>
...

Signed-off-by: Aníbal Limón <anibal.limon@linaro.org>
---
 tests/data/reference.xml |  2 ++
 tests/utils.c            | 20 ++++++++++++--------
 utils.c                  | 19 ++++++++++++-------
 utils.h                  |  2 +-
 4 files changed, 27 insertions(+), 16 deletions(-)

Patch hide | download patch | download mbox

diff --git a/tests/data/reference.xml b/tests/data/reference.xml
index 17f91c2..f6ff389 100644
--- a/tests/data/reference.xml
+++ b/tests/data/reference.xml
@@ -1,8 +1,10 @@ 
 <?xml version='1.0' encoding='UTF-8'?>
 <testsuite name='ptest' tests='2'>
 	<testcase classname='test1' name='run-ptest'>
+		<duration>5</duration>
 	</testcase>
 	<testcase classname='test2' name='run-ptest'>
+		<duration>10</duration>
 		<failure type='exit_code' message='run-ptest exited with code: 1'></failure>
 		<failure type='timeout'/>
 	</testcase>
diff --git a/tests/utils.c b/tests/utils.c
index 662abe8..2ccb1c0 100644
--- a/tests/utils.c
+++ b/tests/utils.c
@@ -188,26 +188,30 @@  START_TEST(test_run_ptests)
 END_TEST
 
 static void
-search_for_timeout(const int rp, FILE *fp_stdout, FILE *fp_stderr)
+search_for_timeout_and_duration(const int rp, FILE *fp_stdout, FILE *fp_stderr)
 {
 	const char *timeout_str = "TIMEOUT";
+	const char *duration_str = "DURATION";
 	char line_buf[PRINT_PTEST_BUF_SIZE];
-	int found_timeout = 0;
+	int found_timeout = 0, found_duration = 0;
 	char *line = NULL;
 
 	ck_assert(rp != 0);
 
-	while ((line = fgets(line_buf, PRINT_PTEST_BUF_SIZE, fp_stdout)) != NULL)
+	while ((line = fgets(line_buf, PRINT_PTEST_BUF_SIZE, fp_stdout)) != NULL) {
 		find_word(&found_timeout, line, timeout_str);
+		find_word(&found_duration, line, duration_str);
+	}
 
 	ck_assert(found_timeout == 1);
+	ck_assert(found_duration == 1);
 }
 
-START_TEST(test_run_timeout_ptest)
+START_TEST(test_run_timeout_duration_ptest)
 	struct ptest_list *head = get_available_ptests(opts_directory);
 	int timeout = 1;
 
-	test_ptest_expected_failure(head, timeout, "hang", search_for_timeout);
+	test_ptest_expected_failure(head, timeout, "hang", search_for_timeout_and_duration);
 
 	ptest_list_free_all(head);
 END_TEST
@@ -257,8 +261,8 @@  START_TEST(test_xml_pass)
 	FILE *xp;
 	xp = xml_create(2, "./test.xml");
 	ck_assert(xp != NULL);
-	xml_add_case(xp, 0,"test1", 0);
-	xml_add_case(xp, 1,"test2", 1);
+	xml_add_case(xp, 0,"test1", 0, 5);
+	xml_add_case(xp, 1,"test2", 1, 10);
 	xml_finish(xp);
 
 	FILE *fp, *fr;
@@ -291,7 +295,7 @@  utils_suite()
 	tcase_add_test(tc_core, test_print_ptests);
 	tcase_add_test(tc_core, test_filter_ptests);
 	tcase_add_test(tc_core, test_run_ptests);
-	tcase_add_test(tc_core, test_run_timeout_ptest);
+	tcase_add_test(tc_core, test_run_timeout_duration_ptest);
 	tcase_add_test(tc_core, test_run_fail_ptest);
 	tcase_add_test(tc_core, test_xml_pass);
 	tcase_add_test(tc_core, test_xml_fail);
diff --git a/utils.c b/utils.c
index 4a38ea1..01ee415 100644
--- a/utils.c
+++ b/utils.c
@@ -45,12 +45,10 @@ 
 #define WAIT_CHILD_BUF_MAX_SIZE 1024
 
 static inline char *
-get_stime(char *stime, size_t size)
+get_stime(char *stime, size_t size, time_t t)
 {
-	time_t t;
 	struct tm *lt;
 
-	t = time(NULL);
 	lt = localtime(&t);
 	strftime(stime, size, "%Y-%m-%dT%H:%M", lt);
 
@@ -334,6 +332,8 @@  run_ptests(struct ptest_list *head, const struct ptest_options opts,
 	int pipefd_stdout[2];
 	int pipefd_stderr[2];
 	int timeouted;
+	time_t sttime, entime;
+	int duration;
 
 	if (opts.xml_filename) {
 		xh = xml_create(ptest_list_length(head), opts.xml_filename);
@@ -373,11 +373,15 @@  run_ptests(struct ptest_list *head, const struct ptest_options opts,
 				int fds[2]; fds[0] = pipefd_stdout[0]; fds[1] = pipefd_stderr[0];
 				FILE *fps[2]; fps[0] = fp; fps[1] = fp_stderr;
 
-				fprintf(fp, "%s\n", get_stime(stime, GET_STIME_BUF_SIZE));
+				sttime = time(NULL);
+				fprintf(fp, "%s\n", get_stime(stime, GET_STIME_BUF_SIZE, sttime));
 				fprintf(fp, "BEGIN: %s\n", ptest_dir);
 
 				status = wait_child(ptest_dir, p->run_ptest, child,
 						opts.timeout, fds, fps, &timeouted);
+				entime = time(NULL);
+				duration = entime - sttime;
+				fprintf(fps[0], "DURATION: %d\n", duration);
 
 				if (status) {
 					fprintf(fps[0], "\nERROR: Exit status is %d\n", status);
@@ -387,10 +391,10 @@  run_ptests(struct ptest_list *head, const struct ptest_options opts,
 					fprintf(fps[0], "TIMEOUT: %s\n", ptest_dir);
 
 				if (opts.xml_filename)
-					xml_add_case(xh, status, ptest_dir, timeouted);
+					xml_add_case(xh, status, ptest_dir, timeouted, duration);
 
 				fprintf(fp, "END: %s\n", ptest_dir);
-				fprintf(fp, "%s\n", get_stime(stime, GET_STIME_BUF_SIZE));
+				fprintf(fp, "%s\n", get_stime(stime, GET_STIME_BUF_SIZE, entime));
 			}
 		PTEST_LIST_ITERATE_END;
 		fprintf(fp, "STOP: %s\n", progname);
@@ -426,9 +430,10 @@  xml_create(int test_count, char *xml_filename)
 }
 
 void
-xml_add_case(FILE *xh, int status, const char *ptest_dir, int timeouted)
+xml_add_case(FILE *xh, int status, const char *ptest_dir, int timeouted, int duration)
 {
 	fprintf(xh, "\t<testcase classname='%s' name='run-ptest'>\n", ptest_dir);
+	fprintf(xh, "\t\t<duration>%d</duration>\n", duration);
 
 	if (status != 0) {
 		fprintf(xh, "\t\t<failure type='exit_code'");
diff --git a/utils.h b/utils.h
index 880105f..5a3f44a 100644
--- a/utils.h
+++ b/utils.h
@@ -48,7 +48,7 @@  extern int run_ptests(struct ptest_list *, const struct ptest_options,
 		const char *, FILE *, FILE *);
 
 extern FILE *xml_create(int, char *);
-extern void xml_add_case(FILE *, int, const char *, int);
+extern void xml_add_case(FILE *, int, const char *, int, int);
 extern void xml_finish(FILE *);
 
 #endif

Comments

Richard Purdie Jan. 8, 2019, 7:59 p.m.
On Mon, 2019-01-07 at 15:18 -0600, Aníbal Limón wrote:
> In stdout reported as,
> 
> ...
> BEGIN: ptest-dir
> ...
> DURATION: N
> END: ptest-dir
> ...
> 
> In XML reported as,
> 
> ...
> <testcase classname='test1' name='run-ptest'>
> 	<duration>N</duration>
> </testcase>
> ...
> 
> Signed-off-by: Aníbal Limón <anibal.limon@linaro.org>
> ---
>  tests/data/reference.xml |  2 ++
>  tests/utils.c            | 20 ++++++++++++--------
>  utils.c                  | 19 ++++++++++++-------
>  utils.h                  |  2 +-
>  4 files changed, 27 insertions(+), 16 deletions(-)

I've not looked at the code detail but the above looks good to me and
will significantly help ptest usage! :)

Once this is in are you able to help update the recipe and pass this
data through to the the oe-selftest json report?

Cheers,

Richard
Aníbal Limón Jan. 9, 2019, 6 p.m.
On Tue, 8 Jan 2019 at 14:00, <richard.purdie@linuxfoundation.org> wrote:

> On Mon, 2019-01-07 at 15:18 -0600, Aníbal Limón wrote:
> > In stdout reported as,
> >
> > ...
> > BEGIN: ptest-dir
> > ...
> > DURATION: N
> > END: ptest-dir
> > ...
> >
> > In XML reported as,
> >
> > ...
> > <testcase classname='test1' name='run-ptest'>
> >       <duration>N</duration>
> > </testcase>
> > ...
> >
> > Signed-off-by: Aníbal Limón <anibal.limon@linaro.org>
> > ---
> >  tests/data/reference.xml |  2 ++
> >  tests/utils.c            | 20 ++++++++++++--------
> >  utils.c                  | 19 ++++++++++++-------
> >  utils.h                  |  2 +-
> >  4 files changed, 27 insertions(+), 16 deletions(-)
>
> I've not looked at the code detail but the above looks good to me and
> will significantly help ptest usage! :)
>
> Once this is in are you able to help update the recipe and pass this
> data through to the the oe-selftest json report?
>

I will send the recipe upgrade, What is the selftest json report?, I'm not
aware.

Regards,
Anibal


>
> Cheers,
>
> Richard
>
>
Richard Purdie Jan. 9, 2019, 6:32 p.m.
On Wed, 2019-01-09 at 12:00 -0600, Anibal Limon wrote:
> 
> 
> On Tue, 8 Jan 2019 at 14:00, <richard.purdie@linuxfoundation.org>
> wrote:
> > On Mon, 2019-01-07 at 15:18 -0600, Aníbal Limón wrote:
> > > In stdout reported as,
> > > 
> > > ...
> > > BEGIN: ptest-dir
> > > ...
> > > DURATION: N
> > > END: ptest-dir
> > > ...
> > > 
> > > In XML reported as,
> > > 
> > > ...
> > > <testcase classname='test1' name='run-ptest'>
> > >       <duration>N</duration>
> > > </testcase>
> > > ...
> > > 
> > > Signed-off-by: Aníbal Limón <anibal.limon@linaro.org>
> > > ---
> > >  tests/data/reference.xml |  2 ++
> > >  tests/utils.c            | 20 ++++++++++++--------
> > >  utils.c                  | 19 ++++++++++++-------
> > >  utils.h                  |  2 +-
> > >  4 files changed, 27 insertions(+), 16 deletions(-)
> > 
> > I've not looked at the code detail but the above looks good to me
> > and
> > will significantly help ptest usage! :)
> > 
> > Once this is in are you able to help update the recipe and pass
> > this
> > data through to the the oe-selftest json report?
> 
> I will send the recipe upgrade, What is the selftest json report?,
> I'm not aware.

Sorry, its not from selftest, its from testimage running against an
image where ptests are present. It generates something like:

https://autobuilder.yocto.io/pub/releases/yocto-2.7_M1.rc1/testresults/qemux86-64-ptest/testresults.json

Cheers,

Richard