| Message ID | 20240220200159.13419-2-alexis.lothore@bootlin.com |
|---|---|
| State | Accepted, archived |
| Commit | c01aa8df0613a103859b4431d3cc5056b2fef1b8 |
| Headers | show |
| Series | testimage: add failed test post actions and fetch more data | expand |
On Tue, 2024-02-20 at 21:01 +0100, Alexis Lothoré via lists.openembedded.org wrote: > From: Alexis Lothoré <alexis.lothore@bootlin.com> > > testimage is able to detect whenever a test run leads to some tests > failing, and execute some actions in this case. The only action > currently > defined in such case is to retrieve artifacts from the target under > test, > as listed in TESTIMAGE_FAILED_QA_ARTIFACTS > > In order to be able to add multiple actions, define a central > function to > gather all "post actions" to run whenever a test has failed > (run_failed_tests_post_actions). This function contains a table > listing all > functions to be called whenever a test fails. Any function in this > table > will be provided with bitbake internal data dictionary ("d") and the > current runtime testing context ("tc"). Isolate all this feature in a > dedicated bbclass file inherited by testimage. > This patch does not bring any functional change. > > Signed-off-by: Alexis Lothoré <alexis.lothore@bootlin.com> > --- > .../failed-tests-post-actions.bbclass | 64 > +++++++++++++++++++ > meta/classes-recipe/testimage.bbclass | 42 +----------- > 2 files changed, 66 insertions(+), 40 deletions(-) > create mode 100644 meta/classes-recipe/failed-tests-post- > actions.bbclass > > diff --git a/meta/classes-recipe/failed-tests-post-actions.bbclass > b/meta/classes-recipe/failed-tests-post-actions.bbclass > new file mode 100644 > index 000000000000..7c7d3391298f > --- /dev/null > +++ b/meta/classes-recipe/failed-tests-post-actions.bbclass > @@ -0,0 +1,64 @@ > +# > +# Copyright OpenEmbedded Contributors > +# > +# SPDX-License-Identifier: MIT > +# > + > + > +################################################################## > +# Artifacts retrieval > +################################################################## > + > +def get_artifacts_list(target, raw_list): > + result = [] > + # Passed list may contains patterns in paths, expand them > directly on target > + for raw_path in raw_list.split(): > + cmd = f"for p in {raw_path}; do if [ -e $p ]; then echo $p; > fi; done" > + try: > + status, output = target.run(cmd) > + if status != 0 or not output: > + raise Exception() > + result += output.split() > + except: > + bb.note(f"No file/directory matching path {raw_path}") > + > + return result > + > +def retrieve_test_artifacts(target, artifacts_list, target_dir): > + import shutil > + > + local_artifacts_dir = os.path.join(target_dir, "artifacts") > + if os.path.isdir(local_artifacts_dir): > + shutil.rmtree(local_artifacts_dir) > + > + os.makedirs(local_artifacts_dir) > + for artifact_path in artifacts_list: > + if not os.path.isabs(artifact_path): > + bb.warn(f"{artifact_path} is not an absolute path") > + continue > + try: > + dest_dir = os.path.join(local_artifacts_dir, > os.path.dirname(artifact_path[1:])) > + os.makedirs(dest_dir, exist_ok=True) > + target.copyFrom(artifact_path, dest_dir) > + except Exception as e: > + bb.warn(f"Can not retrieve {artifact_path} from test > target: {e}") > + > +def list_and_fetch_failed_tests_artifacts(d, tc): > + artifacts_list = get_artifacts_list(tc.target, > d.getVar("TESTIMAGE_FAILED_QA_ARTIFACTS")) > + if not artifacts_list: > + bb.warn("Could not load artifacts list, skip artifacts > retrieval") > + else: > + retrieve_test_artifacts(tc.target, artifacts_list, > get_testimage_json_result_dir(d)) > + > + > +################################################################## > +# General post actions runner > +################################################################## > + > +def run_failed_tests_post_actions(d, tc): > + post_actions=[ > + list_and_fetch_failed_tests_artifacts > + ] > + > + for action in post_actions: > + action(d, tc) Rather than create a bbclass class of python functions, these should move to lib/oe and become a proper python library file? Moving functions out of the class files is on my long term todo list so this seems like an idea opportunity. Cheers, Richard
Hello Richard, On 2/21/24 08:34, Richard Purdie wrote: > On Tue, 2024-02-20 at 21:01 +0100, Alexis Lothoré via > lists.openembedded.org wrote: >> From: Alexis Lothoré <alexis.lothore@bootlin.com> [...] >> +################################################################## >> +# General post actions runner >> +################################################################## >> + >> +def run_failed_tests_post_actions(d, tc): >> + post_actions=[ >> + list_and_fetch_failed_tests_artifacts >> + ] >> + >> + for action in post_actions: >> + action(d, tc) > > Rather than create a bbclass class of python functions, these should > move to lib/oe and become a proper python library file? ACK, I will do that and create a proper python file for this. > Moving functions out of the class files is on my long term todo list so > this seems like an idea opportunity. So should this series take the opportunity to move all already existing python functions from testimage to a lib ? I can certainly do that if that's your point :)
On Wed, 2024-02-21 at 08:53 +0100, Alexis Lothoré wrote: > On 2/21/24 08:34, Richard Purdie wrote: > > On Tue, 2024-02-20 at 21:01 +0100, Alexis Lothoré via > > lists.openembedded.org wrote: > > > From: Alexis Lothoré <alexis.lothore@bootlin.com> > > [...] > > > > +################################################################ > > > ## > > > +# General post actions runner > > > +################################################################ > > > ## > > > + > > > +def run_failed_tests_post_actions(d, tc): > > > + post_actions=[ > > > + list_and_fetch_failed_tests_artifacts > > > + ] > > > + > > > + for action in post_actions: > > > + action(d, tc) > > > > Rather than create a bbclass class of python functions, these > > should > > move to lib/oe and become a proper python library file? > > ACK, I will do that and create a proper python file for this. > > > Moving functions out of the class files is on my long term todo > > list so > > this seems like an idea opportunity. > > So should this series take the opportunity to move all already > existing python > functions from testimage to a lib ? I can certainly do that if that's > your point :) I'm saying over time I think many of the python functions need to move. This close to feature freeze may not be the best time, I just wanted to give a clear view of my intent. Sometimes the variable dependencies don't work the same way from the python library so we need to be careful. Cheers, Richard
On 2/21/24 08:56, Richard Purdie wrote: > On Wed, 2024-02-21 at 08:53 +0100, Alexis Lothoré wrote: >> On 2/21/24 08:34, Richard Purdie wrote: >>> On Tue, 2024-02-20 at 21:01 +0100, Alexis Lothoré via >>> lists.openembedded.org wrote: >>>> From: Alexis Lothoré <alexis.lothore@bootlin.com> >> >> [...] >> >>>> +################################################################ >>>> ## >>>> +# General post actions runner >>>> +################################################################ >>>> ## >>>> + >>>> +def run_failed_tests_post_actions(d, tc): >>>> + post_actions=[ >>>> + list_and_fetch_failed_tests_artifacts >>>> + ] >>>> + >>>> + for action in post_actions: >>>> + action(d, tc) >>> >>> Rather than create a bbclass class of python functions, these >>> should >>> move to lib/oe and become a proper python library file? >> >> ACK, I will do that and create a proper python file for this. >> >>> Moving functions out of the class files is on my long term todo >>> list so >>> this seems like an idea opportunity. >> >> So should this series take the opportunity to move all already >> existing python >> functions from testimage to a lib ? I can certainly do that if that's >> your point :) > > I'm saying over time I think many of the python functions need to move. > This close to feature freeze may not be the best time, I just wanted to > give a clear view of my intent. > > Sometimes the variable dependencies don't work the same way from the > python library so we need to be careful. Ok, thanks for the clarification. I will then only fix my series to not create a new bbclass but a lib for now. > Cheers, > > Richard > > >
diff --git a/meta/classes-recipe/failed-tests-post-actions.bbclass b/meta/classes-recipe/failed-tests-post-actions.bbclass new file mode 100644 index 000000000000..7c7d3391298f --- /dev/null +++ b/meta/classes-recipe/failed-tests-post-actions.bbclass @@ -0,0 +1,64 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + + +################################################################## +# Artifacts retrieval +################################################################## + +def get_artifacts_list(target, raw_list): + result = [] + # Passed list may contains patterns in paths, expand them directly on target + for raw_path in raw_list.split(): + cmd = f"for p in {raw_path}; do if [ -e $p ]; then echo $p; fi; done" + try: + status, output = target.run(cmd) + if status != 0 or not output: + raise Exception() + result += output.split() + except: + bb.note(f"No file/directory matching path {raw_path}") + + return result + +def retrieve_test_artifacts(target, artifacts_list, target_dir): + import shutil + + local_artifacts_dir = os.path.join(target_dir, "artifacts") + if os.path.isdir(local_artifacts_dir): + shutil.rmtree(local_artifacts_dir) + + os.makedirs(local_artifacts_dir) + for artifact_path in artifacts_list: + if not os.path.isabs(artifact_path): + bb.warn(f"{artifact_path} is not an absolute path") + continue + try: + dest_dir = os.path.join(local_artifacts_dir, os.path.dirname(artifact_path[1:])) + os.makedirs(dest_dir, exist_ok=True) + target.copyFrom(artifact_path, dest_dir) + except Exception as e: + bb.warn(f"Can not retrieve {artifact_path} from test target: {e}") + +def list_and_fetch_failed_tests_artifacts(d, tc): + artifacts_list = get_artifacts_list(tc.target, d.getVar("TESTIMAGE_FAILED_QA_ARTIFACTS")) + if not artifacts_list: + bb.warn("Could not load artifacts list, skip artifacts retrieval") + else: + retrieve_test_artifacts(tc.target, artifacts_list, get_testimage_json_result_dir(d)) + + +################################################################## +# General post actions runner +################################################################## + +def run_failed_tests_post_actions(d, tc): + post_actions=[ + list_and_fetch_failed_tests_artifacts + ] + + for action in post_actions: + action(d, tc) diff --git a/meta/classes-recipe/testimage.bbclass b/meta/classes-recipe/testimage.bbclass index bee19674ef4f..d2b525d40f41 100644 --- a/meta/classes-recipe/testimage.bbclass +++ b/meta/classes-recipe/testimage.bbclass @@ -4,6 +4,7 @@ inherit metadata_scm inherit image-artifact-names +inherit failed-tests-post-actions # testimage.bbclass enables testing of qemu images using python unittests. # Most of the tests are commands run on target image over ssh. @@ -177,40 +178,6 @@ def get_testimage_boot_patterns(d): boot_patterns[flag] = flagval.encode().decode('unicode-escape') return boot_patterns -def get_artifacts_list(target, raw_list): - result = [] - # Passed list may contains patterns in paths, expand them directly on target - for raw_path in raw_list.split(): - cmd = f"for p in {raw_path}; do if [ -e $p ]; then echo $p; fi; done" - try: - status, output = target.run(cmd) - if status != 0 or not output: - raise Exception() - result += output.split() - except: - bb.note(f"No file/directory matching path {raw_path}") - - return result - -def retrieve_test_artifacts(target, artifacts_list, target_dir): - import shutil - - local_artifacts_dir = os.path.join(target_dir, "artifacts") - if os.path.isdir(local_artifacts_dir): - shutil.rmtree(local_artifacts_dir) - - os.makedirs(local_artifacts_dir) - for artifact_path in artifacts_list: - if not os.path.isabs(artifact_path): - bb.warn(f"{artifact_path} is not an absolute path") - continue - try: - dest_dir = os.path.join(local_artifacts_dir, os.path.dirname(artifact_path[1:])) - os.makedirs(dest_dir, exist_ok=True) - target.copyFrom(artifact_path, dest_dir) - except Exception as e: - bb.warn(f"Can not retrieve {artifact_path} from test target: {e}") - def testimage_main(d): import os import json @@ -406,12 +373,7 @@ def testimage_main(d): results = tc.runTests() complete = True if results.hasAnyFailingTest(): - artifacts_list = get_artifacts_list(tc.target, d.getVar("TESTIMAGE_FAILED_QA_ARTIFACTS")) - if not artifacts_list: - bb.warn("Could not load artifacts list, skip artifacts retrieval") - else: - bb.warn(f"Retrieving artifacts: {d.getVar('TESTIMAGE_FAILED_QA_ARTIFACTS')}") - retrieve_test_artifacts(tc.target, artifacts_list, get_testimage_json_result_dir(d)) + run_failed_tests_post_actions(d, tc) except (KeyboardInterrupt, BlockingIOError) as err: if isinstance(err, KeyboardInterrupt): bb.error('testimage interrupted, shutting down...')