From patchwork Sat Sep 23 13:04:07 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ross Burton X-Patchwork-Id: 31051 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 4AD50CE7A89 for ; Sat, 23 Sep 2023 13:04:19 +0000 (UTC) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mx.groups.io with SMTP id smtpd.web11.20090.1695474255907103958 for ; Sat, 23 Sep 2023 06:04:16 -0700 Authentication-Results: mx.groups.io; dkim=none (message not signed); spf=pass (domain: arm.com, ip: 217.140.110.172, mailfrom: ross.burton@arm.com) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 8D3DADA7; Sat, 23 Sep 2023 06:04:52 -0700 (PDT) Received: from oss-tx204.lab.cambridge.arm.com (usa-sjc-imap-foss1.foss.arm.com [10.121.207.14]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 784933F59C; Sat, 23 Sep 2023 06:04:14 -0700 (PDT) From: ross.burton@arm.com To: openembedded-core@lists.openembedded.org Cc: nd@arm.com Subject: [PATCH 1/5] oeqa/runtime/parselogs: improve find call Date: Sat, 23 Sep 2023 14:04:07 +0100 Message-Id: <20230923130411.766482-2-ross.burton@arm.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20230923130411.766482-1-ross.burton@arm.com> References: <20230923130411.766482-1-ross.burton@arm.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Sat, 23 Sep 2023 13:04:19 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/188146 From: Ross Burton getLogList() uses remote find invocations to find the logs. Instead of relying on shell expansion of wildcards and redundant use of -maxdepth (pointless as the shell expansion means the find is passed the files to return), invoke find idiomatically by telling it what directory to search for and escape the glob so find processes it. Also remove many pointless str() calls. Signed-off-by: Ross Burton --- meta/lib/oeqa/runtime/cases/parselogs.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/meta/lib/oeqa/runtime/cases/parselogs.py b/meta/lib/oeqa/runtime/cases/parselogs.py index 6e5dc753060..93782b844b9 100644 --- a/meta/lib/oeqa/runtime/cases/parselogs.py +++ b/meta/lib/oeqa/runtime/cases/parselogs.py @@ -229,18 +229,18 @@ class ParseLogsTest(OERuntimeTestCase): def getLogList(self, log_locations): logs = [] for location in log_locations: - status, _ = self.target.run('test -f ' + str(location)) + status, _ = self.target.run('test -f %s' % location) if status == 0: - logs.append(str(location)) + logs.append(location) else: - status, _ = self.target.run('test -d ' + str(location)) + status, _ = self.target.run('test -d %s' % location) if status == 0: - cmd = 'find ' + str(location) + '/*.log -maxdepth 1 -type f' + cmd = 'find %s -name \\*.log -maxdepth 1 -type f' % location status, output = self.target.run(cmd) if status == 0: output = output.splitlines() for logfile in output: - logs.append(os.path.join(location, str(logfile))) + logs.append(os.path.join(location, logfile)) return logs # Copy the log files to be parsed locally From patchwork Sat Sep 23 13:04:08 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ross Burton X-Patchwork-Id: 31050 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 4D0E4CE7A8C for ; Sat, 23 Sep 2023 13:04:19 +0000 (UTC) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mx.groups.io with SMTP id smtpd.web11.20091.1695474256573897798 for ; Sat, 23 Sep 2023 06:04:16 -0700 Authentication-Results: mx.groups.io; dkim=none (message not signed); spf=pass (domain: arm.com, ip: 217.140.110.172, mailfrom: ross.burton@arm.com) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 394BB1007; Sat, 23 Sep 2023 06:04:53 -0700 (PDT) Received: from oss-tx204.lab.cambridge.arm.com (usa-sjc-imap-foss1.foss.arm.com [10.121.207.14]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 247093F59C; Sat, 23 Sep 2023 06:04:15 -0700 (PDT) From: ross.burton@arm.com To: openembedded-core@lists.openembedded.org Cc: nd@arm.com Subject: [PATCH 2/5] oeqa/runtime/parselogs: don't pass around members Date: Sat, 23 Sep 2023 14:04:08 +0100 Message-Id: <20230923130411.766482-3-ross.burton@arm.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20230923130411.766482-1-ross.burton@arm.com> References: <20230923130411.766482-1-ross.burton@arm.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Sat, 23 Sep 2023 13:04:19 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/188147 From: Ross Burton There's no point in passing around member fields, just access them directly. Signed-off-by: Ross Burton --- meta/lib/oeqa/runtime/cases/parselogs.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/meta/lib/oeqa/runtime/cases/parselogs.py b/meta/lib/oeqa/runtime/cases/parselogs.py index 93782b844b9..0262f574d1a 100644 --- a/meta/lib/oeqa/runtime/cases/parselogs.py +++ b/meta/lib/oeqa/runtime/cases/parselogs.py @@ -263,19 +263,19 @@ class ParseLogsTest(OERuntimeTestCase): return logs # Build the grep command to be used with filters and exclusions - def build_grepcmd(self, errors, ignore_errors, log): + def build_grepcmd(self, log): grepcmd = 'grep ' grepcmd += '-Ei "' - for error in errors: + for error in self.errors: grepcmd += r'\<' + error + r'\>' + '|' grepcmd = grepcmd[:-1] grepcmd += '" ' + str(log) + " | grep -Eiv \'" try: - errorlist = ignore_errors[self.td.get('MACHINE')] + errorlist = self.ignore_errors[self.td.get('MACHINE')] except KeyError: self.msg += 'No ignore list found for this machine, using default\n' - errorlist = ignore_errors['default'] + errorlist = self.ignore_errors['default'] for ignore_error in errorlist: ignore_error = ignore_error.replace('(', r'\(') @@ -292,17 +292,21 @@ class ParseLogsTest(OERuntimeTestCase): return grepcmd - # Grep only the errors so that their context could be collected. - # Default context is 10 lines before and after the error itself - def parse_logs(self, errors, ignore_errors, logs, - lines_before = 10, lines_after = 10): + def parse_logs(self, logs, lines_before=10, lines_after=10): + """ + Search the log files @logs looking for error lines (marked by + @self.errors), ignoring anything listed in @self.ignore_errors. + + Returns a dictionary of log filenames to a dictionary of error lines to + the error context (controlled by @lines_before and @lines_after). + """ results = {} rez = [] grep_output = '' for log in logs: result = None - thegrep = self.build_grepcmd(errors, ignore_errors, log) + thegrep = self.build_grepcmd(log) try: result = check_output(thegrep, shell=True).decode('utf-8') @@ -333,7 +337,7 @@ class ParseLogsTest(OERuntimeTestCase): def test_parselogs(self): self.write_dmesg() log_list = self.get_local_log_list(self.log_locations) - result = self.parse_logs(self.errors, self.ignore_errors, log_list) + result = self.parse_logs(log_list) errcount = 0 for log in result: self.msg += 'Log: ' + log + '\n' From patchwork Sat Sep 23 13:04:09 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ross Burton X-Patchwork-Id: 31048 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 58AECCE7A8B for ; Sat, 23 Sep 2023 13:04:19 +0000 (UTC) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mx.groups.io with SMTP id smtpd.web10.19766.1695474256582263521 for ; Sat, 23 Sep 2023 06:04:17 -0700 Authentication-Results: mx.groups.io; dkim=none (message not signed); spf=pass (domain: arm.com, ip: 217.140.110.172, mailfrom: ross.burton@arm.com) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id D992CC15; Sat, 23 Sep 2023 06:04:53 -0700 (PDT) Received: from oss-tx204.lab.cambridge.arm.com (usa-sjc-imap-foss1.foss.arm.com [10.121.207.14]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id C4A793F59C; Sat, 23 Sep 2023 06:04:15 -0700 (PDT) From: ross.burton@arm.com To: openembedded-core@lists.openembedded.org Cc: nd@arm.com Subject: [PATCH 3/5] oeqa/runtime/parselogs: move some variables out of global scope Date: Sat, 23 Sep 2023 14:04:09 +0100 Message-Id: <20230923130411.766482-4-ross.burton@arm.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20230923130411.766482-1-ross.burton@arm.com> References: <20230923130411.766482-1-ross.burton@arm.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Sat, 23 Sep 2023 13:04:19 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/188148 From: Ross Burton errors and log_locations can be trivially set in the class directly, instead of being defined in the module and then copied into the class. Signed-off-by: Ross Burton --- meta/lib/oeqa/runtime/cases/parselogs.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/meta/lib/oeqa/runtime/cases/parselogs.py b/meta/lib/oeqa/runtime/cases/parselogs.py index 0262f574d1a..3f205661ea9 100644 --- a/meta/lib/oeqa/runtime/cases/parselogs.py +++ b/meta/lib/oeqa/runtime/cases/parselogs.py @@ -11,9 +11,6 @@ from shutil import rmtree from oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends -#in the future these lists could be moved outside of module -errors = ["error", "cannot", "can\'t", "failed"] - common_errors = [ "(WW) warning, (EE) error, (NI) not implemented, (??) unknown.", "dma timeout", @@ -201,17 +198,19 @@ ignore_errors = { ] + common_errors, } -log_locations = ["/var/log/","/var/log/dmesg", "/tmp/dmesg_output.log"] - class ParseLogsTest(OERuntimeTestCase): + # Which log files should be collected + log_locations = ["/var/log/", "/var/log/dmesg", "/tmp/dmesg_output.log"] + + # The keywords that identify error messages in the log files + errors = ["error", "cannot", "can't", "failed"] + @classmethod def setUpClass(cls): - cls.errors = errors - # When systemd is enabled we need to notice errors on # circular dependencies in units. - if 'systemd' in cls.td.get('DISTRO_FEATURES', ''): + if 'systemd' in cls.td.get('DISTRO_FEATURES'): cls.errors.extend([ 'Found ordering cycle on', 'Breaking ordering cycle by deleting job', @@ -220,8 +219,6 @@ class ParseLogsTest(OERuntimeTestCase): ]) cls.ignore_errors = ignore_errors - cls.log_locations = log_locations - cls.msg = '' # Go through the log locations provided and if it's a folder # create a list with all the .log files in it, if it's a file @@ -338,7 +335,9 @@ class ParseLogsTest(OERuntimeTestCase): self.write_dmesg() log_list = self.get_local_log_list(self.log_locations) result = self.parse_logs(log_list) + errcount = 0 + self.msg = "" for log in result: self.msg += 'Log: ' + log + '\n' self.msg += '-----------------------\n' From patchwork Sat Sep 23 13:04:10 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ross Burton X-Patchwork-Id: 31049 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 5D5E3CE7A8F for ; Sat, 23 Sep 2023 13:04:19 +0000 (UTC) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mx.groups.io with SMTP id smtpd.web11.20092.1695474257259161284 for ; Sat, 23 Sep 2023 06:04:17 -0700 Authentication-Results: mx.groups.io; dkim=none (message not signed); spf=pass (domain: arm.com, ip: 217.140.110.172, mailfrom: ross.burton@arm.com) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 85C29DA7; Sat, 23 Sep 2023 06:04:54 -0700 (PDT) Received: from oss-tx204.lab.cambridge.arm.com (usa-sjc-imap-foss1.foss.arm.com [10.121.207.14]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 70AE03F59C; Sat, 23 Sep 2023 06:04:16 -0700 (PDT) From: ross.burton@arm.com To: openembedded-core@lists.openembedded.org Cc: nd@arm.com Subject: [PATCH 4/5] oeqa/runtime/parselogs: select the correct machine-specific ignores early Date: Sat, 23 Sep 2023 14:04:10 +0100 Message-Id: <20230923130411.766482-5-ross.burton@arm.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20230923130411.766482-1-ross.burton@arm.com> References: <20230923130411.766482-1-ross.burton@arm.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Sat, 23 Sep 2023 13:04:19 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/188149 From: Ross Burton This has no impact to the execution, but makes the following changes neater. Signed-off-by: Ross Burton --- meta/lib/oeqa/runtime/cases/parselogs.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/meta/lib/oeqa/runtime/cases/parselogs.py b/meta/lib/oeqa/runtime/cases/parselogs.py index 3f205661ea9..3980582dda2 100644 --- a/meta/lib/oeqa/runtime/cases/parselogs.py +++ b/meta/lib/oeqa/runtime/cases/parselogs.py @@ -218,7 +218,11 @@ class ParseLogsTest(OERuntimeTestCase): 'Ordering cycle found, skipping', ]) - cls.ignore_errors = ignore_errors + try: + cls.ignore_errors = ignore_errors[cls.td.get('MACHINE')] + except KeyError: + cls.logger.info('No ignore list found for this machine, using default') + cls.ignore_errors = ignore_errors['default'] # Go through the log locations provided and if it's a folder # create a list with all the .log files in it, if it's a file @@ -268,13 +272,8 @@ class ParseLogsTest(OERuntimeTestCase): grepcmd = grepcmd[:-1] grepcmd += '" ' + str(log) + " | grep -Eiv \'" - try: - errorlist = self.ignore_errors[self.td.get('MACHINE')] - except KeyError: - self.msg += 'No ignore list found for this machine, using default\n' - errorlist = self.ignore_errors['default'] - for ignore_error in errorlist: + for ignore_error in self.ignore_errors: ignore_error = ignore_error.replace('(', r'\(') ignore_error = ignore_error.replace(')', r'\)') ignore_error = ignore_error.replace("'", '.') From patchwork Sat Sep 23 13:04:11 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ross Burton X-Patchwork-Id: 31052 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 66C44CE7A91 for ; Sat, 23 Sep 2023 13:04:19 +0000 (UTC) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mx.groups.io with SMTP id smtpd.web10.19768.1695474257889924598 for ; Sat, 23 Sep 2023 06:04:18 -0700 Authentication-Results: mx.groups.io; dkim=none (message not signed); spf=pass (domain: arm.com, ip: 217.140.110.172, mailfrom: ross.burton@arm.com) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 31E5F1007; Sat, 23 Sep 2023 06:04:55 -0700 (PDT) Received: from oss-tx204.lab.cambridge.arm.com (usa-sjc-imap-foss1.foss.arm.com [10.121.207.14]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 1D1E03F59C; Sat, 23 Sep 2023 06:04:17 -0700 (PDT) From: ross.burton@arm.com To: openembedded-core@lists.openembedded.org Cc: nd@arm.com Subject: [PATCH 5/5] oeqa/runtime/parselogs: parse the logs with Python, not grep Date: Sat, 23 Sep 2023 14:04:11 +0100 Message-Id: <20230923130411.766482-6-ross.burton@arm.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20230923130411.766482-1-ross.burton@arm.com> References: <20230923130411.766482-1-ross.burton@arm.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Sat, 23 Sep 2023 13:04:19 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/188150 From: Ross Burton Instead of constructing huge grep statements, we can simply open the logs in Python and do the relevant string operations directly. The trick is to remember to casefold() all of the strings, so that the "in" operator can be used. Just one of the ignores needs to be adjusted because it uses a regular expression and the new logic doesn't support that. This is handled by simply reducing the size of the ignore match. Signed-off-by: Ross Burton --- meta/lib/oeqa/runtime/cases/parselogs.py | 91 +++++++++++------------- 1 file changed, 41 insertions(+), 50 deletions(-) diff --git a/meta/lib/oeqa/runtime/cases/parselogs.py b/meta/lib/oeqa/runtime/cases/parselogs.py index 3980582dda2..a805edd79dd 100644 --- a/meta/lib/oeqa/runtime/cases/parselogs.py +++ b/meta/lib/oeqa/runtime/cases/parselogs.py @@ -4,9 +4,9 @@ # SPDX-License-Identifier: MIT # +import collections import os -from subprocess import check_output from shutil import rmtree from oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends @@ -64,6 +64,7 @@ common_errors = [ "xf86OpenConsole: Switching VT failed", "Failed to read LoaderConfigTimeoutOneShot variable, ignoring: Operation not supported", "Failed to read LoaderEntryOneShot variable, ignoring: Operation not supported", + "invalid BAR (can't size)", ] x86_common = [ @@ -97,16 +98,13 @@ ignore_errors = { 'default' : common_errors, 'qemux86' : [ 'Failed to access perfctr msr (MSR', - 'pci 0000:00:00.0: [Firmware Bug]: reg 0x..: invalid BAR (can\'t size)', ] + qemux86_common, 'qemux86-64' : qemux86_common, 'qemumips' : [ 'Failed to load module "glx"', - 'pci 0000:00:00.0: [Firmware Bug]: reg 0x..: invalid BAR (can\'t size)', 'cacheinfo: Failed to find cpu0 device node', ] + common_errors, 'qemumips64' : [ - 'pci 0000:00:00.0: [Firmware Bug]: reg 0x..: invalid BAR (can\'t size)', 'cacheinfo: Failed to find cpu0 device node', ] + common_errors, 'qemuppc' : [ @@ -218,11 +216,13 @@ class ParseLogsTest(OERuntimeTestCase): 'Ordering cycle found, skipping', ]) + cls.errors = [s.casefold() for s in cls.errors] + try: - cls.ignore_errors = ignore_errors[cls.td.get('MACHINE')] + cls.ignore_errors = [s.casefold() for s in ignore_errors[cls.td.get('MACHINE')]] except KeyError: cls.logger.info('No ignore list found for this machine, using default') - cls.ignore_errors = ignore_errors['default'] + cls.ignore_errors = [s.casefold() for s in ignore_errors['default']] # Go through the log locations provided and if it's a folder # create a list with all the .log files in it, if it's a file @@ -263,30 +263,33 @@ class ParseLogsTest(OERuntimeTestCase): logs = [f for f in dir_files if os.path.isfile(f)] return logs - # Build the grep command to be used with filters and exclusions - def build_grepcmd(self, log): - grepcmd = 'grep ' - grepcmd += '-Ei "' - for error in self.errors: - grepcmd += r'\<' + error + r'\>' + '|' - grepcmd = grepcmd[:-1] - grepcmd += '" ' + str(log) + " | grep -Eiv \'" + def get_context(self, lines, index, before=6, after=3): + """ + Given a set of lines and the index of the line that is important, return + a number of lines surrounding that line. + """ + last = len(lines) + start = index - before + end = index + after + 1 - for ignore_error in self.ignore_errors: - ignore_error = ignore_error.replace('(', r'\(') - ignore_error = ignore_error.replace(')', r'\)') - ignore_error = ignore_error.replace("'", '.') - ignore_error = ignore_error.replace('?', r'\?') - ignore_error = ignore_error.replace('[', r'\[') - ignore_error = ignore_error.replace(']', r'\]') - ignore_error = ignore_error.replace('*', r'\*') - ignore_error = ignore_error.replace('0-9', '[0-9]') - grepcmd += ignore_error + '|' - grepcmd = grepcmd[:-1] - grepcmd += "\'" + if start < 0: + end -= start + start = 0 + if end > last: + start -= end - last + end = last - return grepcmd + return lines[start:end] + + def test_get_context(self): + """ + A test case for the test case. + """ + lines = list(range(0,10)) + self.assertEqual(self.get_context(lines, 0, 2, 1), [0, 1, 2, 3]) + self.assertEqual(self.get_context(lines, 5, 2, 1), [3, 4, 5, 6]) + self.assertEqual(self.get_context(lines, 9, 2, 1), [6, 7, 8, 9]) def parse_logs(self, logs, lines_before=10, lines_after=10): """ @@ -296,31 +299,19 @@ class ParseLogsTest(OERuntimeTestCase): Returns a dictionary of log filenames to a dictionary of error lines to the error context (controlled by @lines_before and @lines_after). """ - results = {} - rez = [] - grep_output = '' + results = collections.defaultdict(dict) for log in logs: - result = None - thegrep = self.build_grepcmd(log) - - try: - result = check_output(thegrep, shell=True).decode('utf-8') - except: - pass + with open(log) as f: + lines = f.readlines() - if result is not None: - results[log] = {} - rez = result.splitlines() + for i, line in enumerate(lines): + line = line.strip() + line_lower = line.casefold() - for xrez in rez: - try: - cmd = ['grep', '-F', xrez, '-B', str(lines_before)] - cmd += ['-A', str(lines_after), log] - grep_output = check_output(cmd).decode('utf-8') - except: - pass - results[log][xrez]=grep_output + if any(keyword in line_lower for keyword in self.errors): + if not any(ignore in line_lower for ignore in self.ignore_errors): + results[log][line] = "".join(self.get_context(lines, i, lines_before, lines_after)) return results @@ -342,9 +333,9 @@ class ParseLogsTest(OERuntimeTestCase): self.msg += '-----------------------\n' for error in result[log]: errcount += 1 - self.msg += 'Central error: ' + str(error) + '\n' + self.msg += 'Central error: ' + error + '\n' self.msg += '***********************\n' - self.msg += result[str(log)][str(error)] + '\n' + self.msg += result[log][error] + '\n' self.msg += '***********************\n' self.msg += '%s errors found in logs.' % errcount self.assertEqual(errcount, 0, msg=self.msg)