From patchwork Wed Dec 6 22:43:05 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Marlon Rodriguez Garcia X-Patchwork-Id: 35823 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id E7FC5C46CA3 for ; Wed, 6 Dec 2023 22:43:31 +0000 (UTC) Received: from mail.savoirfairelinux.com (mail.savoirfairelinux.com [208.88.110.44]) by mx.groups.io with SMTP id smtpd.web10.51544.1701902601839878470 for ; Wed, 06 Dec 2023 14:43:22 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@savoirfairelinux.com header.s=DFC430D2-D198-11EC-948E-34200CB392D2 header.b=YRMz/UoS; spf=pass (domain: savoirfairelinux.com, ip: 208.88.110.44, mailfrom: marlon.rodriguez-garcia@savoirfairelinux.com) Received: from localhost (localhost [127.0.0.1]) by mail.savoirfairelinux.com (Postfix) with ESMTP id 1B6019C3645; Wed, 6 Dec 2023 17:43:21 -0500 (EST) Received: from mail.savoirfairelinux.com ([127.0.0.1]) by localhost (mail.savoirfairelinux.com [127.0.0.1]) (amavis, port 10032) with ESMTP id hEYQ4h7n1KAv; Wed, 6 Dec 2023 17:43:20 -0500 (EST) Received: from localhost (localhost [127.0.0.1]) by mail.savoirfairelinux.com (Postfix) with ESMTP id 4658F9C406A; Wed, 6 Dec 2023 17:43:20 -0500 (EST) DKIM-Filter: OpenDKIM Filter v2.10.3 mail.savoirfairelinux.com 4658F9C406A DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=savoirfairelinux.com; s=DFC430D2-D198-11EC-948E-34200CB392D2; t=1701902600; bh=44DQ/ovLoq9/yvERhR2Ke//3Tlv6K3kjOSHAKtvpNNc=; h=From:To:Date:Message-Id:MIME-Version; b=YRMz/UoSA++dg8Ii55NscNFPZ/Yu/U1lEBvEzNfoJpH3FUcFwZSL7DzvgPg9ur9lW mGqgT11xuxgImf1P0SrfpaGWiAhnUomvSSvPvJoPyeGHHwq8RKeMYEaqW7UMSeKerB /z0E7yWIkW4+jHEHhgGfNUdrXTfCEEufmDoASVHLd3XKV0xUN++WlDV5JnEeBxY3wS mcyhitlWb/OYa/GAzpWyncgBkf9dWnnjBXPJMfYo3PacO9hw7l5YNyfZYUjw1tWZ8i KPBaCvc9gAqPIGAyMLvHgBpaPbzMsXyZjPykIfaeu3aKF+b5MUHAq+Y9Fqf3DoKmOq w3bqxJ9oAwn4w== X-Virus-Scanned: amavis at mail.savoirfairelinux.com Received: from mail.savoirfairelinux.com ([127.0.0.1]) by localhost (mail.savoirfairelinux.com [127.0.0.1]) (amavis, port 10026) with ESMTP id xDusbt5C26sX; Wed, 6 Dec 2023 17:43:20 -0500 (EST) Received: from savoirfairelinux.ht.home (modemcable141.201-58-74.mc.videotron.ca [74.58.201.141]) by mail.savoirfairelinux.com (Postfix) with ESMTPSA id 2425B9C3552; Wed, 6 Dec 2023 17:43:20 -0500 (EST) From: Marlon Rodriguez Garcia To: bitbake-devel@lists.openembedded.org, toaster@lists.yoctoproject.org Cc: Marlon Rodriguez Garcia Subject: [toaster][PATCHv3 3/3] toaster: Update eventreplay functionality for new eventlog file structure Date: Wed, 6 Dec 2023 17:43:05 -0500 Message-Id: <20231206224305.34686-4-marlon.rodriguez-garcia@savoirfairelinux.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20231206224305.34686-1-marlon.rodriguez-garcia@savoirfairelinux.com> References: <20231206224305.34686-1-marlon.rodriguez-garcia@savoirfairelinux.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Wed, 06 Dec 2023 22:43:31 -0000 X-Groupsio-URL: https://lists.yoctoproject.org/g/toaster/message/6057 Added class EventPlayer to list of libraries under bitbake/bb/ui/ Update file read functionality to match new eventlog format Exclude listing of files that don't contain the allvariables definitions used to replay builds This part of the feature should be revisited. Over a long period of time, the BB_DEFAULT_EVENTLOG will exponentially increase the size of the log file and cause bottlenecks when importing. Signed-off-by: Marlon Rodriguez Garcia --- lib/bb/ui/eventreplay.py | 85 +++++++++++++++++++++ lib/toaster/toastergui/views.py | 131 ++++++++++++-------------------- 2 files changed, 132 insertions(+), 84 deletions(-) create mode 100644 lib/bb/ui/eventreplay.py diff --git a/lib/bb/ui/eventreplay.py b/lib/bb/ui/eventreplay.py new file mode 100644 index 00000000..b5999a1b --- /dev/null +++ b/lib/bb/ui/eventreplay.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +# +# SPDX-License-Identifier: GPL-2.0-only +# +# This file re-uses code spread throughout other Bitbake source files. +# As such, all other copyrights belong to their own right holders. +# + + +import os +import sys +import json +import pickle +import codecs + + +class EventPlayer: + """Emulate a connection to a bitbake server.""" + + def __init__(self, eventfile, variables): + self.eventfile = eventfile + self.variables = variables + self.eventmask = [] + + def waitEvent(self, _timeout): + """Read event from the file.""" + line = self.eventfile.readline().strip() + if not line: + return + try: + decodedline = json.loads(line) + if 'allvariables' in decodedline: + return + if not 'vars' in decodedline: + raise ValueError + event_str = decodedline['vars'].encode('utf-8') + event = pickle.loads(codecs.decode(event_str, 'base64')) + event_name = "%s.%s" % (event.__module__, event.__class__.__name__) + if event_name not in self.eventmask: + return + return event + except ValueError as err: + print("Failed loading ", line) + raise err + + def runCommand(self, command_line): + """Emulate running a command on the server.""" + name = command_line[0] + + if name == "getVariable": + var_name = command_line[1] + variable = self.variables.get(var_name) + if variable: + return variable['v'], None + return None, "Missing variable %s" % var_name + + elif name == "getAllKeysWithFlags": + dump = {} + flaglist = command_line[1] + for key, val in self.variables.items(): + try: + if not key.startswith("__"): + dump[key] = { + 'v': val['v'], + 'history' : val['history'], + } + for flag in flaglist: + dump[key][flag] = val[flag] + except Exception as err: + print(err) + return (dump, None) + + elif name == 'setEventMask': + self.eventmask = command_line[-1] + return True, None + + else: + raise Exception("Command %s not implemented" % command_line[0]) + + def getEventHandle(self): + """ + This method is called by toasterui. + The return value is passed to self.runCommand but not used there. + """ + pass \ No newline at end of file diff --git a/lib/toaster/toastergui/views.py b/lib/toaster/toastergui/views.py index 2e34c8a2..cac72d36 100644 --- a/lib/toaster/toastergui/views.py +++ b/lib/toaster/toastergui/views.py @@ -8,12 +8,12 @@ import ast import re -import pickle -import codecs import subprocess +import sys import bb.cooker from bb.ui import toasterui +from bb.ui import eventreplay from django.db.models import F, Q, Sum from django.db import IntegrityError @@ -1957,71 +1957,6 @@ if True: except (ObjectDoesNotExist, IOError): return toaster_render(request, "unavailable_artifact.html") -class EventPlayer: - """Emulate a connection to a bitbake server.""" - - def __init__(self, eventfile, variables): - self.eventfile = eventfile - self.variables = variables - self.eventmask = [] - - def waitEvent(self, _timeout): - """Read event from the file.""" - line = self.eventfile.readline().strip() - if not line: - return - try: - event_str = json.loads(line)['vars'].encode('utf-8') - event = pickle.loads(codecs.decode(event_str, 'base64')) - event_name = "%s.%s" % (event.__module__, event.__class__.__name__) - if event_name not in self.eventmask: - return - return event - except ValueError as err: - print("Failed loading ", line) - raise err - - def runCommand(self, command_line): - """Emulate running a command on the server.""" - name = command_line[0] - - if name == "getVariable": - var_name = command_line[1] - variable = self.variables.get(var_name) - if variable: - return variable['v'], None - return None, "Missing variable %s" % var_name - - elif name == "getAllKeysWithFlags": - dump = {} - flaglist = command_line[1] - for key, val in self.variables.items(): - try: - if not key.startswith("__"): - dump[key] = { - 'v': val['v'], - 'history' : val['history'], - } - for flag in flaglist: - dump[key][flag] = val[flag] - except Exception as err: - print(err) - return (dump, None) - - elif name == 'setEventMask': - self.eventmask = command_line[-1] - return True, None - - else: - raise Exception("Command %s not implemented" % command_line[0]) - - def getEventHandle(self): - """ - This method is called by toasterui. - The return value is passed to self.runCommand but not used there. - """ - pass - class CommandLineBuilds(TemplateView): model = EventLogsImports @@ -2038,7 +1973,14 @@ class CommandLineBuilds(TemplateView): files_list = [] # Filter files that end with ".json" - event_files = [file for file in files if file.endswith(".json")] + event_files = [] + for file in files: + if file.endswith(".json"): + # because BB_DEFAULT_EVENTLOG is a directory, we need to check if the file is a valid eventlog + with open("{}/{}".format(logs_dir, file)) as efile: + content = efile.read() + if 'allvariables' in content: + event_files.append(file) #build dict for template using db data for event_file in event_files: @@ -2077,7 +2019,6 @@ class CommandLineBuilds(TemplateView): imported_files = EventLogsImports.objects.all() try: if all_files == 'true': - # use of session variable to deactivate icon for builds in progress request.session['all_builds'] = True request.session.modified = True @@ -2090,10 +2031,17 @@ class CommandLineBuilds(TemplateView): else: with open("{}/{}".format(logs_dir, file.get('name'))) as eventfile: # load variables from the first line - variables = json.loads(eventfile.readline().strip())['allvariables'] - + variables = None + while line := eventfile.readline().strip(): + try: + variables = json.loads(line)['allvariables'] + break + except (KeyError, json.JSONDecodeError): + continue + if not variables: + raise Exception("File content missing build variables") params = namedtuple('ConfigParams', ['observe_only'])(True) - player = EventPlayer(eventfile, variables) + player = eventreplay.EventPlayer(eventfile, variables) toasterui.main(player, player, params) event_log_import = EventLogsImports.objects.create(name=file.get('name'), imported=True) @@ -2104,20 +2052,27 @@ class CommandLineBuilds(TemplateView): file = self.request.FILES['eventlog_file'] else: file = request.POST.get('file') - - # use of session variable to deactivate icon for build in progress - request.session['file'] = file - request.session['all_builds'] = False - request.session.modified = True - request.session.save() + # use of session variable to deactivate icon for build in progress + request.session['file'] = file + request.session['all_builds'] = False + request.session.modified = True + request.session.save() if imported_files.filter(name=file).exists(): imported_files.filter(name=file)[0].imported = True else: if isinstance(file, InMemoryUploadedFile) or isinstance(file, TemporaryUploadedFile): - variables = json.loads(file.readline().strip())['allvariables'] + variables = None + while line := file.readline().strip(): + try: + variables = json.loads(line)['allvariables'] + break + except (KeyError, json.JSONDecodeError): + continue + if not variables: + raise Exception("File content missing build variables") params = namedtuple('ConfigParams', ['observe_only'])(True) - player = EventPlayer(file, variables) + player = eventreplay.EventPlayer(file, variables) if not os.path.exists('{}/{}'.format(logs_dir, file.name)): fs = FileSystemStorage(location=logs_dir) fs.save(file.name, file) @@ -2125,18 +2080,26 @@ class CommandLineBuilds(TemplateView): else: with open("{}/{}".format(logs_dir, file)) as eventfile: # load variables from the first line - variables = json.loads(eventfile.readline().strip())['allvariables'] + variables = None + while line := eventfile.readline().strip(): + try: + variables = json.loads(line)['allvariables'] + break + except (KeyError, json.JSONDecodeError): + continue + if not variables: + raise Exception("File content missing build variables") params = namedtuple('ConfigParams', ['observe_only'])(True) - player = EventPlayer(eventfile, variables) + player = eventreplay.EventPlayer(eventfile, variables) toasterui.main(player, player, params) event_log_import = EventLogsImports.objects.create(name=file, imported=True) event_log_import.build_id = Build.objects.last().id event_log_import.save() request.session['file'] = "" - except json.decoder.JSONDecodeError: + except Exception: messages.add_message( self.request, - messages.SUCCESS, + messages.ERROR, "The file content is not in the correct format. Update file content or upload a different file." ) return HttpResponseRedirect("/toastergui/cmdline/")