@@ -385,7 +385,7 @@ def main(server, eventHandler, params):
main.shutdown = 1
logger.info("ToasterUI build done, brbe: %s", brbe)
- continue
+ break
if isinstance(event, (bb.command.CommandCompleted,
bb.command.CommandFailed,
new file mode 100644
@@ -0,0 +1,22 @@
+# Generated by Django 4.2.5 on 2023-11-23 18:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('orm', '0020_models_bigautofield'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='EventLogsImports',
+ fields=[
+ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('name', models.CharField(max_length=255)),
+ ('imported', models.BooleanField(default=False)),
+ ('build_id', models.IntegerField(blank=True, null=True)),
+ ],
+ ),
+ ]
@@ -1868,6 +1868,15 @@ class Distro(models.Model):
def __unicode__(self):
return "Distro " + self.name + "(" + self.description + ")"
+class EventLogsImports(models.Model):
+ name = models.CharField(max_length=255)
+ imported = models.BooleanField(default=False)
+ build_id = models.IntegerField(blank=True, null=True)
+
+ def __str__(self):
+ return self.name
+
+
django.db.models.signals.post_save.connect(invalidate_cache)
django.db.models.signals.post_delete.connect(invalidate_cache)
django.db.models.signals.m2m_changed.connect(invalidate_cache)
new file mode 100644
@@ -0,0 +1,13 @@
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2013 Intel Corporation
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+from django import forms
+from django.core.validators import FileExtensionValidator
+
+class LoadFileForm(forms.Form):
+ eventlog_file = forms.FileField(widget=forms.FileInput(attrs={'accept': '.json'}))
@@ -132,7 +132,8 @@
{% if project_enable %}
<a class="btn btn-default navbar-btn navbar-right" id="new-project-button" href="{% url 'newproject' %}">New project</a>
{% endif %}
- </div>
+ <a class="btn btn-default navbar-btn navbar-right" id="import_page" style="margin-right: 5px !important" id="import-cmdline-button" href="{% url 'cmdlines' %}">Import command line builds</a>
+ </div>
</div>
</nav>
new file mode 100644
@@ -0,0 +1,171 @@
+{% extends "base.html" %}
+{% load projecttags %}
+{% load humanize %}
+
+{% block title %} Import Builds from eventlogs - Toaster {% endblock %}
+
+{% block pagecontent %}
+
+<div class="container-fluid">
+ <div class="row">
+ <div class="col-md-12">
+ <div class="page-header">
+ <div class="row">
+ <div class="col-md-6">
+ <h1>Import command line builds</h1>
+ </div>
+ {% if import_all %}
+ <div class="col-md-6">
+ <button id="import_all" type="button" class="btn btn-primary navbar-btn navbar-right">
+ <span class="glyphicon glyphicon-upload" style="vertical-align: top;"></span> Import All
+ </button>
+ </div>
+ {% endif %}
+ </div>
+ </div>
+ {% if messages %}
+ <div class="row-fluid" id="empty-state-{{table_name}}">
+ {% for message in messages %}
+ <div class="alert alert-danger">{{message}}</div>
+ {%endfor%}
+ </div>
+ {% endif %}
+ <div class="row">
+ <h4 style="margin-left: 15px;"><strong>Import eventlog file</strong></h4>
+ <form method="POST" enctype="multipart/form-data" action="{% url 'cmdlines' %}">
+ {% csrf_token %}
+ <div class="col-md-6" style="padding-left: 20px;">
+ <div class="row">
+ <input type="hidden" value="{{dir}}" name="dir">
+ <div class="col-md-3"> {{ form.eventlog_file}} </div>
+ </div>
+ <div class="row" style="padding-top: 10px;">
+ <div class="col-md-6">
+ <button id="file_import" type="submit" disabled="disabled" class="btn btn-default navbar-btn" >
+ <span class="glyphicon glyphicon-upload" style="vertical-align: top;"></span> Import
+ </button>
+ </div>
+ </div>
+ </div>
+ </form>
+ </div>
+
+ <div class="row" style="padding-top: 20px;">
+ <div class="col-md-6 ">
+ <h4><strong>Import eventlog files from directory</strong>
+ <a href="#" data-toggle="tooltip" title="{{dir}}">
+ <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-info-circle" viewBox="0 0 16 16" data-toggle="tooltip">
+ <path d="M8 15A7 7 0 1 1 8 1a7 7 0 0 1 0 14m0 1A8 8 0 1 0 8 0a8 8 0 0 0 0 16"/>
+ <path d="m8.93 6.588-2.29.287-.082.38.45.083c.294.07.352.176.288.469l-.738 3.468c-.194.897.105 1.319.808 1.319.545 0 1.178-.252 1.465-.598l.088-.416c-.2.176-.492.246-.686.246-.275 0-.375-.193-.304-.533zM9 4.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0"/>
+ </svg>
+ </a>
+ </h4>
+ {% if files %}
+ <div class="table-responsive">
+ <table class="table col-md-4 table-bordered table-hover">
+ <thead>
+ <tr class="row">
+ <th scope="col">Name</th>
+ <th scope="col">Action</th>
+ </tr>
+ </thead>
+ <tbody>
+ {% for file in files %}
+ <tr class="row" style="height: 48px;">
+ <th scope="row" class="col-md-6" style="vertical-align: middle;">
+ <input type="hidden" value="{{file.name}}" name="{{file.name}}">{{file.name}}
+ </th>
+ <td class="col-md-6 align-middle" style="vertical-align: middle;">
+ {% if file.imported == True and file.build_id is not None %}
+ <a href="{% url 'builddashboard' file.build_id %}">Build Details</a>
+ {%else %}
+ <a onclick="_ajax_update('{{file.name}}', false, '{{dir}}')" data-toggle="tooltip" title="Import File">
+ <span class="glyphicon glyphicon-upload" style="font-size: 18px;"></span>
+ </a>
+ {%endif%}
+ </td>
+ </tr>
+ {% endfor%}
+ </tbody>
+ </table>
+ </div>
+ {% else %}
+ <div class="row-fluid" id="empty-state-{{table_name}}">
+ <div class="alert alert-info">Sorry - no files found</div>
+ </div>
+ {%endif%}
+ </div>
+ </div>
+ </div>
+ </div>
+</div>
+
+<script>
+
+function _ajax_update(file, all, dir){
+ function getCookie(name) {
+ var cookieValue = null;
+ if (document.cookie && document.cookie !== '') {
+ var cookies = document.cookie.split(';');
+ for (var i = 0; i < cookies.length; i++) {
+ var cookie = jQuery.trim(cookies[i]);
+ // Does this cookie string begin with the name we want?
+ if (cookie.substring(0, name.length + 1) === (name + '=')) {
+ cookieValue = decodeURIComponent(cookie.substring(name.length + 1));
+ break;
+ }
+ }
+ }
+ return cookieValue;
+ }
+ var csrftoken = getCookie('csrftoken');
+
+ function csrfSafeMethod(method) {
+ // these HTTP methods do not require CSRF protection
+ return (/^(GET|HEAD|OPTIONS|TRACE)$/.test(method));
+ }
+ $.ajaxSetup({
+ beforeSend: function (xhr, settings) {
+ if (!csrfSafeMethod(settings.type) && !this.crossDomain) {
+ xhr.setRequestHeader("X-CSRFToken", csrftoken);
+ }
+ }
+ });
+
+ $.ajax({
+ url:'/toastergui/cmdline/',
+ type: "POST",
+ data: {file: file, all: all, dir: dir},
+ success:function(data){
+ window.location = '/toastergui/builds/'
+ },
+ complete:function(data){
+ },
+ error:function (xhr, textStatus, thrownError){
+ console.log('fail');
+ }
+ });
+}
+
+$('#import_all').on('click', function(){
+ _ajax_update("{{files | safe}}", true, "{{dir | safe}}");
+});
+
+
+$('#import_page').hide();
+
+$(function () {
+ $('[data-toggle="tooltip"]').tooltip()
+})
+
+
+$("#id_eventlog_file").change(function(){
+ console.log($('#file_import'))
+ $('#file_import').prop("disabled", false);
+ $('#file_import').addClass('btn-primary')
+ $('#file_import').removeClass('btn-default')
+})
+
+</script>
+
+{% endblock %}
@@ -95,6 +95,7 @@ urlpatterns = [
# project URLs
url(r'^newproject/$', views.newproject, name='newproject'),
+ url(r'^cmdline/$', views.CommandLineBuilds.as_view(), name='cmdlines'),
url(r'^projects/$',
tables.ProjectsTable.as_view(template_name="projects-toastertable.html"),
name='all-projects'),
@@ -6,24 +6,36 @@
# SPDX-License-Identifier: GPL-2.0-only
#
+import ast
import re
+import pickle
+import codecs
+import subprocess
+
+import bb.cooker
+from bb.ui import toasterui
from django.db.models import F, Q, Sum
from django.db import IntegrityError
-from django.shortcuts import render, redirect, get_object_or_404
+from django.shortcuts import render, redirect, get_object_or_404, HttpResponseRedirect
from django.utils.http import urlencode
from orm.models import Build, Target, Task, Layer, Layer_Version, Recipe
from orm.models import LogMessage, Variable, Package_Dependency, Package
from orm.models import Task_Dependency, Package_File
from orm.models import Target_Installed_Package, Target_File
from orm.models import TargetKernelFile, TargetSDKFile, Target_Image_File
-from orm.models import BitbakeVersion, CustomImageRecipe
+from orm.models import BitbakeVersion, CustomImageRecipe, EventLogsImports
from django.urls import reverse, resolve
+from django.contrib import messages
+
from django.core.exceptions import ObjectDoesNotExist
+from django.core.files.storage import FileSystemStorage
+from django.core.files.uploadedfile import InMemoryUploadedFile, TemporaryUploadedFile
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.http import HttpResponseNotFound, JsonResponse
from django.utils import timezone
+from django.views.generic import TemplateView
from datetime import timedelta, datetime
from toastergui.templatetags.projecttags import json as jsonfilter
from decimal import Decimal
@@ -32,6 +44,10 @@ import os
from os.path import dirname
import mimetypes
+from toastergui.forms import LoadFileForm
+
+from collections import namedtuple
+
import logging
from toastermain.logs import log_view_mixin
@@ -41,6 +57,7 @@ logger = logging.getLogger("toaster")
# Project creation and managed build enable
project_enable = ('1' == os.environ.get('TOASTER_BUILDSERVER'))
is_project_specific = ('1' == os.environ.get('TOASTER_PROJECTSPECIFIC'))
+import_page = False
class MimeTypeFinder(object):
# setting this to False enables additional non-standard mimetypes
@@ -1940,3 +1957,170 @@ if True:
except (ObjectDoesNotExist, IOError):
return toaster_render(request, "unavailable_artifact.html")
+class EventPlayer:
+ """Emulate a connection to a bitbake server."""
+
+ def __init__(self, eventfile, variables):
+ self.eventfile = eventfile
+ self.variables = variables
+ self.eventmask = []
+
+ def waitEvent(self, _timeout):
+ """Read event from the file."""
+ line = self.eventfile.readline().strip()
+ if not line:
+ return
+ try:
+ event_str = json.loads(line)['vars'].encode('utf-8')
+ event = pickle.loads(codecs.decode(event_str, 'base64'))
+ event_name = "%s.%s" % (event.__module__, event.__class__.__name__)
+ if event_name not in self.eventmask:
+ return
+ return event
+ except ValueError as err:
+ print("Failed loading ", line)
+ raise err
+
+ def runCommand(self, command_line):
+ """Emulate running a command on the server."""
+ name = command_line[0]
+
+ if name == "getVariable":
+ var_name = command_line[1]
+ variable = self.variables.get(var_name)
+ if variable:
+ return variable['v'], None
+ return None, "Missing variable %s" % var_name
+
+ elif name == "getAllKeysWithFlags":
+ dump = {}
+ flaglist = command_line[1]
+ for key, val in self.variables.items():
+ try:
+ if not key.startswith("__"):
+ dump[key] = {
+ 'v': val['v'],
+ 'history' : val['history'],
+ }
+ for flag in flaglist:
+ dump[key][flag] = val[flag]
+ except Exception as err:
+ print(err)
+ return (dump, None)
+
+ elif name == 'setEventMask':
+ self.eventmask = command_line[-1]
+ return True, None
+
+ else:
+ raise Exception("Command %s not implemented" % command_line[0])
+
+ def getEventHandle(self):
+ """
+ This method is called by toasterui.
+ The return value is passed to self.runCommand but not used there.
+ """
+ pass
+
+
+class CommandLineBuilds(TemplateView):
+ model = EventLogsImports
+ template_name = 'command_line_builds.html'
+
+ def get_context_data(self, **kwargs):
+ context = super(CommandLineBuilds, self).get_context_data(**kwargs)
+ #get value from BB_DEFAULT_EVENTLOG defined in bitbake.conf
+ eventlog = subprocess.check_output(['bitbake-getvar', 'BB_DEFAULT_EVENTLOG', '--value'])
+ if eventlog:
+ logs_dir = os.path.dirname(eventlog.decode().strip('\n'))
+ files = os.listdir(logs_dir)
+ imported_files = EventLogsImports.objects.all()
+ files_list = []
+
+ # Filter files that end with ".json"
+ event_files = [file for file in files if file.endswith(".json")]
+
+ #build dict for template using db data
+ for event_file in event_files:
+ if imported_files.filter(name=event_file):
+ files_list.append({
+ 'name': event_file,
+ 'imported': True,
+ 'build_id': imported_files.filter(name=event_file)[0].build_id
+ })
+ else:
+ files_list.append({
+ 'name': event_file,
+ 'imported': False,
+ 'build_id': None
+ })
+ context['import_all'] = True
+
+ context['files'] = files_list
+ context['dir'] = logs_dir
+ else:
+ context['files'] = []
+ context['dir'] = ''
+
+ context['form'] = LoadFileForm()
+ context['project_enable'] = project_enable
+ return context
+
+ def post(self, request, **kwargs):
+ logs_dir = request.POST.get('dir')
+ all_files = request.POST.get('all')
+
+ imported_files = EventLogsImports.objects.all()
+ try:
+ if all_files == 'true':
+ files = ast.literal_eval(request.POST.get('file'))
+ for file in files:
+ if imported_files.filter(name=file.get('name')).exists():
+ imported_files.filter(name=file.get('name'))[0].imported = True
+ else:
+ with open("{}/{}".format(logs_dir, file.get('name'))) as eventfile:
+ # load variables from the first line
+ variables = json.loads(eventfile.readline().strip())['allvariables']
+
+ params = namedtuple('ConfigParams', ['observe_only'])(True)
+ player = EventPlayer(eventfile, variables)
+
+ toasterui.main(player, player, params)
+ event_log_import = EventLogsImports.objects.create(name=file.get('name'), imported=True)
+ event_log_import.build_id = Build.objects.last().id
+ event_log_import.save()
+ else:
+ if self.request.FILES.get('eventlog_file'):
+ file = self.request.FILES['eventlog_file']
+ else:
+ file = request.POST.get('file')
+
+ if imported_files.filter(name=file).exists():
+ imported_files.filter(name=file)[0].imported = True
+ else:
+ if isinstance(file, InMemoryUploadedFile) or isinstance(file, TemporaryUploadedFile):
+ variables = json.loads(file.readline().strip())['allvariables']
+ params = namedtuple('ConfigParams', ['observe_only'])(True)
+ player = EventPlayer(file, variables)
+ if not os.path.exists('{}/{}'.format(logs_dir, file.name)):
+ fs = FileSystemStorage(location=logs_dir)
+ fs.save(file.name, file)
+ toasterui.main(player, player, params)
+ else:
+ with open("{}/{}".format(logs_dir, file)) as eventfile:
+ # load variables from the first line
+ variables = json.loads(eventfile.readline().strip())['allvariables']
+ params = namedtuple('ConfigParams', ['observe_only'])(True)
+ player = EventPlayer(eventfile, variables)
+ toasterui.main(player, player, params)
+ event_log_import = EventLogsImports.objects.create(name=file, imported=True)
+ event_log_import.build_id = Build.objects.last().id
+ event_log_import.save()
+ except json.decoder.JSONDecodeError:
+ messages.add_message(
+ self.request,
+ messages.SUCCESS,
+ "The file content is not in the correct format. Update file content or upload a different file."
+ )
+ return HttpResponseRedirect("/toastergui/cmdline/")
+ return HttpResponseRedirect('/toastergui/builds/')
New feature added to recreate command line builds that are made when toaster is off. The feature contains a new button on the base template to access a new template. A new model was added to register the information on the builds and generate access links A new form was added to include the option to load specific files This feature uses the value from the variable BB_DEFAULT_EVENTLOG to read the files created by bitbake Signed-off-by: Marlon Rodriguez Garcia <marlon.rodriguez-garcia@savoirfairelinux.com> --- lib/bb/ui/toasterui.py | 2 +- .../orm/migrations/0021_eventlogsimports.py | 22 ++ lib/toaster/orm/models.py | 9 + lib/toaster/toastergui/forms.py | 13 ++ lib/toaster/toastergui/templates/base.html | 3 +- .../templates/command_line_builds.html | 171 ++++++++++++++++ lib/toaster/toastergui/urls.py | 1 + lib/toaster/toastergui/views.py | 188 +++++++++++++++++- 8 files changed, 405 insertions(+), 4 deletions(-) create mode 100644 lib/toaster/orm/migrations/0021_eventlogsimports.py create mode 100644 lib/toaster/toastergui/forms.py create mode 100644 lib/toaster/toastergui/templates/command_line_builds.html