Patchwork [bitbake-devel,06/94] bitbake: webhob: make DSI store build information

login
register
mail settings
Submitter Alexandru DAMIAN
Date Sept. 24, 2013, 4:51 p.m.
Message ID <81c2ea58343caeb8e0c7c90edd1a6066ad90f0b7.1380041477.git.alexandru.damian@intel.com>
Download mbox | patch
Permalink /patch/58703/
State New
Headers show

Comments

Alexandru DAMIAN - Sept. 24, 2013, 4:51 p.m.
From: Calin Dragomir <calinx.l.dragomir@intel.com>

This patch will enable DSI to store build information defined in
the Django models. This will be done for each executed build and it
will use a helper file to save this information.

Moved code from DSI interface to the helper file in to classes,
one which wraps around Django model interface operations, and one that
fills in information from the event system into the model data.

Signed-off-by: Calin Dragomir <calinx.l.dragomir@intel.com>
Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
 bitbake/lib/bb/ui/buildinfohelper.py | 127 ++++++++++++++++++++++++
 bitbake/lib/bb/ui/dsi.py             | 180 ++++-------------------------------
 2 files changed, 148 insertions(+), 159 deletions(-)
 create mode 100644 bitbake/lib/bb/ui/buildinfohelper.py

Patch

diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py
new file mode 100644
index 0000000..c77248c
--- /dev/null
+++ b/bitbake/lib/bb/ui/buildinfohelper.py
@@ -0,0 +1,127 @@ 
+import datetime
+import sys
+import uuid
+
+
+class ORMWrapper(object):
+    """ This class creates the dictionaries needed to store information in the database
+        following the format defined by the Django models. It is also used to save this
+        information in the database.
+    """
+
+    def __init__(self):
+        self.uuid = None
+
+    def get_machine_information(self, server):
+        machine_info = {}
+
+        machine_info['name'] = server.runCommand(["getVariable", "MACHINE"])[0]
+        machine_info['description'] = 'Not Available'
+
+        return machine_info
+
+    def create_machine_object(self, machine_information):
+        # This needs to be imported after we have configured the Django settings file
+        from webhob.orm.models import Machine
+
+        machine = Machine.objects.get_or_create(name=machine_information['name'],
+                                                description=machine_information['description'])
+
+        return machine[0]
+
+    def get_build_information(self, server, machine_obj):
+        build_info = {}
+
+        # Generate an identifier for each new build
+        self.uuid = str(uuid.uuid4())
+
+        build_info['uuid'] = self.uuid
+        build_info['target'] = 'Not Available'
+        build_info['machine'] = machine_obj
+        build_info['distro'] = server.runCommand(["getVariable", "DISTRO"])[0]
+        build_info['distro_version'] = server.runCommand(["getVariable", "DISTRO_VERSION"])[0]
+        build_info['started_on'] = datetime.datetime.now()
+        build_info['completed_on'] = datetime.datetime.now()
+        build_info['outcome'] = 0
+        build_info['number_of_errors'] = 0
+        build_info['number_of_warnings'] = 0
+        build_info['cpu_usage'] = 0
+        build_info['disk_io'] = 0
+        build_info['cooker_log_path'] = server.runCommand(["getVariable", "BB_CONSOLELOG"])[0]
+        build_info['build_name'] = server.runCommand(["getVariable", "BUILDNAME"])[0]
+        build_info['bitbake_version'] = server.runCommand(["getVariable", "BB_VERSION"])[0]
+
+        return build_info
+
+    def create_build_object(self, build_info):
+        # This needs to be imported after we have configured the Django settings file
+        from webhob.orm.models import Build
+
+        build = Build.objects.create(uuid=build_info['uuid'],
+                                    target=build_info['target'],
+                                    machine=build_info['machine'],
+                                    distro=build_info['distro'],
+                                    distro_version=build_info['distro_version'],
+                                    started_on=build_info['started_on'],
+                                    completed_on=build_info['completed_on'],
+                                    outcome=build_info['outcome'],
+                                    errors_no=build_info['number_of_errors'],
+                                    warnings_no=build_info['number_of_warnings'],
+                                    cpu_usage=build_info['cpu_usage'],
+                                    disk_io=build_info['disk_io'],
+                                    cooker_log_path=build_info['cooker_log_path'],
+                                    build_name=build_info['build_name'],
+                                    bitbake_version=build_info['bitbake_version'])
+
+        return build
+
+    def update_build_object(self, build_obj, errors, warnings):
+        # This needs to be imported after we have configured the Django settings file
+        from webhob.orm.models import Build
+
+        outcome = 0
+        if errors or warnings:
+            outcome = 1
+
+        build = Build.objects.get(uuid=build_obj.uuid)
+        build.completed_on = datetime.datetime.now()
+        build.errors_no = errors
+        build.warnings_no = warnings
+        build.outcome = outcome
+        build.save()
+
+        return build_obj
+
+
+class BuildInfoHelper(object):
+    """ This class gathers the build information from the server and sends it
+        towards the ORM wrapper for storing in the database
+    """
+
+    def __init__(self):
+        self.configure_django()
+        self.task_order = 0
+        self.tasks_information = {}
+        self.uuid = None
+        self.transport_utils = {}
+        self.orm_wrapper = ORMWrapper()
+
+    def configure_django(self):
+        import webhob.whbmain.settings as whb_django_settings
+        from django.core.management import setup_environ
+        setup_environ(whb_django_settings)
+        # Add webhob to sys path for importing modules
+        sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'webhob'))
+
+    def store_started_build(self, event, server):
+
+        machine_information = self.orm_wrapper.get_machine_information(server)
+        machine_obj = self.orm_wrapper.create_machine_object(machine_information)
+
+        build_information = self.orm_wrapper.get_build_information(server, machine_obj)
+        build_obj = self.orm_wrapper.create_build_object(build_information)
+        self.transport_utils['build'] = build_obj
+
+    def update_build_information(self, event, errors, warnings):
+        build_obj = self.orm_wrapper.update_build_object(self.transport_utils['build'], errors, warnings)
+        del self.transport_utils['build']
diff --git a/bitbake/lib/bb/ui/dsi.py b/bitbake/lib/bb/ui/dsi.py
index f832806..d7b038c 100644
--- a/bitbake/lib/bb/ui/dsi.py
+++ b/bitbake/lib/bb/ui/dsi.py
@@ -17,157 +17,25 @@ 
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 from __future__ import division
+from bb.ui import uihelper
+from bb.ui.buildinfohelper import BuildInfoHelper
 
-import os
-import sys
-import xmlrpclib
+import bb.msg
+import copy
+import fcntl
 import logging
+import os
 import progressbar
 import signal
-import bb.msg
-import time
-import fcntl
 import struct
-import copy
-import datetime
+import sys
+import time
+import xmlrpclib
 
-from bb.ui import uihelper
 
 logger = logging.getLogger("BitBake")
 interactive = sys.stdout.isatty()
 
-class WebHOBHelper(object):
-    
-    def __init__(self):
-        self.configure_django()
-        self.task_order = 0
-        self.tasks_information = {}
-
-    def configure_django(self):
-        import webhob.whbmain.settings as whb_django_settings
-        from django.core.management import setup_environ
-        setup_environ(whb_django_settings)
-        # Add webhob to sys path for importing modules
-        sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'webhob'))
-    
-    def store_build(self, uuid):
-        # This needs to be imported after we have configured the Django settings file
-        from webhob.orm.models import Builds, Machines, Layers
-        machine_obj = Machines.objects.create(
-                                name='Test Machine',
-                                description='This is our great test machine'
-                                )
-        machine_obj.save()
-
-        layer_obj = Layers.objects.create(
-                                name='Test Layer',
-                                branch='master_test',
-                                commit='2148u323nsau83smdmbnca3312f',
-                                priority=2,
-                                link_to_oe_core='http://test.oecore.com'
-                                )
-        layer_obj.save()
-
-        build_obj = Builds.objects.create(
-                                uuid=uuid,
-                                target='test target',
-                                host_machine='Host machine',
-                                distro='1.234',
-                                distro_version='v2',
-                                host_system='Test hosting system',
-                                host_distribution='Test host distrib',
-                                target_system=machine_obj,
-                                started_on=datetime.datetime.now(),
-                                completed_on=datetime.datetime.now(),
-                                outcome=3,
-                                errors_no=0,
-                                warnings_no=0,
-                                cpu_usage=54,
-                                disk_io=23.3232,
-                                cooker_log_path='/home/test/cooker_test',
-                                build_name='tester_build',
-                                bitbake_version='1.0.5')
-        build_obj.layers.add(layer_obj)
-        build_obj.save()
-
-        return build_obj, layer_obj
-
-    def store_started_task(self, event, build_object):
-        self.task_order += 1
-        identifier = event._package + event._task
-        self.tasks_information[identifier] = {
-                'uuid': build_object,
-                'task_executed': True,
-                'order': self.task_order,
-                'recipe': event._package,
-                'task_name': event._task,
-                'start_time': time.time(),
-                }
-
-    def write_stored_tasks(self, event, build_object, layer_object):
-        identifier = event._package + event._task
-
-        # fix this
-        if event.getDisplayName().lower() == 'succeded':
-            outcome = 0
-        else:
-            outcome = 1
-
-        self.tasks_information[identifier].update({
-                'outcome': outcome,
-                'end_time': time.time(),
-                })
-
-        task = self.tasks_information[identifier]
-
-        # This needs to be imported after we have configured the Django settings file
-        from webhob.orm.models import Tasks, Recipes
-
-        recipe_obj = Recipes.objects.create(
-                                    name=task['recipe'],
-                                    version='1.0',
-                                    layer=layer_object,
-                                    summary='Short summary',
-                                    description='Longer description',
-                                    section='24',
-                                    license='license title',
-                                    licensing_info='longer license information',
-                                    homepage='http://homepage.com',
-                                    bugtracker='2456',
-                                    author='Mr.Test',
-                                    file_path='home/test/testproject/',
-                                    )
-
-        # Store finished task
-        task_obj = Tasks.objects.create(uuid=task['uuid'],
-                                        order=task['order'],
-                                        outcome=task['outcome'],
-                                        recipe=recipe_obj,
-                                        task_name=task['task_name'],
-                                        elapsed_time=task['end_time'] - task['start_time'],
-                                        # The next lines are just for testing reasons
-                                        sstate_checksum = '2134123sdscsdf232342tgfdv',
-                                        task_executed=True,
-                                        path_to_sstate_obj='/home/calin',
-                                        source_url='/',
-                                        log_file='/',
-                                        work_directory='/',
-                                        script_type=0,
-                                        file_path='/',
-                                        line_number=55,
-                                        py_stack_trace='Testing traceback',
-                                        disk_io = 23.55633,
-                                        cpu_usage=43,
-                                        errors_no=0,
-                                        warnings_no=0,
-                                        error='None',
-                                        warning='None',
-                                        sstate_result=0,
-                                        diffsigs="Testing diffsigs")
-        task_obj.save()
-        # Clean memory
-        del self.tasks_information[identifier]
-
 
 class BBProgress(progressbar.ProgressBar):
     def __init__(self, msg, maxval):
@@ -350,12 +218,7 @@  class TerminalFilter(object):
             self.termios.tcsetattr(fd, self.termios.TCSADRAIN, self.stdinbackup)
 
 def main(server, eventHandler, params, tf = TerminalFilter):
-    print "This is running in DSI mode"
-
-    # Generate an unique ID for this build
-    # TODO: for multiple build commands this might not work as expected
-    import uuid
-    uuid = sessionid = str(uuid.uuid4())
+    print "DSI - Data Store Interface"
 
     # Get values of variables which control our output
     includelogs, error = server.runCommand(["getVariable", "BBINCLUDELOGS"])
@@ -422,8 +285,7 @@  def main(server, eventHandler, params, tf = TerminalFilter):
     errors = 0
     warnings = 0
     taskfailures = []
-    wbhbhelper = WebHOBHelper()
-    build_object, layer_object = wbhbhelper.store_build(uuid)
+    buildinfohelper = BuildInfoHelper()
 
     termfilter = tf(main, helper, console, format)
 
@@ -443,11 +305,8 @@  def main(server, eventHandler, params, tf = TerminalFilter):
                 if not main.shutdown:
                     main.shutdown = 1
 
-            if isinstance(event, bb.build.TaskStarted):
-                wbhbhelper.store_started_task(event, build_object)
-
-            if isinstance(event, (bb.build.TaskSucceeded, bb.build.TaskFailedSilent, bb.build.TaskFailed)):
-                wbhbhelper.write_stored_tasks(event, build_object, layer_object)
+            if isinstance(event, bb.event.BuildStarted):
+                buildinfohelper.store_started_build(event, server)
 
             if isinstance(event, bb.event.LogExecTTY):
                 if log_exec_tty:
@@ -470,7 +329,7 @@  def main(server, eventHandler, params, tf = TerminalFilter):
                 elif event.levelno == format.WARNING:
                     warnings = warnings + 1
                 # For "normal" logging conditions, don't show note logs from tasks
-                # but do show them if the user has changed the default log level to 
+                # but do show them if the user has changed the default log level to
                 # include verbose/debug messages
                 if event.taskpid != 0 and event.levelno <= format.NOTE:
                     continue
@@ -577,20 +436,24 @@  def main(server, eventHandler, params, tf = TerminalFilter):
                 logger.error("Task %s (%s) failed with exit code '%s'",
                              event.taskid, event.taskstring, event.exitcode)
                 continue
- 
+
             if isinstance(event, bb.runqueue.sceneQueueTaskFailed):
                 logger.warn("Setscene task %s (%s) failed with exit code '%s' - real task will be run instead",
                              event.taskid, event.taskstring, event.exitcode)
                 continue
 
+            if isinstance(event, (bb.command.CommandFailed, bb.command.CommandExit, bb.command.CommandCompleted, bb.cooker.CookerExit)):
+                buildinfohelper.update_build_information(event, errors, warnings)
+                main.shutdown = 2
+
             if isinstance(event, bb.event.ConfigParsed):
                 # timestamp should be added for this
                 continue
-            
+
             if isinstance(event, bb.event.RecipeParsed):
                 # timestamp should be added for this
                 continue
-                
+
             if isinstance(event, bb.event.OperationStarted):
                 # timestamp should be added for this
                 continue
@@ -664,4 +527,3 @@  def main(server, eventHandler, params, tf = TerminalFilter):
     termfilter.finish()
 
     return return_value
-