[1/2] toaster: layerindex updater Take into account layers being predefined

Submitted by Michael Wood on Nov. 4, 2016, 4:42 p.m. | Patch ID: 133604

Details

Message ID 1478277739-15869-1-git-send-email-michael.g.wood@intel.com
State New
Headers show

Commit Message

Michael Wood Nov. 4, 2016, 4:42 p.m.
As we can now provide layer definitions through fixtures we need to be
more clever how we update the metadata in the database to avoid
duplicate metadata being created. To do this we make more effort to
match existing data in the database and update only the fields which
will be better provided by the layer index.

This removes the need for us to special case layers which are provided
as part of poky such as openembedded-core or meta-poky which exist on
the layerindex but with different git urls.

Signed-off-by: Michael Wood <michael.g.wood@intel.com>
---
 .../toaster/orm/management/commands/lsupdates.py   | 60 +++++++---------------
 1 file changed, 19 insertions(+), 41 deletions(-)

Patch hide | download patch | download mbox

diff --git a/bitbake/lib/toaster/orm/management/commands/lsupdates.py b/bitbake/lib/toaster/orm/management/commands/lsupdates.py
index 8ff120e..688918e 100644
--- a/bitbake/lib/toaster/orm/management/commands/lsupdates.py
+++ b/bitbake/lib/toaster/orm/management/commands/lsupdates.py
@@ -154,41 +154,17 @@  class Command(NoArgsCommand):
 
         total = len(layers_info)
         for i, li in enumerate(layers_info):
-            # Special case for the openembedded-core layer
-            if li['name'] == oe_core_layer:
-                try:
-                    # If we have an existing openembedded-core for example
-                    # from the toasterconf.json augment the info using the
-                    # layerindex rather than duplicate it
-                    oe_core_l = Layer.objects.get(name=oe_core_layer)
-                    # Take ownership of the layer as now coming from the
-                    # layerindex
-                    oe_core_l.summary = li['summary']
-                    oe_core_l.description = li['description']
-                    oe_core_l.vcs_web_url = li['vcs_web_url']
-                    oe_core_l.vcs_web_tree_base_url = \
-                        li['vcs_web_tree_base_url']
-                    oe_core_l.vcs_web_file_base_url = \
-                        li['vcs_web_file_base_url']
-
-                    oe_core_l.save()
-                    li_layer_id_to_toaster_layer_id[li['id']] = oe_core_l.pk
-                    self.mini_progress("layers", i, total)
-                    continue
-
-                except Layer.DoesNotExist:
-                    pass
-
             try:
-                l, created = Layer.objects.get_or_create(name=li['name'],
-                                                         vcs_url=li['vcs_url'])
+                l, created = Layer.objects.get_or_create(name=li['name'])
                 l.up_date = li['updated']
-                l.vcs_url = li['vcs_url']
-                l.vcs_web_url = li['vcs_web_url']
-                l.vcs_web_tree_base_url = li['vcs_web_tree_base_url']
-                l.vcs_web_file_base_url = li['vcs_web_file_base_url']
                 l.summary = li['summary']
                 l.description = li['description']
+
+                if created:
+                    l.vcs_url = li['vcs_url']
+                    l.vcs_web_url = li['vcs_web_url']
+                    l.vcs_web_tree_base_url = li['vcs_web_tree_base_url']
+                    l.vcs_web_file_base_url = li['vcs_web_file_base_url']
                 l.save()
             except Layer.MultipleObjectsReturned:
                 logger.info("Skipped %s as we found multiple layers and "
@@ -211,12 +187,14 @@  class Command(NoArgsCommand):
 
         total = len(layerbranches_info)
         for i, lbi in enumerate(layerbranches_info):
+            # release as defined by toaster map to layerindex branch
+            release = li_branch_id_to_toaster_release[lbi['branch']]
 
             try:
                 lv, created = Layer_Version.objects.get_or_create(
-                    layer_source=LayerSource.TYPE_LAYERINDEX,
                     layer=Layer.objects.get(
-                        pk=li_layer_id_to_toaster_layer_id[lbi['layer']])
+                        pk=li_layer_id_to_toaster_layer_id[lbi['layer']]),
+                    release=release
                 )
             except KeyError:
                 logger.warning(
@@ -224,11 +202,12 @@  class Command(NoArgsCommand):
                     lbi['layer'])
                 continue
 
-            lv.release = li_branch_id_to_toaster_release[lbi['branch']]
-            lv.up_date = lbi['updated']
-            lv.commit = lbi['actual_branch']
-            lv.dirpath = lbi['vcs_subdir']
-            lv.save()
+            if created:
+                lv.release = li_branch_id_to_toaster_release[lbi['branch']]
+                lv.up_date = lbi['updated']
+                lv.commit = lbi['actual_branch']
+                lv.dirpath = lbi['vcs_subdir']
+                lv.save()
 
             li_layer_branch_id_to_toaster_lv_id[lbi['id']] =\
                 lv.pk
@@ -255,9 +234,8 @@  class Command(NoArgsCommand):
                 layer_id = li_layer_id_to_toaster_layer_id[ldi['dependency']]
 
                 dependlist[lv].append(
-                    Layer_Version.objects.get(
-                        layer_source=LayerSource.TYPE_LAYERINDEX,
-                        layer__pk=layer_id))
+                    Layer_Version.objects.get(layer__pk=layer_id,
+                                              release=lv.release))
 
             except Layer_Version.DoesNotExist:
                 logger.warning("Cannot find layer version (ls:%s),"

Comments

brian avery Nov. 5, 2016, 3:37 a.m.
Hi,
2 things:
1) Could we change the     <*field* type="CharField" name="name"
>DEFAULT_RELEASE</*field*>
to master in bitbake/lib/toaster/orm/fixtures/settings.xml

right now, we are defaulting to morty for new projects which is a change in
behaviour.

2) The browser tests are failing on :

It's not related to this patch and I filed a separate bug for it

https://bugzilla.yoctoproject.org/show_bug.cgi?id=10609


If we can do 1, I can upstream the patches. Thanks for fixing the layers!

-brian

an intel employee





On Fri, Nov 4, 2016 at 9:42 AM, Michael Wood <michael.g.wood@intel.com>
wrote:

> As we can now provide layer definitions through fixtures we need to be
> more clever how we update the metadata in the database to avoid
> duplicate metadata being created. To do this we make more effort to
> match existing data in the database and update only the fields which
> will be better provided by the layer index.
>
> This removes the need for us to special case layers which are provided
> as part of poky such as openembedded-core or meta-poky which exist on
> the layerindex but with different git urls.
>
> Signed-off-by: Michael Wood <michael.g.wood@intel.com>
> ---
>  .../toaster/orm/management/commands/lsupdates.py   | 60
> +++++++---------------
>  1 file changed, 19 insertions(+), 41 deletions(-)
>
> diff --git a/bitbake/lib/toaster/orm/management/commands/lsupdates.py
> b/bitbake/lib/toaster/orm/management/commands/lsupdates.py
> index 8ff120e..688918e 100644
> --- a/bitbake/lib/toaster/orm/management/commands/lsupdates.py
> +++ b/bitbake/lib/toaster/orm/management/commands/lsupdates.py
> @@ -154,41 +154,17 @@ class Command(NoArgsCommand):
>
>          total = len(layers_info)
>          for i, li in enumerate(layers_info):
> -            # Special case for the openembedded-core layer
> -            if li['name'] == oe_core_layer:
> -                try:
> -                    # If we have an existing openembedded-core for example
> -                    # from the toasterconf.json augment the info using the
> -                    # layerindex rather than duplicate it
> -                    oe_core_l = Layer.objects.get(name=oe_core_layer)
> -                    # Take ownership of the layer as now coming from the
> -                    # layerindex
> -                    oe_core_l.summary = li['summary']
> -                    oe_core_l.description = li['description']
> -                    oe_core_l.vcs_web_url = li['vcs_web_url']
> -                    oe_core_l.vcs_web_tree_base_url = \
> -                        li['vcs_web_tree_base_url']
> -                    oe_core_l.vcs_web_file_base_url = \
> -                        li['vcs_web_file_base_url']
> -
> -                    oe_core_l.save()
> -                    li_layer_id_to_toaster_layer_id[li['id']] =
> oe_core_l.pk
> -                    self.mini_progress("layers", i, total)
> -                    continue
> -
> -                except Layer.DoesNotExist:
> -                    pass
> -
>              try:
> -                l, created = Layer.objects.get_or_create(name=li['name'],
> -
>  vcs_url=li['vcs_url'])
> +                l, created = Layer.objects.get_or_create(name=li['name'])
>                  l.up_date = li['updated']
> -                l.vcs_url = li['vcs_url']
> -                l.vcs_web_url = li['vcs_web_url']
> -                l.vcs_web_tree_base_url = li['vcs_web_tree_base_url']
> -                l.vcs_web_file_base_url = li['vcs_web_file_base_url']
>                  l.summary = li['summary']
>                  l.description = li['description']
> +
> +                if created:
> +                    l.vcs_url = li['vcs_url']
> +                    l.vcs_web_url = li['vcs_web_url']
> +                    l.vcs_web_tree_base_url = li['vcs_web_tree_base_url']
> +                    l.vcs_web_file_base_url = li['vcs_web_file_base_url']
>                  l.save()
>              except Layer.MultipleObjectsReturned:
>                  logger.info("Skipped %s as we found multiple layers and "
> @@ -211,12 +187,14 @@ class Command(NoArgsCommand):
>
>          total = len(layerbranches_info)
>          for i, lbi in enumerate(layerbranches_info):
> +            # release as defined by toaster map to layerindex branch
> +            release = li_branch_id_to_toaster_release[lbi['branch']]
>
>              try:
>                  lv, created = Layer_Version.objects.get_or_create(
> -                    layer_source=LayerSource.TYPE_LAYERINDEX,
>                      layer=Layer.objects.get(
> -                        pk=li_layer_id_to_toaster_layer_id[lbi['layer']])
> +                        pk=li_layer_id_to_toaster_laye
> r_id[lbi['layer']]),
> +                    release=release
>                  )
>              except KeyError:
>                  logger.warning(
> @@ -224,11 +202,12 @@ class Command(NoArgsCommand):
>                      lbi['layer'])
>                  continue
>
> -            lv.release = li_branch_id_to_toaster_release[lbi['branch']]
> -            lv.up_date = lbi['updated']
> -            lv.commit = lbi['actual_branch']
> -            lv.dirpath = lbi['vcs_subdir']
> -            lv.save()
> +            if created:
> +                lv.release = li_branch_id_to_toaster_releas
> e[lbi['branch']]
> +                lv.up_date = lbi['updated']
> +                lv.commit = lbi['actual_branch']
> +                lv.dirpath = lbi['vcs_subdir']
> +                lv.save()
>
>              li_layer_branch_id_to_toaster_lv_id[lbi['id']] =\
>                  lv.pk
> @@ -255,9 +234,8 @@ class Command(NoArgsCommand):
>                  layer_id = li_layer_id_to_toaster_layer_i
> d[ldi['dependency']]
>
>                  dependlist[lv].append(
> -                    Layer_Version.objects.get(
> -                        layer_source=LayerSource.TYPE_LAYERINDEX,
> -                        layer__pk=layer_id))
> +                    Layer_Version.objects.get(layer__pk=layer_id,
> +                                              release=lv.release))
>
>              except Layer_Version.DoesNotExist:
>                  logger.warning("Cannot find layer version (ls:%s),"
> --
> 2.7.4
>
> --
> _______________________________________________
> toaster mailing list
> toaster@yoctoproject.org
> https://lists.yoctoproject.org/listinfo/toaster
>