Patchwork [bitbake-devel] siggen: add quotes around variable values to see whitespace

login
register
mail settings
Submitter Martin Jansa
Date March 2, 2013, 4:50 p.m.
Message ID <1362243005-12124-1-git-send-email-Martin.Jansa@gmail.com>
Download mbox | patch
Permalink /patch/45433/
State Accepted
Commit 7d85b1a383feaa072503d71d3baeb3bdcd1012a8
Headers show

Comments

Martin Jansa - March 2, 2013, 4:50 p.m.
* without this change it's hard to see what's changed:
  bitbake-diffsigs sstate-cache/*/sstate-depmodwrapper-cross-cortexa8-vfp-neon-oe-linux-gnueabi-1.0-r0-cortexa8-vfp-neon-*_populate-sysroot.tgz.siginfo
  basehash changed from 62f28c13172d38af917716d6d4b15ac5 to 9cf38842bcc79e0a4356856a64141548
  Variable TUNE_CCARGS value changed from  -march=armv7-a    -mthumb -mthumb-interwork -mfloat-abi=softfp -mfpu=neon -mtune=cortex-a8  to  -march=armv7-a    -mthumb -mthumb-interwork -mfloat-abi=softfp -mfpu=neon -mtune=cortex-a8
  Hash for dependent task depmodwrapper-cross_1.0.bb.do_install changed from 4494cb6c117309a9b4c541ad86c0aac5 to 1e3bcf2d14e5bd30a38350b41bb98db3

* with this change, we can see extra trailing space in first version:
  basehash changed from 62f28c13172d38af917716d6d4b15ac5 to 9cf38842bcc79e0a4356856a64141548
  Variable TUNE_CCARGS value changed from ' -march=armv7-a    -mthumb -mthumb-interwork -mfloat-abi=softfp -mfpu=neon -mtune=cortex-a8 ' to ' -march=armv7-a    -mthumb -mthumb-interwork -mfloat-abi=softfp -mfpu=neon -mtune=cortex-a8'
  Hash for dependent task depmodwrapper-cross_1.0.bb.do_install changed from 4494cb6c117309a9b4c541ad86c0aac5 to 1e3bcf2d14e5bd30a38350b41bb98db3

Signed-off-by: Martin Jansa <Martin.Jansa@gmail.com>
---
 lib/bb/siggen.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)
Mark Hatle - March 4, 2013, 3:49 p.m.
On 3/2/13 10:50 AM, Martin Jansa wrote:
> * without this change it's hard to see what's changed:
>    bitbake-diffsigs sstate-cache/*/sstate-depmodwrapper-cross-cortexa8-vfp-neon-oe-linux-gnueabi-1.0-r0-cortexa8-vfp-neon-*_populate-sysroot.tgz.siginfo
>    basehash changed from 62f28c13172d38af917716d6d4b15ac5 to 9cf38842bcc79e0a4356856a64141548
>    Variable TUNE_CCARGS value changed from  -march=armv7-a    -mthumb -mthumb-interwork -mfloat-abi=softfp -mfpu=neon -mtune=cortex-a8  to  -march=armv7-a    -mthumb -mthumb-interwork -mfloat-abi=softfp -mfpu=neon -mtune=cortex-a8
>    Hash for dependent task depmodwrapper-cross_1.0.bb.do_install changed from 4494cb6c117309a9b4c541ad86c0aac5 to 1e3bcf2d14e5bd30a38350b41bb98db3
>
> * with this change, we can see extra trailing space in first version:
>    basehash changed from 62f28c13172d38af917716d6d4b15ac5 to 9cf38842bcc79e0a4356856a64141548
>    Variable TUNE_CCARGS value changed from ' -march=armv7-a    -mthumb -mthumb-interwork -mfloat-abi=softfp -mfpu=neon -mtune=cortex-a8 ' to ' -march=armv7-a    -mthumb -mthumb-interwork -mfloat-abi=softfp -mfpu=neon -mtune=cortex-a8'
>    Hash for dependent task depmodwrapper-cross_1.0.bb.do_install changed from 4494cb6c117309a9b4c541ad86c0aac5 to 1e3bcf2d14e5bd30a38350b41bb98db3

No objection to this patch, but it does raise the question.  Should we somehow 
strip the leading and training white space, maybe even multiple spaces (like 
above) and then use the results to generate the checksum?

If I saw the above, my first comment would be "well that's stupid, the output is 
the same"..  (Of course performance might be impacted by all of that stripping..)

--Mark

> Signed-off-by: Martin Jansa <Martin.Jansa@gmail.com>
> ---
>   lib/bb/siggen.py | 8 ++++----
>   1 file changed, 4 insertions(+), 4 deletions(-)
>
> diff --git a/lib/bb/siggen.py b/lib/bb/siggen.py
> index 50e4558..8861337 100644
> --- a/lib/bb/siggen.py
> +++ b/lib/bb/siggen.py
> @@ -331,12 +331,12 @@ def compare_sigfiles(a, b, recursecb = None):
>           return changed, added, removed
>
>       if 'basewhitelist' in a_data and a_data['basewhitelist'] != b_data['basewhitelist']:
> -        output.append("basewhitelist changed from %s to %s" % (a_data['basewhitelist'], b_data['basewhitelist']))
> +        output.append("basewhitelist changed from '%s' to '%s'" % (a_data['basewhitelist'], b_data['basewhitelist']))
>           if a_data['basewhitelist'] and b_data['basewhitelist']:
>               output.append("changed items: %s" % a_data['basewhitelist'].symmetric_difference(b_data['basewhitelist']))
>
>       if 'taskwhitelist' in a_data and a_data['taskwhitelist'] != b_data['taskwhitelist']:
> -        output.append("taskwhitelist changed from %s to %s" % (a_data['taskwhitelist'], b_data['taskwhitelist']))
> +        output.append("taskwhitelist changed from '%s' to '%s'" % (a_data['taskwhitelist'], b_data['taskwhitelist']))
>           if a_data['taskwhitelist'] and b_data['taskwhitelist']:
>               output.append("changed items: %s" % a_data['taskwhitelist'].symmetric_difference(b_data['taskwhitelist']))
>
> @@ -349,7 +349,7 @@ def compare_sigfiles(a, b, recursecb = None):
>       changed, added, removed = dict_diff(a_data['gendeps'], b_data['gendeps'], a_data['basewhitelist'] & b_data['basewhitelist'])
>       if changed:
>           for dep in changed:
> -            output.append("List of dependencies for variable %s changed from %s to %s" % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep]))
> +            output.append("List of dependencies for variable %s changed from '%s' to '%s'" % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep]))
>               if a_data['gendeps'][dep] and b_data['gendeps'][dep]:
>                   output.append("changed items: %s" % a_data['gendeps'][dep].symmetric_difference(b_data['gendeps'][dep]))
>       if added:
> @@ -363,7 +363,7 @@ def compare_sigfiles(a, b, recursecb = None):
>       changed, added, removed = dict_diff(a_data['varvals'], b_data['varvals'])
>       if changed:
>           for dep in changed:
> -            output.append("Variable %s value changed from %s to %s" % (dep, a_data['varvals'][dep], b_data['varvals'][dep]))
> +            output.append("Variable %s value changed from '%s' to '%s'" % (dep, a_data['varvals'][dep], b_data['varvals'][dep]))
>
>       changed, added, removed = dict_diff(a_data['file_checksum_values'], b_data['file_checksum_values'])
>       if changed:
>
Martin Jansa - March 4, 2013, 6:24 p.m.
On Mon, Mar 04, 2013 at 09:49:06AM -0600, Mark Hatle wrote:
> On 3/2/13 10:50 AM, Martin Jansa wrote:
> > * without this change it's hard to see what's changed:
> >    bitbake-diffsigs sstate-cache/*/sstate-depmodwrapper-cross-cortexa8-vfp-neon-oe-linux-gnueabi-1.0-r0-cortexa8-vfp-neon-*_populate-sysroot.tgz.siginfo
> >    basehash changed from 62f28c13172d38af917716d6d4b15ac5 to 9cf38842bcc79e0a4356856a64141548
> >    Variable TUNE_CCARGS value changed from  -march=armv7-a    -mthumb -mthumb-interwork -mfloat-abi=softfp -mfpu=neon -mtune=cortex-a8  to  -march=armv7-a    -mthumb -mthumb-interwork -mfloat-abi=softfp -mfpu=neon -mtune=cortex-a8
> >    Hash for dependent task depmodwrapper-cross_1.0.bb.do_install changed from 4494cb6c117309a9b4c541ad86c0aac5 to 1e3bcf2d14e5bd30a38350b41bb98db3
> >
> > * with this change, we can see extra trailing space in first version:
> >    basehash changed from 62f28c13172d38af917716d6d4b15ac5 to 9cf38842bcc79e0a4356856a64141548
> >    Variable TUNE_CCARGS value changed from ' -march=armv7-a    -mthumb -mthumb-interwork -mfloat-abi=softfp -mfpu=neon -mtune=cortex-a8 ' to ' -march=armv7-a    -mthumb -mthumb-interwork -mfloat-abi=softfp -mfpu=neon -mtune=cortex-a8'
> >    Hash for dependent task depmodwrapper-cross_1.0.bb.do_install changed from 4494cb6c117309a9b4c541ad86c0aac5 to 1e3bcf2d14e5bd30a38350b41bb98db3
> 
> No objection to this patch, but it does raise the question.  Should we somehow 
> strip the leading and training white space, maybe even multiple spaces (like 
> above) and then use the results to generate the checksum?
> 
> If I saw the above, my first comment would be "well that's stupid, the output is 
> the same"..  (Of course performance might be impacted by all of that stripping..)

I had the same comment when I've seen it, that's why I added those
quotes to be able to spot that space at all.

I'm not sure what caused that extra space and so many rebuilds, but I've
noticed it today in completely different build where space in multiline
variable were squashed to one space separator and I think the only
difference were pool patches in bitbake.

> > Signed-off-by: Martin Jansa <Martin.Jansa@gmail.com>
> > ---
> >   lib/bb/siggen.py | 8 ++++----
> >   1 file changed, 4 insertions(+), 4 deletions(-)
> >
> > diff --git a/lib/bb/siggen.py b/lib/bb/siggen.py
> > index 50e4558..8861337 100644
> > --- a/lib/bb/siggen.py
> > +++ b/lib/bb/siggen.py
> > @@ -331,12 +331,12 @@ def compare_sigfiles(a, b, recursecb = None):
> >           return changed, added, removed
> >
> >       if 'basewhitelist' in a_data and a_data['basewhitelist'] != b_data['basewhitelist']:
> > -        output.append("basewhitelist changed from %s to %s" % (a_data['basewhitelist'], b_data['basewhitelist']))
> > +        output.append("basewhitelist changed from '%s' to '%s'" % (a_data['basewhitelist'], b_data['basewhitelist']))
> >           if a_data['basewhitelist'] and b_data['basewhitelist']:
> >               output.append("changed items: %s" % a_data['basewhitelist'].symmetric_difference(b_data['basewhitelist']))
> >
> >       if 'taskwhitelist' in a_data and a_data['taskwhitelist'] != b_data['taskwhitelist']:
> > -        output.append("taskwhitelist changed from %s to %s" % (a_data['taskwhitelist'], b_data['taskwhitelist']))
> > +        output.append("taskwhitelist changed from '%s' to '%s'" % (a_data['taskwhitelist'], b_data['taskwhitelist']))
> >           if a_data['taskwhitelist'] and b_data['taskwhitelist']:
> >               output.append("changed items: %s" % a_data['taskwhitelist'].symmetric_difference(b_data['taskwhitelist']))
> >
> > @@ -349,7 +349,7 @@ def compare_sigfiles(a, b, recursecb = None):
> >       changed, added, removed = dict_diff(a_data['gendeps'], b_data['gendeps'], a_data['basewhitelist'] & b_data['basewhitelist'])
> >       if changed:
> >           for dep in changed:
> > -            output.append("List of dependencies for variable %s changed from %s to %s" % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep]))
> > +            output.append("List of dependencies for variable %s changed from '%s' to '%s'" % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep]))
> >               if a_data['gendeps'][dep] and b_data['gendeps'][dep]:
> >                   output.append("changed items: %s" % a_data['gendeps'][dep].symmetric_difference(b_data['gendeps'][dep]))
> >       if added:
> > @@ -363,7 +363,7 @@ def compare_sigfiles(a, b, recursecb = None):
> >       changed, added, removed = dict_diff(a_data['varvals'], b_data['varvals'])
> >       if changed:
> >           for dep in changed:
> > -            output.append("Variable %s value changed from %s to %s" % (dep, a_data['varvals'][dep], b_data['varvals'][dep]))
> > +            output.append("Variable %s value changed from '%s' to '%s'" % (dep, a_data['varvals'][dep], b_data['varvals'][dep]))
> >
> >       changed, added, removed = dict_diff(a_data['file_checksum_values'], b_data['file_checksum_values'])
> >       if changed:
> >
> 
> 
> _______________________________________________
> bitbake-devel mailing list
> bitbake-devel@lists.openembedded.org
> http://lists.linuxtogo.org/cgi-bin/mailman/listinfo/bitbake-devel
Richard Purdie - March 7, 2013, 10:48 a.m.
On Mon, 2013-03-04 at 09:49 -0600, Mark Hatle wrote:
> On 3/2/13 10:50 AM, Martin Jansa wrote:
> > * without this change it's hard to see what's changed:
> >    bitbake-diffsigs sstate-cache/*/sstate-depmodwrapper-cross-cortexa8-vfp-neon-oe-linux-gnueabi-1.0-r0-cortexa8-vfp-neon-*_populate-sysroot.tgz.siginfo
> >    basehash changed from 62f28c13172d38af917716d6d4b15ac5 to 9cf38842bcc79e0a4356856a64141548
> >    Variable TUNE_CCARGS value changed from  -march=armv7-a    -mthumb -mthumb-interwork -mfloat-abi=softfp -mfpu=neon -mtune=cortex-a8  to  -march=armv7-a    -mthumb -mthumb-interwork -mfloat-abi=softfp -mfpu=neon -mtune=cortex-a8
> >    Hash for dependent task depmodwrapper-cross_1.0.bb.do_install changed from 4494cb6c117309a9b4c541ad86c0aac5 to 1e3bcf2d14e5bd30a38350b41bb98db3
> >
> > * with this change, we can see extra trailing space in first version:
> >    basehash changed from 62f28c13172d38af917716d6d4b15ac5 to 9cf38842bcc79e0a4356856a64141548
> >    Variable TUNE_CCARGS value changed from ' -march=armv7-a    -mthumb -mthumb-interwork -mfloat-abi=softfp -mfpu=neon -mtune=cortex-a8 ' to ' -march=armv7-a    -mthumb -mthumb-interwork -mfloat-abi=softfp -mfpu=neon -mtune=cortex-a8'
> >    Hash for dependent task depmodwrapper-cross_1.0.bb.do_install changed from 4494cb6c117309a9b4c541ad86c0aac5 to 1e3bcf2d14e5bd30a38350b41bb98db3
> 
> No objection to this patch, but it does raise the question.  Should we somehow 
> strip the leading and training white space, maybe even multiple spaces (like 
> above) and then use the results to generate the checksum?
>
> If I saw the above, my first comment would be "well that's stupid, the output is 
> the same"..  (Of course performance might be impacted by all of that stripping..)

Bitbake has no idea whether the whitespace means anything in any given
context when its constructing the checksums. Deciding when to strip and
what to strip would be the kind of thing nightmares are made of and
we're not going there. It was a conscious choice it behaves like it does
now and I still believe its the right one.

Cheers,

Richard
Richard Purdie - March 7, 2013, 10:49 a.m.
On Mon, 2013-03-04 at 19:24 +0100, Martin Jansa wrote:
> On Mon, Mar 04, 2013 at 09:49:06AM -0600, Mark Hatle wrote:
> > On 3/2/13 10:50 AM, Martin Jansa wrote:
> > > * without this change it's hard to see what's changed:
> > >    bitbake-diffsigs sstate-cache/*/sstate-depmodwrapper-cross-cortexa8-vfp-neon-oe-linux-gnueabi-1.0-r0-cortexa8-vfp-neon-*_populate-sysroot.tgz.siginfo
> > >    basehash changed from 62f28c13172d38af917716d6d4b15ac5 to 9cf38842bcc79e0a4356856a64141548
> > >    Variable TUNE_CCARGS value changed from  -march=armv7-a    -mthumb -mthumb-interwork -mfloat-abi=softfp -mfpu=neon -mtune=cortex-a8  to  -march=armv7-a    -mthumb -mthumb-interwork -mfloat-abi=softfp -mfpu=neon -mtune=cortex-a8
> > >    Hash for dependent task depmodwrapper-cross_1.0.bb.do_install changed from 4494cb6c117309a9b4c541ad86c0aac5 to 1e3bcf2d14e5bd30a38350b41bb98db3
> > >
> > > * with this change, we can see extra trailing space in first version:
> > >    basehash changed from 62f28c13172d38af917716d6d4b15ac5 to 9cf38842bcc79e0a4356856a64141548
> > >    Variable TUNE_CCARGS value changed from ' -march=armv7-a    -mthumb -mthumb-interwork -mfloat-abi=softfp -mfpu=neon -mtune=cortex-a8 ' to ' -march=armv7-a    -mthumb -mthumb-interwork -mfloat-abi=softfp -mfpu=neon -mtune=cortex-a8'
> > >    Hash for dependent task depmodwrapper-cross_1.0.bb.do_install changed from 4494cb6c117309a9b4c541ad86c0aac5 to 1e3bcf2d14e5bd30a38350b41bb98db3
> > 
> > No objection to this patch, but it does raise the question.  Should we somehow 
> > strip the leading and training white space, maybe even multiple spaces (like 
> > above) and then use the results to generate the checksum?
> > 
> > If I saw the above, my first comment would be "well that's stupid, the output is 
> > the same"..  (Of course performance might be impacted by all of that stripping..)
> 
> I had the same comment when I've seen it, that's why I added those
> quotes to be able to spot that space at all.
> 
> I'm not sure what caused that extra space and so many rebuilds, but I've
> noticed it today in completely different build where space in multiline
> variable were squashed to one space separator and I think the only
> difference were pool patches in bitbake.

The pool patches shouldn't make any difference to this so this is rather
bizarre. Do you have anything you can reproduce at will (ignoring the
pool patches since they've been reverted for other reasons now).

Cheers,

Richard

Patch

diff --git a/lib/bb/siggen.py b/lib/bb/siggen.py
index 50e4558..8861337 100644
--- a/lib/bb/siggen.py
+++ b/lib/bb/siggen.py
@@ -331,12 +331,12 @@  def compare_sigfiles(a, b, recursecb = None):
         return changed, added, removed
 
     if 'basewhitelist' in a_data and a_data['basewhitelist'] != b_data['basewhitelist']:
-        output.append("basewhitelist changed from %s to %s" % (a_data['basewhitelist'], b_data['basewhitelist']))
+        output.append("basewhitelist changed from '%s' to '%s'" % (a_data['basewhitelist'], b_data['basewhitelist']))
         if a_data['basewhitelist'] and b_data['basewhitelist']:
             output.append("changed items: %s" % a_data['basewhitelist'].symmetric_difference(b_data['basewhitelist']))
 
     if 'taskwhitelist' in a_data and a_data['taskwhitelist'] != b_data['taskwhitelist']:
-        output.append("taskwhitelist changed from %s to %s" % (a_data['taskwhitelist'], b_data['taskwhitelist']))
+        output.append("taskwhitelist changed from '%s' to '%s'" % (a_data['taskwhitelist'], b_data['taskwhitelist']))
         if a_data['taskwhitelist'] and b_data['taskwhitelist']:
             output.append("changed items: %s" % a_data['taskwhitelist'].symmetric_difference(b_data['taskwhitelist']))
 
@@ -349,7 +349,7 @@  def compare_sigfiles(a, b, recursecb = None):
     changed, added, removed = dict_diff(a_data['gendeps'], b_data['gendeps'], a_data['basewhitelist'] & b_data['basewhitelist'])
     if changed:
         for dep in changed:
-            output.append("List of dependencies for variable %s changed from %s to %s" % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep]))
+            output.append("List of dependencies for variable %s changed from '%s' to '%s'" % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep]))
             if a_data['gendeps'][dep] and b_data['gendeps'][dep]:
                 output.append("changed items: %s" % a_data['gendeps'][dep].symmetric_difference(b_data['gendeps'][dep]))
     if added:
@@ -363,7 +363,7 @@  def compare_sigfiles(a, b, recursecb = None):
     changed, added, removed = dict_diff(a_data['varvals'], b_data['varvals'])
     if changed:
         for dep in changed:
-            output.append("Variable %s value changed from %s to %s" % (dep, a_data['varvals'][dep], b_data['varvals'][dep]))
+            output.append("Variable %s value changed from '%s' to '%s'" % (dep, a_data['varvals'][dep], b_data['varvals'][dep]))
 
     changed, added, removed = dict_diff(a_data['file_checksum_values'], b_data['file_checksum_values'])
     if changed: