Patchwork [1/1] list-packageconfig-flag.py: add a script to list all PACKAGECONFIG's flags

login
register
mail settings
Submitter Hongxu Jia
Date Aug. 1, 2013, 11:04 a.m.
Message ID <2339c72ba0e5afb7379f065ad403e3209b334956.1375354695.git.hongxu.jia@windriver.com>
Download mbox | patch
Permalink /patch/54897/
State New
Headers show

Comments

Hongxu Jia - Aug. 1, 2013, 11:04 a.m.
list-packageconfig-flag.py will walk the METADIR and collect recipes which
have PACKAGECONFIG's flags.

The default display is to list recipes which have PACKAGECONFIG's flags in
METADIR. If option '-f' is used, it will list PACKAGECONFIG's flags and all
affected recipes in METADIR

EXAMPLE:
list-packageconfig-flag.py poky/meta poky/meta-yocto
RECIPE NAME          PACKAGECONFIG's flag
Otavio Salvador - Aug. 1, 2013, 4:46 p.m.
On Thu, Aug 1, 2013 at 8:04 AM, Hongxu Jia <hongxu.jia@windriver.com> wrote:
> list-packageconfig-flag.py will walk the METADIR and collect recipes which
> have PACKAGECONFIG's flags.
>
> The default display is to list recipes which have PACKAGECONFIG's flags in
> METADIR. If option '-f' is used, it will list PACKAGECONFIG's flags and all
> affected recipes in METADIR
>
> EXAMPLE:
> list-packageconfig-flag.py poky/meta poky/meta-yocto
> RECIPE NAME          PACKAGECONFIG's flag
> =========================================
> libarchive_2.8.5.bb  acl xattr largefile zlib bz2 xz openssl libxml2 expat
> strace_4.8.bb        libaio acl
> connman.inc          wifi bluetooth 3g tist openvpn vpnc l2tp pptp wispr
>
> list-packageconfig-flag.py -f poky/meta poky/meta-yocto
> PACKAGECONFIG's flag     RECIPE NAME
> ====================================
> speex                    gstreamer1.0-plugins-good.inc
> keyutils                 rpm_5.4.9.bb
> gallium-egl              mesa.inc
>
> [YOCTO #4368]
>
> Signed-off-by: Hongxu Jia <hongxu.jia@windriver.com>

Cool! :-)

>  scripts/contrib/list-packageconfig-flag.py | 180 +++++++++++++++++++++++++++++

I'd drop .py extension and use +x permission.

>  1 file changed, 180 insertions(+)
>  create mode 100755 scripts/contrib/list-packageconfig-flag.py
>
> diff --git a/scripts/contrib/list-packageconfig-flag.py b/scripts/contrib/list-packageconfig-flag.py
> new file mode 100755
> index 0000000..7a5568f
> --- /dev/null
> +++ b/scripts/contrib/list-packageconfig-flag.py
> @@ -0,0 +1,180 @@
> +#!/usr/bin/env python
> +
> +# This program is free software; you can redistribute it and/or modify
> +# it under the terms of the GNU General Public License as published by
> +# the Free Software Foundation; either version 2 of the License, or
> +# (at your option) any later version.
> +#
> +# This program is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> +# GNU General Public License for more details.
> +#
> +# You should have received a copy of the GNU General Public License
> +# along with this program; if not, write to the Free Software Foundation.
> +#
> +# Copyright (C) 2013 Wind River Systems, Inc.
> +#
> +# list recipes which have PACKAGECONFIG's flags in METADIR
> +# list PACKAGECONFIG's flags and all affected recipes in METADIR
> +#
> +
> +import sys
> +import getopt
> +import os
> +import re
> +
> +usage_body = '''  list recipes which have PACKAGECONFIG's flags in METADIR
> +
> +OPTION:
> +  -h, --help  display this help and exit
> +  -f, --flag  list PACKAGECONFIG's flags and all affected recipes in METADIR
> +
> +EXAMPLE:
> +list-packageconfig-flag.py poky/meta poky/meta-yocto
> +list-packageconfig-flag.py -f poky/meta poky/meta-yocto
> +'''
> +
> +def usage():
> +    print 'Usage: %s [OPTION] [METADIR]...' % os.path.basename(sys.argv[0])
> +    print usage_body
> +
> +def parse_recipe(recipe):
> +    ''' Parse a recipe to collect PACKAGECONFIG's flags '''
> +    prog = re.compile(r' *PACKAGECONFIG\[(?P<flag>.*)\] *=.*')
> +    try:
> +        r = open(recipe)
> +    except IOError as (errno, strerror):
> +        print >> sys.stderr, 'WARNING: Failed to open recipe', recipe
> +
> +    flaglist = []
> +    for line in r:
> +        # Strip any comments from the line
> +        line = line.rsplit('#')[0]
> +        m = prog.match(line)
> +        if m:
> +            flaglist.append(m.group('flag'))
> +    r.close()
> +
> +    return flaglist
> +
> +def process_recipes(metadir):
> +    ''' Collect recipes which have PACKAGECONFIG's flags in METADIR '''
> +    # recipesdict = {'recipe': ['flag1', 'flag2',...]}
> +    recipesdict = {}
> +    for root,dirs,files in os.walk(metadir):
> +        for name in files:
> +            if name.find(".bb") >= 0 or name.find(".inc") >= 0:
> +                flaglist = parse_recipe(os.path.join(root,name))
> +                if flaglist:
> +                    recipesdict[name] = flaglist
> +
> +    return recipesdict

In case a PACKAGECONFIG is add by a class. I think this case is missing here.
Hongxu Jia - Aug. 2, 2013, 1:50 a.m.
On 08/02/2013 12:46 AM, Otavio Salvador wrote:
> On Thu, Aug 1, 2013 at 8:04 AM, Hongxu Jia <hongxu.jia@windriver.com> wrote:
>> list-packageconfig-flag.py will walk the METADIR and collect recipes which
>> have PACKAGECONFIG's flags.
>>
>> The default display is to list recipes which have PACKAGECONFIG's flags in
>> METADIR. If option '-f' is used, it will list PACKAGECONFIG's flags and all
>> affected recipes in METADIR
>>
>> EXAMPLE:
>> list-packageconfig-flag.py poky/meta poky/meta-yocto
>> RECIPE NAME          PACKAGECONFIG's flag
>> =========================================
>> libarchive_2.8.5.bb  acl xattr largefile zlib bz2 xz openssl libxml2 expat
>> strace_4.8.bb        libaio acl
>> connman.inc          wifi bluetooth 3g tist openvpn vpnc l2tp pptp wispr
>>
>> list-packageconfig-flag.py -f poky/meta poky/meta-yocto
>> PACKAGECONFIG's flag     RECIPE NAME
>> ====================================
>> speex                    gstreamer1.0-plugins-good.inc
>> keyutils                 rpm_5.4.9.bb
>> gallium-egl              mesa.inc
>>
>> [YOCTO #4368]
>>
>> Signed-off-by: Hongxu Jia <hongxu.jia@windriver.com>
> Cool! :-)
>
>>   scripts/contrib/list-packageconfig-flag.py | 180 +++++++++++++++++++++++++++++
> I'd drop .py extension and use +x permission.
>
Most scripts in scripts/contrib have '.py' or '.sh' extension, is it the 
rule to be followed or not?

It looks like the following 'new file mode 10755' means +x permission is 
used.
>>   1 file changed, 180 insertions(+)
>>   create mode 100755 scripts/contrib/list-packageconfig-flag.py
>>
>> diff --git a/scripts/contrib/list-packageconfig-flag.py b/scripts/contrib/list-packageconfig-flag.py
>> new file mode 100755
>> index 0000000..7a5568f
>> --- /dev/null
>> +++ b/scripts/contrib/list-packageconfig-flag.py
>> @@ -0,0 +1,180 @@
>> +#!/usr/bin/env python
>> +
>> +# This program is free software; you can redistribute it and/or modify
>> +# it under the terms of the GNU General Public License as published by
>> +# the Free Software Foundation; either version 2 of the License, or
>> +# (at your option) any later version.
>> +#
>> +# This program is distributed in the hope that it will be useful,
>> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
>> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
>> +# GNU General Public License for more details.
>> +#
>> +# You should have received a copy of the GNU General Public License
>> +# along with this program; if not, write to the Free Software Foundation.
>> +#
>> +# Copyright (C) 2013 Wind River Systems, Inc.
>> +#
>> +# list recipes which have PACKAGECONFIG's flags in METADIR
>> +# list PACKAGECONFIG's flags and all affected recipes in METADIR
>> +#
>> +
>> +import sys
>> +import getopt
>> +import os
>> +import re
>> +
>> +usage_body = '''  list recipes which have PACKAGECONFIG's flags in METADIR
>> +
>> +OPTION:
>> +  -h, --help  display this help and exit
>> +  -f, --flag  list PACKAGECONFIG's flags and all affected recipes in METADIR
>> +
>> +EXAMPLE:
>> +list-packageconfig-flag.py poky/meta poky/meta-yocto
>> +list-packageconfig-flag.py -f poky/meta poky/meta-yocto
>> +'''
>> +
>> +def usage():
>> +    print 'Usage: %s [OPTION] [METADIR]...' % os.path.basename(sys.argv[0])
>> +    print usage_body
>> +
>> +def parse_recipe(recipe):
>> +    ''' Parse a recipe to collect PACKAGECONFIG's flags '''
>> +    prog = re.compile(r' *PACKAGECONFIG\[(?P<flag>.*)\] *=.*')
>> +    try:
>> +        r = open(recipe)
>> +    except IOError as (errno, strerror):
>> +        print >> sys.stderr, 'WARNING: Failed to open recipe', recipe
>> +
>> +    flaglist = []
>> +    for line in r:
>> +        # Strip any comments from the line
>> +        line = line.rsplit('#')[0]
>> +        m = prog.match(line)
>> +        if m:
>> +            flaglist.append(m.group('flag'))
>> +    r.close()
>> +
>> +    return flaglist
>> +
>> +def process_recipes(metadir):
>> +    ''' Collect recipes which have PACKAGECONFIG's flags in METADIR '''
>> +    # recipesdict = {'recipe': ['flag1', 'flag2',...]}
>> +    recipesdict = {}
>> +    for root,dirs,files in os.walk(metadir):
>> +        for name in files:
>> +            if name.find(".bb") >= 0 or name.find(".inc") >= 0:
>> +                flaglist = parse_recipe(os.path.join(root,name))
>> +                if flaglist:
>> +                    recipesdict[name] = flaglist
>> +
>> +    return recipesdict
> In case a PACKAGECONFIG is add by a class. I think this case is missing here.
Agree, sorry for the missing, I will add it.

Thanks,
Hongxu
>
Otavio Salvador - Aug. 2, 2013, 1:54 a.m.
On Thu, Aug 1, 2013 at 10:50 PM, Hongxu Jia <hongxu.jia@windriver.com> wrote:
> On 08/02/2013 12:46 AM, Otavio Salvador wrote:
>>
>> On Thu, Aug 1, 2013 at 8:04 AM, Hongxu Jia <hongxu.jia@windriver.com>
>> wrote:
>>>
>>> list-packageconfig-flag.py will walk the METADIR and collect recipes
>>> which
>>> have PACKAGECONFIG's flags.
>>>
>>> The default display is to list recipes which have PACKAGECONFIG's flags
>>> in
>>> METADIR. If option '-f' is used, it will list PACKAGECONFIG's flags and
>>> all
>>> affected recipes in METADIR
>>>
>>> EXAMPLE:
>>> list-packageconfig-flag.py poky/meta poky/meta-yocto
>>> RECIPE NAME          PACKAGECONFIG's flag
>>> =========================================
>>> libarchive_2.8.5.bb  acl xattr largefile zlib bz2 xz openssl libxml2
>>> expat
>>> strace_4.8.bb        libaio acl
>>> connman.inc          wifi bluetooth 3g tist openvpn vpnc l2tp pptp wispr
>>>
>>> list-packageconfig-flag.py -f poky/meta poky/meta-yocto
>>> PACKAGECONFIG's flag     RECIPE NAME
>>> ====================================
>>> speex                    gstreamer1.0-plugins-good.inc
>>> keyutils                 rpm_5.4.9.bb
>>> gallium-egl              mesa.inc
>>>
>>> [YOCTO #4368]
>>>
>>> Signed-off-by: Hongxu Jia <hongxu.jia@windriver.com>
>>
>> Cool! :-)
>>
>>>   scripts/contrib/list-packageconfig-flag.py | 180
>>> +++++++++++++++++++++++++++++
>>
>> I'd drop .py extension and use +x permission.
>>
> Most scripts in scripts/contrib have '.py' or '.sh' extension, is it the
> rule to be followed or not?

I'd prefer to not have the .py and .sh extensions. The script language
is a technical detail and it does not seem important to be evident.
But I'd say for now, keep it and  we can change it in a later patch.

> It looks like the following 'new file mode 10755' means +x permission is
> used.

Missed.

>>>   1 file changed, 180 insertions(+)
>>>   create mode 100755 scripts/contrib/list-packageconfig-flag.py
>>>
>>> diff --git a/scripts/contrib/list-packageconfig-flag.py
>>> b/scripts/contrib/list-packageconfig-flag.py
>>> new file mode 100755
>>> index 0000000..7a5568f
>>> --- /dev/null
>>> +++ b/scripts/contrib/list-packageconfig-flag.py
>>> @@ -0,0 +1,180 @@
>>> +#!/usr/bin/env python
>>> +
>>> +# This program is free software; you can redistribute it and/or modify
>>> +# it under the terms of the GNU General Public License as published by
>>> +# the Free Software Foundation; either version 2 of the License, or
>>> +# (at your option) any later version.
>>> +#
>>> +# This program is distributed in the hope that it will be useful,
>>> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
>>> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
>>> +# GNU General Public License for more details.
>>> +#
>>> +# You should have received a copy of the GNU General Public License
>>> +# along with this program; if not, write to the Free Software
>>> Foundation.
>>> +#
>>> +# Copyright (C) 2013 Wind River Systems, Inc.
>>> +#
>>> +# list recipes which have PACKAGECONFIG's flags in METADIR
>>> +# list PACKAGECONFIG's flags and all affected recipes in METADIR
>>> +#
>>> +
>>> +import sys
>>> +import getopt
>>> +import os
>>> +import re
>>> +
>>> +usage_body = '''  list recipes which have PACKAGECONFIG's flags in
>>> METADIR
>>> +
>>> +OPTION:
>>> +  -h, --help  display this help and exit
>>> +  -f, --flag  list PACKAGECONFIG's flags and all affected recipes in
>>> METADIR
>>> +
>>> +EXAMPLE:
>>> +list-packageconfig-flag.py poky/meta poky/meta-yocto
>>> +list-packageconfig-flag.py -f poky/meta poky/meta-yocto
>>> +'''
>>> +
>>> +def usage():
>>> +    print 'Usage: %s [OPTION] [METADIR]...' %
>>> os.path.basename(sys.argv[0])
>>> +    print usage_body
>>> +
>>> +def parse_recipe(recipe):
>>> +    ''' Parse a recipe to collect PACKAGECONFIG's flags '''
>>> +    prog = re.compile(r' *PACKAGECONFIG\[(?P<flag>.*)\] *=.*')
>>> +    try:
>>> +        r = open(recipe)
>>> +    except IOError as (errno, strerror):
>>> +        print >> sys.stderr, 'WARNING: Failed to open recipe', recipe
>>> +
>>> +    flaglist = []
>>> +    for line in r:
>>> +        # Strip any comments from the line
>>> +        line = line.rsplit('#')[0]
>>> +        m = prog.match(line)
>>> +        if m:
>>> +            flaglist.append(m.group('flag'))
>>> +    r.close()
>>> +
>>> +    return flaglist
>>> +
>>> +def process_recipes(metadir):
>>> +    ''' Collect recipes which have PACKAGECONFIG's flags in METADIR '''
>>> +    # recipesdict = {'recipe': ['flag1', 'flag2',...]}
>>> +    recipesdict = {}
>>> +    for root,dirs,files in os.walk(metadir):
>>> +        for name in files:
>>> +            if name.find(".bb") >= 0 or name.find(".inc") >= 0:
>>> +                flaglist = parse_recipe(os.path.join(root,name))
>>> +                if flaglist:
>>> +                    recipesdict[name] = flaglist
>>> +
>>> +    return recipesdict
>>
>> In case a PACKAGECONFIG is add by a class. I think this case is missing
>> here.
>
> Agree, sorry for the missing, I will add it.

Thanks.
Paul Eggleton - Aug. 2, 2013, 8:24 a.m.
Hi Hongxu,

Great idea! This should be very useful indeed.

Just a few feedback points:

On Thursday 01 August 2013 19:04:05 Hongxu Jia wrote:
> list-packageconfig-flag.py will walk the METADIR and collect recipes which
> have PACKAGECONFIG's flags.
> 
> The default display is to list recipes which have PACKAGECONFIG's flags in
> METADIR. If option '-f' is used, it will list PACKAGECONFIG's flags and all
> affected recipes in METADIR
> 
> EXAMPLE:
> list-packageconfig-flag.py poky/meta poky/meta-yocto
> RECIPE NAME          PACKAGECONFIG's flag
> =========================================
> libarchive_2.8.5.bb  acl xattr largefile zlib bz2 xz openssl libxml2 expat
> strace_4.8.bb        libaio acl
> connman.inc          wifi bluetooth 3g tist openvpn vpnc l2tp pptp wispr
> 
> list-packageconfig-flag.py -f poky/meta poky/meta-yocto
> PACKAGECONFIG's flag     RECIPE NAME
> ====================================
> speex                    gstreamer1.0-plugins-good.inc
> keyutils                 rpm_5.4.9.bb
> gallium-egl              mesa.inc
> 
> [YOCTO #4368]
> 
> Signed-off-by: Hongxu Jia <hongxu.jia@windriver.com>
> ---
>  scripts/contrib/list-packageconfig-flag.py | 180
> +++++++++++++++++++++++++++++ 1 file changed, 180 insertions(+)
>  create mode 100755 scripts/contrib/list-packageconfig-flag.py

I think the script ought to be called "list-packageconfig-flags" rather than 
"list-packageconfig-flag".

>...
> +def parse_recipe(recipe):
> +    ''' Parse a recipe to collect PACKAGECONFIG's flags '''

This should say "collect available PACKAGECONFIG flags". The same applies to 
other messages/comments.

> +    prog = re.compile(r' *PACKAGECONFIG\[(?P<flag>.*)\] *=.*')
> +    try:
> +        r = open(recipe)
> +    except IOError as (errno, strerror):
> +        print >> sys.stderr, 'WARNING: Failed to open recipe', recipe
> +
> +    flaglist = []
> +    for line in r:
> +        # Strip any comments from the line
> +        line = line.rsplit('#')[0]
> +        m = prog.match(line)
> +        if m:
> +            flaglist.append(m.group('flag'))
> +    r.close()
> +
> +    return flaglist
>
> +def process_recipes(metadir):
> +    ''' Collect recipes which have PACKAGECONFIG's flags in METADIR '''
> +    # recipesdict = {'recipe': ['flag1', 'flag2',...]}
> +    recipesdict = {}
> +    for root,dirs,files in os.walk(metadir):
> +        for name in files:
> +            if name.find(".bb") >= 0 or name.find(".inc") >= 0:
> +                flaglist = parse_recipe(os.path.join(root,name))
> +                if flaglist:
> +                    recipesdict[name] = flaglist
> +
> +    return recipesdict

So, this is fairly simple and works for most cases; however it would probably 
be a bit more robust if the script used bitbake's own parser to extract the 
flags. The "tinfoil" module should help with this. We can always address this 
in a follow-up patch later on, though - I can certainly show you some examples 
of how to do this or I can do it, whichever you prefer.

Cheers,
Paul
Hongxu Jia - Aug. 2, 2013, 9:14 a.m.
On 08/02/2013 04:24 PM, Paul Eggleton wrote:
> Hi Hongxu,
>
> Great idea! This should be very useful indeed.
>
> Just a few feedback points:
>
> On Thursday 01 August 2013 19:04:05 Hongxu Jia wrote:
>> list-packageconfig-flag.py will walk the METADIR and collect recipes which
>> have PACKAGECONFIG's flags.
>>
>> The default display is to list recipes which have PACKAGECONFIG's flags in
>> METADIR. If option '-f' is used, it will list PACKAGECONFIG's flags and all
>> affected recipes in METADIR
>>
>> EXAMPLE:
>> list-packageconfig-flag.py poky/meta poky/meta-yocto
>> RECIPE NAME          PACKAGECONFIG's flag
>> =========================================
>> libarchive_2.8.5.bb  acl xattr largefile zlib bz2 xz openssl libxml2 expat
>> strace_4.8.bb        libaio acl
>> connman.inc          wifi bluetooth 3g tist openvpn vpnc l2tp pptp wispr
>>
>> list-packageconfig-flag.py -f poky/meta poky/meta-yocto
>> PACKAGECONFIG's flag     RECIPE NAME
>> ====================================
>> speex                    gstreamer1.0-plugins-good.inc
>> keyutils                 rpm_5.4.9.bb
>> gallium-egl              mesa.inc
>>
>> [YOCTO #4368]
>>
>> Signed-off-by: Hongxu Jia <hongxu.jia@windriver.com>
>> ---
>>   scripts/contrib/list-packageconfig-flag.py | 180
>> +++++++++++++++++++++++++++++ 1 file changed, 180 insertions(+)
>>   create mode 100755 scripts/contrib/list-packageconfig-flag.py
> I think the script ought to be called "list-packageconfig-flags" rather than
> "list-packageconfig-flag".
>
Agree, I will rename it.
>> ...
>> +def parse_recipe(recipe):
>> +    ''' Parse a recipe to collect PACKAGECONFIG's flags '''
> This should say "collect available PACKAGECONFIG flags". The same applies to
> other messages/comments.
>
Agree
>> +    prog = re.compile(r' *PACKAGECONFIG\[(?P<flag>.*)\] *=.*')
>> +    try:
>> +        r = open(recipe)
>> +    except IOError as (errno, strerror):
>> +        print >> sys.stderr, 'WARNING: Failed to open recipe', recipe
>> +
>> +    flaglist = []
>> +    for line in r:
>> +        # Strip any comments from the line
>> +        line = line.rsplit('#')[0]
>> +        m = prog.match(line)
>> +        if m:
>> +            flaglist.append(m.group('flag'))
>> +    r.close()
>> +
>> +    return flaglist
>>
>> +def process_recipes(metadir):
>> +    ''' Collect recipes which have PACKAGECONFIG's flags in METADIR '''
>> +    # recipesdict = {'recipe': ['flag1', 'flag2',...]}
>> +    recipesdict = {}
>> +    for root,dirs,files in os.walk(metadir):
>> +        for name in files:
>> +            if name.find(".bb") >= 0 or name.find(".inc") >= 0:
>> +                flaglist = parse_recipe(os.path.join(root,name))
>> +                if flaglist:
>> +                    recipesdict[name] = flaglist
>> +
>> +    return recipesdict
> So, this is fairly simple and works for most cases; however it would probably
> be a bit more robust if the script used bitbake's own parser to extract the
> flags. The "tinfoil" module should help with this. We can always address this
> in a follow-up patch later on, though - I can certainly show you some examples
> of how to do this or I can do it, whichever you prefer.
Great, this is what I hope to do in the follow-up, I am very pleased to
do the investigate on bitbake parser, very thankful to share some
examples of how to do this with me.

Should this simple patch be accepted? And I will include bitbake parser
in follow-up patch.
Or
Drop this patch and waiting till bitbake parser has been included.

Thanks,
Hongxu
> Cheers,
> Paul
>
Hongxu Jia - Aug. 2, 2013, 10:03 a.m.
Hi Paul,

I have updated the pull branch to rename the script and correct the comments
as your suggestions.

The suggestion about using bitbake parser to make the script more robust,
I need some time to do the investigation and I am very pleased to do that,
I will address this in follow-up pathch, would you think it's proper or not?

Thanks,
Hongxu

On 08/02/2013 04:24 PM, Paul Eggleton wrote:
> Hi Hongxu,
>
> Great idea! This should be very useful indeed.
>
> Just a few feedback points:
>
> On Thursday 01 August 2013 19:04:05 Hongxu Jia wrote:
>> list-packageconfig-flag.py will walk the METADIR and collect recipes which
>> have PACKAGECONFIG's flags.
>>
>> The default display is to list recipes which have PACKAGECONFIG's flags in
>> METADIR. If option '-f' is used, it will list PACKAGECONFIG's flags and all
>> affected recipes in METADIR
>>
>> EXAMPLE:
>> list-packageconfig-flag.py poky/meta poky/meta-yocto
>> RECIPE NAME          PACKAGECONFIG's flag
>> =========================================
>> libarchive_2.8.5.bb  acl xattr largefile zlib bz2 xz openssl libxml2 expat
>> strace_4.8.bb        libaio acl
>> connman.inc          wifi bluetooth 3g tist openvpn vpnc l2tp pptp wispr
>>
>> list-packageconfig-flag.py -f poky/meta poky/meta-yocto
>> PACKAGECONFIG's flag     RECIPE NAME
>> ====================================
>> speex                    gstreamer1.0-plugins-good.inc
>> keyutils                 rpm_5.4.9.bb
>> gallium-egl              mesa.inc
>>
>> [YOCTO #4368]
>>
>> Signed-off-by: Hongxu Jia <hongxu.jia@windriver.com>
>> ---
>>   scripts/contrib/list-packageconfig-flag.py | 180
>> +++++++++++++++++++++++++++++ 1 file changed, 180 insertions(+)
>>   create mode 100755 scripts/contrib/list-packageconfig-flag.py
> I think the script ought to be called "list-packageconfig-flags" rather than
> "list-packageconfig-flag".
>
>> ...
>> +def parse_recipe(recipe):
>> +    ''' Parse a recipe to collect PACKAGECONFIG's flags '''
> This should say "collect available PACKAGECONFIG flags". The same applies to
> other messages/comments.
>
>> +    prog = re.compile(r' *PACKAGECONFIG\[(?P<flag>.*)\] *=.*')
>> +    try:
>> +        r = open(recipe)
>> +    except IOError as (errno, strerror):
>> +        print >> sys.stderr, 'WARNING: Failed to open recipe', recipe
>> +
>> +    flaglist = []
>> +    for line in r:
>> +        # Strip any comments from the line
>> +        line = line.rsplit('#')[0]
>> +        m = prog.match(line)
>> +        if m:
>> +            flaglist.append(m.group('flag'))
>> +    r.close()
>> +
>> +    return flaglist
>>
>> +def process_recipes(metadir):
>> +    ''' Collect recipes which have PACKAGECONFIG's flags in METADIR '''
>> +    # recipesdict = {'recipe': ['flag1', 'flag2',...]}
>> +    recipesdict = {}
>> +    for root,dirs,files in os.walk(metadir):
>> +        for name in files:
>> +            if name.find(".bb") >= 0 or name.find(".inc") >= 0:
>> +                flaglist = parse_recipe(os.path.join(root,name))
>> +                if flaglist:
>> +                    recipesdict[name] = flaglist
>> +
>> +    return recipesdict
> So, this is fairly simple and works for most cases; however it would probably
> be a bit more robust if the script used bitbake's own parser to extract the
> flags. The "tinfoil" module should help with this. We can always address this
> in a follow-up patch later on, though - I can certainly show you some examples
> of how to do this or I can do it, whichever you prefer.
>
> Cheers,
> Paul
>
Paul Eggleton - Aug. 2, 2013, 10:38 a.m.
On Friday 02 August 2013 17:14:34 Hongxu Jia wrote:
> On 08/02/2013 04:24 PM, Paul Eggleton wrote:
> > So, this is fairly simple and works for most cases; however it would
> > probably be a bit more robust if the script used bitbake's own parser to
> > extract the flags. The "tinfoil" module should help with this. We can
> > always address this in a follow-up patch later on, though - I can
> > certainly show you some examples of how to do this or I can do it,
> > whichever you prefer.
> 
> Great, this is what I hope to do in the follow-up, I am very pleased to
> do the investigate on bitbake parser, very thankful to share some
> examples of how to do this with me.

Sure - we use the parser in the layer index. The latest in-development code
has a module that does most of the work:

http://git.yoctoproject.org/cgit/cgit.cgi/layerindex-web/tree/layerindex/recipeparse.py?h=paule/features

Specifically, you can call bb.cache.Cache.loadDataFull() for each recipe file
and that will give you a datastore you can query for information.

I'm not sure, but you may wish to use bitbake's cache to get the list of recipes
rather than just walking the directory tree so the utility has the same
understanding of PN and preferred versions as bitbake. If you do want to
go down this route I've put together this example:

https://gist.github.com/bluelightning/6138980

Hope this helps.

> Should this simple patch be accepted? And I will include bitbake parser
> in follow-up patch.
> Or
> Drop this patch and waiting till bitbake parser has been included.

I'd say let's accept this simple version for now after the naming/message
corrections (especially since it's a contrib script) and then follow up 
with another patch when you have the bitbake-based version working.

Cheers,
Paul
Paul Eggleton - Aug. 2, 2013, 10:40 a.m.
On Friday 02 August 2013 18:03:05 Hongxu Jia wrote:
> I have updated the pull branch to rename the script and correct the comments
> as your suggestions.

There still seem to be a lot of instances of "PACKAGECONFIG's flags" on the 
branch.

> The suggestion about using bitbake parser to make the script more robust,
> I need some time to do the investigation and I am very pleased to do that,
> I will address this in follow-up pathch, would you think it's proper or not?

Yes, that's fine.

Cheers,
Paul
Hongxu Jia - Aug. 3, 2013, 3:15 a.m.
On 08/02/2013 06:40 PM, Paul Eggleton wrote:
> On Friday 02 August 2013 18:03:05 Hongxu Jia wrote:
>> I have updated the pull branch to rename the script and correct the comments
>> as your suggestions.
> There still seem to be a lot of instances of "PACKAGECONFIG's flags" on the
> branch.
Thanks Paul for the reviewing, I am sorry for the missing,
I have updated the pull branch again.

Thanks,
Hongxu
>> The suggestion about using bitbake parser to make the script more robust,
>> I need some time to do the investigation and I am very pleased to do that,
>> I will address this in follow-up pathch, would you think it's proper or not?
> Yes, that's fine.
>
> Cheers,
> Paul
>

Patch

=========================================
libarchive_2.8.5.bb  acl xattr largefile zlib bz2 xz openssl libxml2 expat
strace_4.8.bb        libaio acl
connman.inc          wifi bluetooth 3g tist openvpn vpnc l2tp pptp wispr

list-packageconfig-flag.py -f poky/meta poky/meta-yocto
PACKAGECONFIG's flag     RECIPE NAME
====================================
speex                    gstreamer1.0-plugins-good.inc
keyutils                 rpm_5.4.9.bb
gallium-egl              mesa.inc

[YOCTO #4368]

Signed-off-by: Hongxu Jia <hongxu.jia@windriver.com>
---
 scripts/contrib/list-packageconfig-flag.py | 180 +++++++++++++++++++++++++++++
 1 file changed, 180 insertions(+)
 create mode 100755 scripts/contrib/list-packageconfig-flag.py

diff --git a/scripts/contrib/list-packageconfig-flag.py b/scripts/contrib/list-packageconfig-flag.py
new file mode 100755
index 0000000..7a5568f
--- /dev/null
+++ b/scripts/contrib/list-packageconfig-flag.py
@@ -0,0 +1,180 @@ 
+#!/usr/bin/env python
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation.
+#
+# Copyright (C) 2013 Wind River Systems, Inc.
+#
+# list recipes which have PACKAGECONFIG's flags in METADIR
+# list PACKAGECONFIG's flags and all affected recipes in METADIR
+#
+
+import sys
+import getopt
+import os
+import re
+
+usage_body = '''  list recipes which have PACKAGECONFIG's flags in METADIR
+
+OPTION:
+  -h, --help  display this help and exit
+  -f, --flag  list PACKAGECONFIG's flags and all affected recipes in METADIR
+
+EXAMPLE:
+list-packageconfig-flag.py poky/meta poky/meta-yocto
+list-packageconfig-flag.py -f poky/meta poky/meta-yocto
+'''
+
+def usage():
+    print 'Usage: %s [OPTION] [METADIR]...' % os.path.basename(sys.argv[0])
+    print usage_body
+
+def parse_recipe(recipe):
+    ''' Parse a recipe to collect PACKAGECONFIG's flags '''
+    prog = re.compile(r' *PACKAGECONFIG\[(?P<flag>.*)\] *=.*')
+    try:
+        r = open(recipe)
+    except IOError as (errno, strerror):
+        print >> sys.stderr, 'WARNING: Failed to open recipe', recipe
+
+    flaglist = []
+    for line in r:
+        # Strip any comments from the line
+        line = line.rsplit('#')[0]
+        m = prog.match(line)
+        if m:
+            flaglist.append(m.group('flag'))
+    r.close()
+
+    return flaglist
+
+def process_recipes(metadir):
+    ''' Collect recipes which have PACKAGECONFIG's flags in METADIR '''
+    # recipesdict = {'recipe': ['flag1', 'flag2',...]}
+    recipesdict = {}
+    for root,dirs,files in os.walk(metadir):
+        for name in files:
+            if name.find(".bb") >= 0 or name.find(".inc") >= 0:
+                flaglist = parse_recipe(os.path.join(root,name))
+                if flaglist:
+                    recipesdict[name] = flaglist
+
+    return recipesdict
+
+def collect_flags(recipesdict):
+    ''' Collect PACKAGECONFIG's flags and all affected recipes in recipesdict '''
+    # flagsdict = {'flag': ['recipe1', 'recipe2',...]}
+    flagsdict = {}
+    for recipename,flaglist in recipesdict.iteritems():
+        for flag in flaglist:
+            if flag in flagsdict:
+                flagsdict[flag].append(recipename)
+            else:
+                flagsdict[flag] = [recipename]
+
+    return flagsdict
+
+def display_recipes(recipesdict):
+    ''' Display recipes which have PACKAGECONFIG's flags in recipesdict '''
+    recipename_len = len("RECIPE NAME") + 1
+    for recipename in recipesdict:
+        if recipename_len < len(recipename):
+            recipename_len = len(recipename)
+    recipename_len += 1
+
+    header = '%-*s%s' % (recipename_len, str("RECIPE NAME"), str("PACKAGECONFIG's flag"))
+    print header
+    print str("").ljust(len(header), '=')
+    for recipename, flaglist in recipesdict.iteritems():
+        print('%-*s%s' % (recipename_len, recipename, ' '.join(flaglist)))
+
+
+def display_flags(flagsdict):
+    ''' Display PACKAGECONFIG's flags and all affected recipes in flagsdict '''
+    flag_len = len("PACKAGECONFIG's flag") + 5
+
+    header = '%-*s%s' % (flag_len, str("PACKAGECONFIG's flag"), str("RECIPE NAME"))
+    print header
+    print str("").ljust(len(header), '=')
+
+    for flag in flagsdict:
+        print('%-*s%s' % (flag_len, flag, '  '.join(flagsdict[flag])))
+
+def filter_subdirs(dirlist):
+    '''Filter out subdirs in dirlist'''
+    def dir_is_subdir(dir, dirlist):
+        for curdir in dirlist:
+            dirs = dir.split('/')
+            curdirs = curdir.split('/')
+            curdirsdepth = len(curdirs)
+            if curdirsdepth < len(dirs):
+                if dirs[:curdirsdepth] == curdirs[:curdirsdepth]:
+                    return True
+        return False
+
+    for curdir in dirlist[:]:
+        if dir_is_subdir(curdir, dirlist):
+            dirlist.remove(curdir)
+
+def main():
+    metadirs = []
+    listtype = 'recipes'
+    recipesdict = {}
+    flagsdict = {}
+
+    # Collect and validate input
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "hf", ["help", "flag"])
+    except getopt.GetoptError, err:
+        print >> sys.stderr,'%s' % str(err)
+        usage()
+        sys.exit(2)
+    for opt, value in opts:
+        if opt in ('-h', '--help'):
+            usage()
+            sys.exit(0)
+        elif opt in ('-f', '--flag'):
+            listtype = 'flags'
+        else:
+            assert False, "unhandled option"
+
+    for metadir in args:
+        if os.path.isdir(metadir):
+            metadir = os.path.abspath(metadir)
+            if metadir not in metadirs:
+                metadirs.append(metadir)
+        else:
+            print >> sys.stderr, 'ERROR: meta directory \'%s\' is not a directory' % (metadir)
+            sys.exit(3)
+
+    if len(metadirs) == 0:
+        print >> sys.stderr, 'ERROR: no metadir specified'
+        usage()
+        sys.exit(4)
+    else:
+        # Filter out subdirs to save time
+        filter_subdirs(metadirs)
+
+    # Collect recipes which have PACKAGECONFIG's flags in metadirs
+    for metadir in metadirs:
+        recipesdict.update(process_recipes(metadir))
+
+    if listtype == 'flags':
+        # Collect PACKAGECONFIG's flags and all affected recipes in recipesdict
+        flagsdict = collect_flags(recipesdict)
+        display_flags(flagsdict)
+    elif listtype == 'recipes':
+        display_recipes(recipesdict)
+
+if __name__ == "__main__":
+    main()