Patchwork [03/22] package: Process package stripping in parallel

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes/package.bbclass |   58 ++++++++++--------------------------------
 meta/lib/oe/package.py       |   45 ++++++++++++++++++++++++++++++++
 2 files changed, 58 insertions(+), 45 deletions(-)

On 2/3/13 5:55 PM, Richard Purdie wrote:
> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> ---
>   meta/classes/package.bbclass |   58 ++++++++++--------------------------------
>   meta/lib/oe/package.py       |   45 ++++++++++++++++++++++++++++++++
>   2 files changed, 58 insertions(+), 45 deletions(-)
>
> diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
> index 527ef31..9c8cdbc 100644
> --- a/meta/classes/package.bbclass
> +++ b/meta/classes/package.bbclass
> @@ -309,49 +309,6 @@ def copydebugsources(debugsrcdir, d):
>               if os.path.exists(p) and not os.listdir(p):
>                   os.rmdir(p)
>
> -def runstrip(file, elftype, d):
> -    # Function to strip a single file, called from split_and_strip_files below
> -    # A working 'file' (one which works on the target architecture)
> -    #
> -    # The elftype is a bit pattern (explained in split_and_strip_files) to tell
> -    # us what type of file we're processing...
> -    # 4 - executable
> -    # 8 - shared library
> -
> -    import commands, stat, subprocess
> -
> -    strip = d.getVar("STRIP", True)
> -
> -    newmode = None
> -    if not os.access(file, os.W_OK) or os.access(file, os.R_OK):
> -        origmode = os.stat(file)[stat.ST_MODE]
> -        newmode = origmode | stat.S_IWRITE | stat.S_IREAD
> -        os.chmod(file, newmode)
> -
> -    extraflags = ""
> -
> -    # .so and shared library
> -    if elftype & 16:
> -        extraflags = "--strip-debug --remove-section=.comment --remove-section=.note --preserve-dates"
> -    elif ".so" in file and elftype & 8:
> -        extraflags = "--remove-section=.comment --remove-section=.note --strip-unneeded"
> -    # shared or executable:
> -    elif elftype & 8 or elftype & 4:
> -        extraflags = "--remove-section=.comment --remove-section=.note"
> -
> -    stripcmd = "'%s' %s '%s'" % (strip, extraflags, file)
> -    bb.debug(1, "runstrip: %s" % stripcmd)
> -
> -    ret = subprocess.call(stripcmd, shell=True)
> -
> -    if newmode:
> -        os.chmod(file, origmode)
> -
> -    if ret:
> -        bb.error("runstrip: '%s' strip command failed" % stripcmd)
> -
> -    return 0
> -
>   #
>   # Package data handling routines
>   #
> @@ -902,13 +859,24 @@ python split_and_strip_files () {
>       # Now lets go back over things and strip them
>       #
>       if (d.getVar('INHIBIT_PACKAGE_STRIP', True) != '1'):
> +        strip = d.getVar("STRIP", True)
> +        sfiles = []
>           for file in file_list:
>               if file_list[file].startswith("ELF: "):
>                   elf_file = int(file_list[file][5:])
>                   #bb.note("Strip %s" % file)
> -                runstrip(file, elf_file, d)
> +                sfiles.append((file, elf_file, strip))
>           for f in kernmods:
> -            runstrip(f, 16, d)
> +            sfiles.append((f, 16, strip))
> +
> +
> +        import multiprocessing
> +        nproc = multiprocessing.cpu_count()
> +        pool = multiprocessing.Pool(nproc)
> +        processed = pool.imap(oe.package.runstrip, sfiles)
> +        pool.close()
> +        pool.join()
> +
>       #
>       # End of strip
>       #
> diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py
> index 6b1c1f4..9a0ddb8 100644
> --- a/meta/lib/oe/package.py
> +++ b/meta/lib/oe/package.py
> @@ -1,3 +1,48 @@
> +def runstrip(arg):
> +    # Function to strip a single file, called from split_and_strip_files below

Minor note, the comment above is now confusing as it's unclear where the 
split_and_strip_files is.  It might be reasonable at this point to simply remove 
that -- or change the comment to indicate it's from the package.bbclass.

--Mark

> +    # A working 'file' (one which works on the target architecture)
> +    #
> +    # The elftype is a bit pattern (explained in split_and_strip_files) to tell
> +    # us what type of file we're processing...
> +    # 4 - executable
> +    # 8 - shared library
> +    # 16 - kernel module
> +
> +    import commands, stat, subprocess
> +
> +    (file, elftype, strip) = arg
> +
> +    newmode = None
> +    if not os.access(file, os.W_OK) or os.access(file, os.R_OK):
> +        origmode = os.stat(file)[stat.ST_MODE]
> +        newmode = origmode | stat.S_IWRITE | stat.S_IREAD
> +        os.chmod(file, newmode)
> +
> +    extraflags = ""
> +
> +    # kernel module
> +    if elftype & 16:
> +        extraflags = "--strip-debug --remove-section=.comment --remove-section=.note --preserve-dates"
> +    # .so and shared library
> +    elif ".so" in file and elftype & 8:
> +        extraflags = "--remove-section=.comment --remove-section=.note --strip-unneeded"
> +    # shared or executable:
> +    elif elftype & 8 or elftype & 4:
> +        extraflags = "--remove-section=.comment --remove-section=.note"
> +
> +    stripcmd = "'%s' %s '%s'" % (strip, extraflags, file)
> +    bb.debug(1, "runstrip: %s" % stripcmd)
> +
> +    ret = subprocess.call(stripcmd, shell=True)
> +
> +    if newmode:
> +        os.chmod(file, origmode)
> +
> +    if ret:
> +        bb.error("runstrip: '%s' strip command failed" % stripcmd)
> +
> +    return
> +
>
>   def file_translate(file):
>       ft = file.replace("@", "@at@")
>