Patchwork [03/22] package: Process package stripping in parallel

login
register
mail settings
Submitter Richard Purdie
Date Feb. 3, 2013, 11:55 p.m.
Message ID <bda5674ec4d39456cc53b1cf22f349f4880d3202.1359935562.git.richard.purdie@linuxfoundation.org>
Download mbox | patch
Permalink /patch/43895/
State Accepted
Commit 981fed49ee80560fb067b3f47aeada1fdee792ca
Headers show

Comments

Richard Purdie - Feb. 3, 2013, 11:55 p.m.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes/package.bbclass |   58 ++++++++++--------------------------------
 meta/lib/oe/package.py       |   45 ++++++++++++++++++++++++++++++++
 2 files changed, 58 insertions(+), 45 deletions(-)
Mark Hatle - Feb. 4, 2013, 5:23 p.m.
On 2/3/13 5:55 PM, Richard Purdie wrote:
> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> ---
>   meta/classes/package.bbclass |   58 ++++++++++--------------------------------
>   meta/lib/oe/package.py       |   45 ++++++++++++++++++++++++++++++++
>   2 files changed, 58 insertions(+), 45 deletions(-)
>
> diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
> index 527ef31..9c8cdbc 100644
> --- a/meta/classes/package.bbclass
> +++ b/meta/classes/package.bbclass
> @@ -309,49 +309,6 @@ def copydebugsources(debugsrcdir, d):
>               if os.path.exists(p) and not os.listdir(p):
>                   os.rmdir(p)
>
> -def runstrip(file, elftype, d):
> -    # Function to strip a single file, called from split_and_strip_files below
> -    # A working 'file' (one which works on the target architecture)
> -    #
> -    # The elftype is a bit pattern (explained in split_and_strip_files) to tell
> -    # us what type of file we're processing...
> -    # 4 - executable
> -    # 8 - shared library
> -
> -    import commands, stat, subprocess
> -
> -    strip = d.getVar("STRIP", True)
> -
> -    newmode = None
> -    if not os.access(file, os.W_OK) or os.access(file, os.R_OK):
> -        origmode = os.stat(file)[stat.ST_MODE]
> -        newmode = origmode | stat.S_IWRITE | stat.S_IREAD
> -        os.chmod(file, newmode)
> -
> -    extraflags = ""
> -
> -    # .so and shared library
> -    if elftype & 16:
> -        extraflags = "--strip-debug --remove-section=.comment --remove-section=.note --preserve-dates"
> -    elif ".so" in file and elftype & 8:
> -        extraflags = "--remove-section=.comment --remove-section=.note --strip-unneeded"
> -    # shared or executable:
> -    elif elftype & 8 or elftype & 4:
> -        extraflags = "--remove-section=.comment --remove-section=.note"
> -
> -    stripcmd = "'%s' %s '%s'" % (strip, extraflags, file)
> -    bb.debug(1, "runstrip: %s" % stripcmd)
> -
> -    ret = subprocess.call(stripcmd, shell=True)
> -
> -    if newmode:
> -        os.chmod(file, origmode)
> -
> -    if ret:
> -        bb.error("runstrip: '%s' strip command failed" % stripcmd)
> -
> -    return 0
> -
>   #
>   # Package data handling routines
>   #
> @@ -902,13 +859,24 @@ python split_and_strip_files () {
>       # Now lets go back over things and strip them
>       #
>       if (d.getVar('INHIBIT_PACKAGE_STRIP', True) != '1'):
> +        strip = d.getVar("STRIP", True)
> +        sfiles = []
>           for file in file_list:
>               if file_list[file].startswith("ELF: "):
>                   elf_file = int(file_list[file][5:])
>                   #bb.note("Strip %s" % file)
> -                runstrip(file, elf_file, d)
> +                sfiles.append((file, elf_file, strip))
>           for f in kernmods:
> -            runstrip(f, 16, d)
> +            sfiles.append((f, 16, strip))
> +
> +
> +        import multiprocessing
> +        nproc = multiprocessing.cpu_count()
> +        pool = multiprocessing.Pool(nproc)
> +        processed = pool.imap(oe.package.runstrip, sfiles)
> +        pool.close()
> +        pool.join()
> +
>       #
>       # End of strip
>       #
> diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py
> index 6b1c1f4..9a0ddb8 100644
> --- a/meta/lib/oe/package.py
> +++ b/meta/lib/oe/package.py
> @@ -1,3 +1,48 @@
> +def runstrip(arg):
> +    # Function to strip a single file, called from split_and_strip_files below

Minor note, the comment above is now confusing as it's unclear where the 
split_and_strip_files is.  It might be reasonable at this point to simply remove 
that -- or change the comment to indicate it's from the package.bbclass.

--Mark

> +    # A working 'file' (one which works on the target architecture)
> +    #
> +    # The elftype is a bit pattern (explained in split_and_strip_files) to tell
> +    # us what type of file we're processing...
> +    # 4 - executable
> +    # 8 - shared library
> +    # 16 - kernel module
> +
> +    import commands, stat, subprocess
> +
> +    (file, elftype, strip) = arg
> +
> +    newmode = None
> +    if not os.access(file, os.W_OK) or os.access(file, os.R_OK):
> +        origmode = os.stat(file)[stat.ST_MODE]
> +        newmode = origmode | stat.S_IWRITE | stat.S_IREAD
> +        os.chmod(file, newmode)
> +
> +    extraflags = ""
> +
> +    # kernel module
> +    if elftype & 16:
> +        extraflags = "--strip-debug --remove-section=.comment --remove-section=.note --preserve-dates"
> +    # .so and shared library
> +    elif ".so" in file and elftype & 8:
> +        extraflags = "--remove-section=.comment --remove-section=.note --strip-unneeded"
> +    # shared or executable:
> +    elif elftype & 8 or elftype & 4:
> +        extraflags = "--remove-section=.comment --remove-section=.note"
> +
> +    stripcmd = "'%s' %s '%s'" % (strip, extraflags, file)
> +    bb.debug(1, "runstrip: %s" % stripcmd)
> +
> +    ret = subprocess.call(stripcmd, shell=True)
> +
> +    if newmode:
> +        os.chmod(file, origmode)
> +
> +    if ret:
> +        bb.error("runstrip: '%s' strip command failed" % stripcmd)
> +
> +    return
> +
>
>   def file_translate(file):
>       ft = file.replace("@", "@at@")
>

Patch

diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index 527ef31..9c8cdbc 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -309,49 +309,6 @@  def copydebugsources(debugsrcdir, d):
             if os.path.exists(p) and not os.listdir(p):
                 os.rmdir(p)
 
-def runstrip(file, elftype, d):
-    # Function to strip a single file, called from split_and_strip_files below
-    # A working 'file' (one which works on the target architecture)
-    #
-    # The elftype is a bit pattern (explained in split_and_strip_files) to tell
-    # us what type of file we're processing...
-    # 4 - executable
-    # 8 - shared library
-
-    import commands, stat, subprocess
-
-    strip = d.getVar("STRIP", True)
-
-    newmode = None
-    if not os.access(file, os.W_OK) or os.access(file, os.R_OK):
-        origmode = os.stat(file)[stat.ST_MODE]
-        newmode = origmode | stat.S_IWRITE | stat.S_IREAD
-        os.chmod(file, newmode)
-
-    extraflags = ""
-    
-    # .so and shared library
-    if elftype & 16:
-        extraflags = "--strip-debug --remove-section=.comment --remove-section=.note --preserve-dates"
-    elif ".so" in file and elftype & 8:
-        extraflags = "--remove-section=.comment --remove-section=.note --strip-unneeded"
-    # shared or executable:
-    elif elftype & 8 or elftype & 4:
-        extraflags = "--remove-section=.comment --remove-section=.note"
-
-    stripcmd = "'%s' %s '%s'" % (strip, extraflags, file)
-    bb.debug(1, "runstrip: %s" % stripcmd)
-
-    ret = subprocess.call(stripcmd, shell=True)
-
-    if newmode:
-        os.chmod(file, origmode)
-
-    if ret:
-        bb.error("runstrip: '%s' strip command failed" % stripcmd)
-
-    return 0
-
 #
 # Package data handling routines
 #
@@ -902,13 +859,24 @@  python split_and_strip_files () {
     # Now lets go back over things and strip them
     #
     if (d.getVar('INHIBIT_PACKAGE_STRIP', True) != '1'):
+        strip = d.getVar("STRIP", True)
+        sfiles = []
         for file in file_list:
             if file_list[file].startswith("ELF: "):
                 elf_file = int(file_list[file][5:])
                 #bb.note("Strip %s" % file)
-                runstrip(file, elf_file, d)
+                sfiles.append((file, elf_file, strip))
         for f in kernmods:
-            runstrip(f, 16, d)
+            sfiles.append((f, 16, strip))
+
+
+        import multiprocessing
+        nproc = multiprocessing.cpu_count()
+        pool = multiprocessing.Pool(nproc)
+        processed = pool.imap(oe.package.runstrip, sfiles)
+        pool.close()
+        pool.join()
+
     #
     # End of strip
     #
diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py
index 6b1c1f4..9a0ddb8 100644
--- a/meta/lib/oe/package.py
+++ b/meta/lib/oe/package.py
@@ -1,3 +1,48 @@ 
+def runstrip(arg):
+    # Function to strip a single file, called from split_and_strip_files below
+    # A working 'file' (one which works on the target architecture)
+    #
+    # The elftype is a bit pattern (explained in split_and_strip_files) to tell
+    # us what type of file we're processing...
+    # 4 - executable
+    # 8 - shared library
+    # 16 - kernel module
+
+    import commands, stat, subprocess
+
+    (file, elftype, strip) = arg
+
+    newmode = None
+    if not os.access(file, os.W_OK) or os.access(file, os.R_OK):
+        origmode = os.stat(file)[stat.ST_MODE]
+        newmode = origmode | stat.S_IWRITE | stat.S_IREAD
+        os.chmod(file, newmode)
+
+    extraflags = ""
+
+    # kernel module    
+    if elftype & 16:
+        extraflags = "--strip-debug --remove-section=.comment --remove-section=.note --preserve-dates"
+    # .so and shared library
+    elif ".so" in file and elftype & 8:
+        extraflags = "--remove-section=.comment --remove-section=.note --strip-unneeded"
+    # shared or executable:
+    elif elftype & 8 or elftype & 4:
+        extraflags = "--remove-section=.comment --remove-section=.note"
+
+    stripcmd = "'%s' %s '%s'" % (strip, extraflags, file)
+    bb.debug(1, "runstrip: %s" % stripcmd)
+
+    ret = subprocess.call(stripcmd, shell=True)
+
+    if newmode:
+        os.chmod(file, origmode)
+
+    if ret:
+        bb.error("runstrip: '%s' strip command failed" % stripcmd)
+
+    return
+
 
 def file_translate(file):
     ft = file.replace("@", "@at@")