Patchwork sstate.bbclass: Optimise sstate_hardcode_path

login
register
mail settings
Submitter Richard Purdie
Date Feb. 10, 2012, 12:07 a.m.
Message ID <1328832425.10451.48.camel@ted>
Download mbox | patch
Permalink /patch/21071/
State Accepted
Commit 4159ff761cf29a03aeb56c7686a4e7af02b52219
Headers show

Comments

Richard Purdie - Feb. 10, 2012, 12:07 a.m.
The sstate_hardcode_path() function triggered large numbers of exec()
calls when processing packages with large numbers of file relocations
(e.g. perl). This patch optimises those calls into longer single commands
which make the code significantly more efficient.

This reduced the do_package time for perl by 2 minutes (from 4.75 minutes)
for me.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
Joshua Lock - Feb. 10, 2012, 2:23 a.m.
On 09/02/12 16:07, Richard Purdie wrote:
> The sstate_hardcode_path() function triggered large numbers of exec()
> calls when processing packages with large numbers of file relocations
> (e.g. perl). This patch optimises those calls into longer single commands
> which make the code significantly more efficient.
>
> This reduced the do_package time for perl by 2 minutes (from 4.75 minutes)
> for me.

Very nice speed up!

> Signed-off-by: Richard Purdie<richard.purdie@linuxfoundation.org>

A minor api/style nit below but:

Acked-by: Joshua Lock <josh@linux.intel.com>

> ---
> diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
> index 4bd3712..d4f95c1 100644
> --- a/meta/classes/sstate.bbclass
> +++ b/meta/classes/sstate.bbclass
> @@ -317,19 +317,24 @@ def sstate_hardcode_path(d):
>   	staging_host = d.getVar('STAGING_DIR_HOST', True)
>   	sstate_builddir = d.getVar('SSTATE_BUILDDIR', True)
>
> -	for i in file_list.split('\n'):
> -		if not i:
> -			continue
> -		if bb.data.inherits_class('native', d) or bb.data.inherits_class('nativesdk', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('cross-canadian', d):
> -			cmd = "sed -i -e s:%s:FIXMESTAGINGDIR:g %s" % (staging, i)
> -		elif bb.data.inherits_class('cross', d):
> -			cmd = "sed -i -e s:%s:FIXMESTAGINGDIRTARGET:g %s \
> -				sed -i -e s:%s:FIXMESTAGINGDIR:g %s" % (staging_target, i, staging, i)
> -		else:
> -			cmd = "sed -i -e s:%s:FIXMESTAGINGDIRHOST:g %s" % (staging_host, i)
> +	files = " ".join(file_list.split('\n'))
>
> +	if bb.data.inherits_class('native', d) or bb.data.inherits_class('nativesdk', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('cross-canadian', d):

I think this can be replaced by a single call to the inherits method of 
the oe.utils module:

oe.utils.inherits(d, 'native', 'nativesdk', 'crosssdk', 'cross-canadian')

which makes this a little more concise.

> +		cmd = "sed -i -e s:%s:FIXMESTAGINGDIR:g %s" % (staging, files)
> +	elif bb.data.inherits_class('cross', d):
> +		cmd = "sed -i -e s:%s:FIXMESTAGINGDIRTARGET:g %s \
> +			sed -i -e s:%s:FIXMESTAGINGDIR:g %s" % (staging_target, files, staging, files)
> +	else:
> +		cmd = "sed -i -e s:%s:FIXMESTAGINGDIRHOST:g %s" % (staging_host, files)
> +
> +	if files:
>   		os.system(cmd)
> -		os.system("echo %s | sed -e 's:%s::'>>  %sfixmepath" % (i, sstate_builddir, sstate_builddir))
> +		fix = open("%sfixmepath" % (sstate_builddir), "w")
> +		fixme = []
> +		for f in file_list.split('\n'):
> +			fixme.append(f.replace(sstate_builddir, ""))
> +		fix.write("\n".join(fixme))
> +		fix.close()
>   	p.close()
>
>   def sstate_package(ss, d):
>
>
>
> _______________________________________________
> Openembedded-core mailing list
> Openembedded-core@lists.openembedded.org
> http://lists.linuxtogo.org/cgi-bin/mailman/listinfo/openembedded-core

Patch

diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index 4bd3712..d4f95c1 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -317,19 +317,24 @@  def sstate_hardcode_path(d):
 	staging_host = d.getVar('STAGING_DIR_HOST', True)
 	sstate_builddir = d.getVar('SSTATE_BUILDDIR', True)
 
-	for i in file_list.split('\n'):
-		if not i:
-			continue           
-		if bb.data.inherits_class('native', d) or bb.data.inherits_class('nativesdk', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('cross-canadian', d):
-			cmd = "sed -i -e s:%s:FIXMESTAGINGDIR:g %s" % (staging, i)
-		elif bb.data.inherits_class('cross', d):
-			cmd = "sed -i -e s:%s:FIXMESTAGINGDIRTARGET:g %s \
-				sed -i -e s:%s:FIXMESTAGINGDIR:g %s" % (staging_target, i, staging, i)
-		else:
-			cmd = "sed -i -e s:%s:FIXMESTAGINGDIRHOST:g %s" % (staging_host, i)
+	files = " ".join(file_list.split('\n'))
 
+	if bb.data.inherits_class('native', d) or bb.data.inherits_class('nativesdk', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('cross-canadian', d):
+		cmd = "sed -i -e s:%s:FIXMESTAGINGDIR:g %s" % (staging, files)
+	elif bb.data.inherits_class('cross', d):
+		cmd = "sed -i -e s:%s:FIXMESTAGINGDIRTARGET:g %s \
+			sed -i -e s:%s:FIXMESTAGINGDIR:g %s" % (staging_target, files, staging, files)
+	else:
+		cmd = "sed -i -e s:%s:FIXMESTAGINGDIRHOST:g %s" % (staging_host, files)
+
+	if files:
 		os.system(cmd)
-		os.system("echo %s | sed -e 's:%s::' >> %sfixmepath" % (i, sstate_builddir, sstate_builddir))
+		fix = open("%sfixmepath" % (sstate_builddir), "w")
+		fixme = []
+		for f in file_list.split('\n'):
+			fixme.append(f.replace(sstate_builddir, ""))
+		fix.write("\n".join(fixme))
+		fix.close()
 	p.close()
 
 def sstate_package(ss, d):