package.bbclass: Optimise the per file rpm handling

Currently a process was being forked off for each individual file
this class wanted to inspect with rpmdeps. This converts it to use
rpmdeps-oecore which allows batch processing of these dependencies.

For do_package for perl, this reduced the time by about 1 minute (33%).

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---

On 09/02/12 16:10, Richard Purdie wrote:
> Currently a process was being forked off for each individual file
> this class wanted to inspect with rpmdeps. This converts it to use
> rpmdeps-oecore which allows batch processing of these dependencies.
>
> For do_package for perl, this reduced the time by about 1 minute (33%).

Awesome! Sounds like this addresses #1718

http://bugzilla.pokylinux.org/show_bug.cgi?id=1718

>
> Signed-off-by: Richard Purdie<richard.purdie@linuxfoundation.org>

A minor nit below, otherwise:

Acked-by: Joshua Lock <josh@linux.intel.com>

> ---
> diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
> index 348c13c..dfabcf8 100644
> --- a/meta/classes/package.bbclass
> +++ b/meta/classes/package.bbclass
> @@ -1125,7 +1125,7 @@ if [ x"$D" = "x" ]; then
>   fi
>   }
>
> -RPMDEPS = "${STAGING_LIBDIR_NATIVE}/rpm/bin/rpmdeps --macros ${STAGING_LIBDIR_NATIVE}/rpm/macros --define '_rpmfc_magic_path ${STAGING_DIR_NATIVE}${datadir_native}/misc/magic.mgc' --rpmpopt ${STAGING_LIBDIR_NATIVE}/rpm/rpmpopt"
> +RPMDEPS = "${STAGING_LIBDIR_NATIVE}/rpm/bin/rpmdeps-oecore --macros ${STAGING_LIBDIR_NATIVE}/rpm/macros --define '_rpmfc_magic_path ${STAGING_DIR_NATIVE}${datadir_native}/misc/magic.mgc' --rpmpopt ${STAGING_LIBDIR_NATIVE}/rpm/rpmpopt"
>
>   # Collect perfile run-time dependency metadata
>   # Output:
> @@ -1136,7 +1136,7 @@ RPMDEPS = "${STAGING_LIBDIR_NATIVE}/rpm/bin/rpmdeps --macros ${STAGING_LIBDIR_NA
>   #  FILERDEPENDS_filepath_pkg - per file dep
>
>   python package_do_filedeps() {
> -	import os, re
> +	import re, subprocess

Was this accidentally left in? I don't see any calls to subprocess?

>
>   	pkgdest = d.getVar('PKGDEST', True)
>   	packages = d.getVar('PACKAGES', True)
> @@ -1145,39 +1145,49 @@ python package_do_filedeps() {
>   	r = re.compile(r'[<>=]+ +[^ ]*')
>
>   	# Quick routine to process the results of the rpmdeps call...
> -	def process_deps(pipe, pkg, f, provides_files, requires_files):
> -		provides = []
> -		requires = []
> -		file = f.replace(pkgdest + "/" + pkg, "")
> -		file = file.replace("@", "@at@")
> -		file = file.replace(" ", "@space@")
> -		file = file.replace("\t", "@tab@")
> -		file = file.replace("[", "@openbrace@")
> -		file = file.replace("]", "@closebrace@")
> -		file = file.replace("_", "@underscore@")
> +	def process_deps(pipe, pkg, provides_files, requires_files):
> +		provides = {}
> +		requires = {}
>
>   		for line in pipe:
> +			f = line.split(" ", 1)[0].strip()
> +			line = line.split(" ", 1)[1].strip()
> +
>   			if line.startswith("Requires:"):
>   				i = requires
>   			elif line.startswith("Provides:"):
>   				i = provides
>   			else:
>   				continue
> +
> +			file = f.replace(pkgdest + "/" + pkg, "")
> +			file = file.replace("@", "@at@")
> +			file = file.replace(" ", "@space@")
> +			file = file.replace("\t", "@tab@")
> +			file = file.replace("[", "@openbrace@")
> +			file = file.replace("]", "@closebrace@")
> +			file = file.replace("_", "@underscore@")
>   			value = line.split(":", 1)[1].strip()
>   			value = r.sub(r'(\g<0>)', value)
> +
>   			if value.startswith("rpmlib("):
>   				continue
> -			i.append(value)
> +			if file not in i:
> +				i[file] = []
> +			i[file].append(value)
>
> -		if len(provides)>  0:
> +		for file in provides:
>   			provides_files.append(file)
>   			key = "FILERPROVIDES_" + file + "_" + pkg
> -			d.setVar(key, " ".join(provides))
> +			d.setVar(key, " ".join(provides[file]))
>
> -		if len(requires)>  0:
> +		for file in requires:
>   			requires_files.append(file)
>   			key = "FILERDEPENDS_" + file + "_" + pkg
> -			d.setVar(key, " ".join(requires))
> +			d.setVar(key, " ".join(requires[file]))
> +
> +	def chunks(files, n):
> +		return [files[i:i+n] for i in range(0, len(files), n)]
>
>   	# Determine dependencies
>   	for pkg in packages.split():
> @@ -1186,13 +1196,15 @@ python package_do_filedeps() {
>
>   		provides_files = []
>   		requires_files = []
> +		rpfiles = []
>   		for root, dirs, files in os.walk(pkgdest + "/" + pkg):
>   			for file in files:
> -				f = os.path.join(root, file)
> +				rpfiles.append(os.path.join(root, file))
>
> -				dep_pipe = os.popen(rpmdeps + " --provides --requires -v " + f)
> +		for files in chunks(rpfiles, 100):
> +			dep_pipe = os.popen(rpmdeps + " " + " ".join(files))
>
> -				process_deps(dep_pipe, pkg, f, provides_files, requires_files)
> +			process_deps(dep_pipe, pkg, provides_files, requires_files)
>
>   		d.setVar("FILERDEPENDSFLIST_" + pkg, " ".join(requires_files))
>   		d.setVar("FILERPROVIDESFLIST_" + pkg, " ".join(provides_files))
> diff --git a/meta/recipes-core/busybox/busybox.inc b/meta/recipes-core/busybox/busybox.inc
> index f35779d..5fb436e 100644
> --- a/meta/recipes-core/busybox/busybox.inc
> +++ b/meta/recipes-core/busybox/busybox.inc
> @@ -216,9 +216,9 @@ python package_do_filedeps_append () {
>   	# Load/backup original set
>   	filerprovides = d.getVar('FILERPROVIDES_%s_%s' % (f_busybox, pkg), True) or ""
>
> -	dep_pipe = os.popen('sed -e "s,^,Provides: ," %s/%s%s' % (pkgdest, pkg, f_busybox_links))
> +	dep_pipe = os.popen('sed -e "s,^,%s/%s%s Provides: ," %s/%s%s' % (pkgdest, pkg, f_busybox, pkgdest, pkg, f_busybox_links))
>
> -	process_deps(dep_pipe, pkg, "%s/%s%s" % (pkgdest, pkg, f_busybox), provides_files, requires_files)
> +	process_deps(dep_pipe, pkg, provides_files, requires_files)
>
>   	# Add the new set
>   	filerprovides += d.getVar('FILERPROVIDES_%s_%s' % (f_busybox, pkg), True) or ""
>
>
> _______________________________________________
> Openembedded-core mailing list
> Openembedded-core@lists.openembedded.org
> http://lists.linuxtogo.org/cgi-bin/mailman/listinfo/openembedded-core

On Thu, 2012-02-09 at 18:23 -0800, Joshua Lock wrote:
> On 09/02/12 16:10, Richard Purdie wrote:
> > Currently a process was being forked off for each individual file
> > this class wanted to inspect with rpmdeps. This converts it to use
> > rpmdeps-oecore which allows batch processing of these dependencies.
> >
> > For do_package for perl, this reduced the time by about 1 minute (33%).
> 
> Awesome! Sounds like this addresses #1718
> 
> http://bugzilla.pokylinux.org/show_bug.cgi?id=1718

It will certainly help with that issue, yes.

> > @@ -1136,7 +1136,7 @@ RPMDEPS = "${STAGING_LIBDIR_NATIVE}/rpm/bin/rpmdeps --macros ${STAGING_LIBDIR_NA
> >   #  FILERDEPENDS_filepath_pkg - per file dep
> >
> >   python package_do_filedeps() {
> > -	import os, re
> > +	import re, subprocess
> 
> Was this accidentally left in? I don't see any calls to subprocess?

It was indeed from a different experiment, well spotted. I've removed
that.

Cheers,

Richard