Patchwork [RFC] buildhistory: Record size of installed package not compressed archive

login
register
mail settings
Submitter Martin Jansa
Date June 30, 2013, 1:36 a.m.
Message ID <1372556176-25711-1-git-send-email-Martin.Jansa@gmail.com>
Download mbox | patch
Permalink /patch/52697/
State Accepted, archived
Headers show

Comments

Martin Jansa - June 30, 2013, 1:36 a.m.
* usually it's more important to know how much space will each
  package take on target device then size of compressed package

Signed-off-by: Martin Jansa <Martin.Jansa@gmail.com>
---
 meta/classes/buildhistory.bbclass | 11 ++++----
 scripts/oe-pkgdata-util           | 59 ++++++++++++++++++++++++++++++++++++---
 2 files changed, 61 insertions(+), 9 deletions(-)
Otavio Salvador - July 1, 2013, 4:53 p.m.
On Sat, Jun 29, 2013 at 10:36 PM, Martin Jansa <martin.jansa@gmail.com> wrote:
> * usually it's more important to know how much space will each
>   package take on target device then size of compressed package
>
> Signed-off-by: Martin Jansa <Martin.Jansa@gmail.com>

I fully agree; this is a more useful information indeed.

--
Otavio Salvador                             O.S. Systems
http://www.ossystems.com.br        http://projetos.ossystems.com.br
Mobile: +55 (53) 9981-7854            Mobile: +1 (347) 903-9750
Saul Wold - July 4, 2013, 6:53 a.m.
On 06/29/2013 06:36 PM, Martin Jansa wrote:
> * usually it's more important to know how much space will each
>    package take on target device then size of compressed package
>
> Signed-off-by: Martin Jansa <Martin.Jansa@gmail.com>
> ---
>   meta/classes/buildhistory.bbclass | 11 ++++----
>   scripts/oe-pkgdata-util           | 59 ++++++++++++++++++++++++++++++++++++---
>   2 files changed, 61 insertions(+), 9 deletions(-)
>

I am not sure how, but this patch is causing some failures in poky with
core-image-sato build, it seems that that the args coming from the 
rootfs_install_complementary get goofed up somehow, not sure but I 
bisected down to this patch.

the -poky seems to have a space added so it gets -poky -linux on the 
command line for oe-pkgdata-utils.

Sau!

> diff --git a/meta/classes/buildhistory.bbclass b/meta/classes/buildhistory.bbclass
> index 36e7fe1..1ebe68e 100644
> --- a/meta/classes/buildhistory.bbclass
> +++ b/meta/classes/buildhistory.bbclass
> @@ -317,6 +317,8 @@ buildhistory_get_installed() {
>   	# Get list of installed packages
>   	pkgcache="$1/installed-packages.tmp"
>   	list_installed_packages file | sort > $pkgcache
> +	pkgcachearch="$1/installed-packages-arch.tmp"
> +	list_installed_packages arch | sort > $pkgcachearch
>
>   	cat $pkgcache | awk '{ print $1 }' > $1/installed-package-names.txt
>   	if [ -s $pkgcache ] ; then
> @@ -338,18 +340,17 @@ buildhistory_get_installed() {
>
>   	# Produce installed package sizes list
>   	printf "" > $1/installed-package-sizes.tmp
> -	cat $pkgcache | while read pkg pkgfile
> +	cat $pkgcachearch | while read pkg arch
>   	do
> -		if [ -f $pkgfile ] ; then
> -			pkgsize=`du -k $pkgfile | head -n1 | awk '{ print $1 }'`
> -			echo $pkgsize $pkg >> $1/installed-package-sizes.tmp
> -		fi
> +		size=`oe-pkgdata-util read_values ${TMPDIR}/pkgdata ${TARGET_VENDOR}-${TARGET_OS} "PKGSIZE" ${pkg}_${arch}`
> +		echo "$size $pkg" >> $1/installed-package-sizes.tmp
>   	done
>   	cat $1/installed-package-sizes.tmp | sort -n -r | awk '{print $1 "\tKiB " $2}' > $1/installed-package-sizes.txt
>   	rm $1/installed-package-sizes.tmp
>
>   	# We're now done with the cache, delete it
>   	rm $pkgcache
> +	rm $pkgcachearch
>
>   	if [ "$2" != "sdk" ] ; then
>   		# Produce some cut-down graphs (for readability)
> diff --git a/scripts/oe-pkgdata-util b/scripts/oe-pkgdata-util
> index 629b2d5..88b8122 100755
> --- a/scripts/oe-pkgdata-util
> +++ b/scripts/oe-pkgdata-util
> @@ -20,9 +20,12 @@
>   # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
>   #
>   #
> -# Currently only has one function - mapping of packages to their dev/dbg/doc/locale etc.
> -# counterparts ("glob" command). Could be extended in future to perform other useful querying
> -# functions on the pkgdata though.
> +# Currently only has two functions
> +# 1) glob - mapping of packages to their dev/dbg/doc/locale etc. counterparts.
> +# 2) read_values - mapping of packagenames to their location in
> +#    pkgdata and then returns value of selected variable (e.g. PKGSIZE)
> +# Could be extended in future to perform other useful querying functions on the
> +# pkgdata though.
>   #
>
>   import sys
> @@ -32,7 +35,8 @@ import fnmatch
>   import re
>
>   def usage():
> -    print("syntax: pkgdata-util glob [-d] <pkgdatadir> <vendor-os> <pkglist> \"<globs>\"");
> +    print("syntax: pkgdata-util glob [-d] <pkgdatadir> <vendor-os> <pkglist> \"<globs>\"\n \
> +                    read_values [-d] <pkgdatadir> <vendor-os> <variable-name> \"<ipk filenames>\"");
>
>
>
> @@ -151,7 +155,52 @@ def glob(args):
>
>       print("\n".join(mappedpkgs))
>
> +def read_values(args):
> +    if len(args) < 4:
> +        usage()
> +        sys.exit(1)
> +
> +    pkgdata_dir = args[0]
> +    target_suffix = args[1]
> +    var = args[2]
> +    packages = args[3].split()
>
> +    if target_suffix.startswith("-"):
> +        target_suffix = target_suffix[1:]
> +
> +    def readvar(pkgdata_file, var):
> +        val = ""
> +        with open(pkgdata_file, 'r') as f:
> +            for line in f:
> +                if line.startswith(var + ":"):
> +                    val = line.split(': ')[1].rstrip()
> +        return val
> +
> +    if debug:
> +        print "read_values('%s', '%s', '%s' '%s'" % (pkgdata_dir, target_suffix, var, packages)
> +    for package in packages:
> +        pkg_split = package.split('_')
> +        pkg_name = pkg_split[0]
> +        pkg_arch = '_'.join(pkg_split[1:])
> +        if debug:
> +            print "package: name: '%s', arch: '%s'" % (pkg_name, pkg_arch)
> +        multimach_target_sys = "%s-%s" % (pkg_arch, target_suffix)
> +        revlink = os.path.join(pkgdata_dir, multimach_target_sys, "runtime-reverse", pkg_name)
> +        if debug:
> +            print(revlink)
> +        if not os.path.exists(revlink):
> +            # [YOCTO #4227] try to drop -gnueabi from TARGET_OS
> +            multimach_target_sys = '-'.join(multimach_target_sys.split('-')[:-1])
> +            revlink = os.path.join(pkgdata_dir, multimach_target_sys, "runtime-reverse", pkg_name)
> +            if debug:
> +                print(revlink)
> +        if os.path.exists(revlink):
> +            mappedpkg = os.path.basename(os.readlink(revlink))
> +            qvar = var
> +            if qvar == "PKGSIZE":
> +                # append packagename
> +                qvar = "%s_%s" % (var, mappedpkg)
> +            print(readvar(revlink, qvar))
>
>   # Too lazy to use getopt
>   debug = False
> @@ -173,6 +222,8 @@ if len(args) < 1:
>
>   if args[0] == "glob":
>       glob(args[1:])
> +if args[0] == "read_values":
> +    read_values(args[1:])
>   else:
>       usage()
>       sys.exit(1)
>
Martin Jansa - July 4, 2013, 8:30 a.m.
On Wed, Jul 03, 2013 at 11:53:06PM -0700, Saul Wold wrote:
> On 06/29/2013 06:36 PM, Martin Jansa wrote:
> > * usually it's more important to know how much space will each
> >    package take on target device then size of compressed package
> >
> > Signed-off-by: Martin Jansa <Martin.Jansa@gmail.com>
> > ---
> >   meta/classes/buildhistory.bbclass | 11 ++++----
> >   scripts/oe-pkgdata-util           | 59 ++++++++++++++++++++++++++++++++++++---
> >   2 files changed, 61 insertions(+), 9 deletions(-)
> >
> 
> I am not sure how, but this patch is causing some failures in poky with
> core-image-sato build, it seems that that the args coming from the 
> rootfs_install_complementary get goofed up somehow, not sure but I 
> bisected down to this patch.
> 
> the -poky seems to have a space added so it gets -poky -linux on the 
> command line for oe-pkgdata-utils.

I'm sorry I haven't replied to that patch yet, it was sent only as RFC
to see if people agree that we should just replace current sizes with
size when installed on target (I have different version which records
both, but is a bit slower and resulting file bigger without good
reasons)

I'm aware of the issue and will fix it soon.

> > diff --git a/meta/classes/buildhistory.bbclass b/meta/classes/buildhistory.bbclass
> > index 36e7fe1..1ebe68e 100644
> > --- a/meta/classes/buildhistory.bbclass
> > +++ b/meta/classes/buildhistory.bbclass
> > @@ -317,6 +317,8 @@ buildhistory_get_installed() {
> >   	# Get list of installed packages
> >   	pkgcache="$1/installed-packages.tmp"
> >   	list_installed_packages file | sort > $pkgcache
> > +	pkgcachearch="$1/installed-packages-arch.tmp"
> > +	list_installed_packages arch | sort > $pkgcachearch
> >
> >   	cat $pkgcache | awk '{ print $1 }' > $1/installed-package-names.txt
> >   	if [ -s $pkgcache ] ; then
> > @@ -338,18 +340,17 @@ buildhistory_get_installed() {
> >
> >   	# Produce installed package sizes list
> >   	printf "" > $1/installed-package-sizes.tmp
> > -	cat $pkgcache | while read pkg pkgfile
> > +	cat $pkgcachearch | while read pkg arch
> >   	do
> > -		if [ -f $pkgfile ] ; then
> > -			pkgsize=`du -k $pkgfile | head -n1 | awk '{ print $1 }'`
> > -			echo $pkgsize $pkg >> $1/installed-package-sizes.tmp
> > -		fi
> > +		size=`oe-pkgdata-util read_values ${TMPDIR}/pkgdata ${TARGET_VENDOR}-${TARGET_OS} "PKGSIZE" ${pkg}_${arch}`
> > +		echo "$size $pkg" >> $1/installed-package-sizes.tmp
> >   	done
> >   	cat $1/installed-package-sizes.tmp | sort -n -r | awk '{print $1 "\tKiB " $2}' > $1/installed-package-sizes.txt
> >   	rm $1/installed-package-sizes.tmp
> >
> >   	# We're now done with the cache, delete it
> >   	rm $pkgcache
> > +	rm $pkgcachearch
> >
> >   	if [ "$2" != "sdk" ] ; then
> >   		# Produce some cut-down graphs (for readability)
> > diff --git a/scripts/oe-pkgdata-util b/scripts/oe-pkgdata-util
> > index 629b2d5..88b8122 100755
> > --- a/scripts/oe-pkgdata-util
> > +++ b/scripts/oe-pkgdata-util
> > @@ -20,9 +20,12 @@
> >   # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
> >   #
> >   #
> > -# Currently only has one function - mapping of packages to their dev/dbg/doc/locale etc.
> > -# counterparts ("glob" command). Could be extended in future to perform other useful querying
> > -# functions on the pkgdata though.
> > +# Currently only has two functions
> > +# 1) glob - mapping of packages to their dev/dbg/doc/locale etc. counterparts.
> > +# 2) read_values - mapping of packagenames to their location in
> > +#    pkgdata and then returns value of selected variable (e.g. PKGSIZE)
> > +# Could be extended in future to perform other useful querying functions on the
> > +# pkgdata though.
> >   #
> >
> >   import sys
> > @@ -32,7 +35,8 @@ import fnmatch
> >   import re
> >
> >   def usage():
> > -    print("syntax: pkgdata-util glob [-d] <pkgdatadir> <vendor-os> <pkglist> \"<globs>\"");
> > +    print("syntax: pkgdata-util glob [-d] <pkgdatadir> <vendor-os> <pkglist> \"<globs>\"\n \
> > +                    read_values [-d] <pkgdatadir> <vendor-os> <variable-name> \"<ipk filenames>\"");
> >
> >
> >
> > @@ -151,7 +155,52 @@ def glob(args):
> >
> >       print("\n".join(mappedpkgs))
> >
> > +def read_values(args):
> > +    if len(args) < 4:
> > +        usage()
> > +        sys.exit(1)
> > +
> > +    pkgdata_dir = args[0]
> > +    target_suffix = args[1]
> > +    var = args[2]
> > +    packages = args[3].split()
> >
> > +    if target_suffix.startswith("-"):
> > +        target_suffix = target_suffix[1:]
> > +
> > +    def readvar(pkgdata_file, var):
> > +        val = ""
> > +        with open(pkgdata_file, 'r') as f:
> > +            for line in f:
> > +                if line.startswith(var + ":"):
> > +                    val = line.split(': ')[1].rstrip()
> > +        return val
> > +
> > +    if debug:
> > +        print "read_values('%s', '%s', '%s' '%s'" % (pkgdata_dir, target_suffix, var, packages)
> > +    for package in packages:
> > +        pkg_split = package.split('_')
> > +        pkg_name = pkg_split[0]
> > +        pkg_arch = '_'.join(pkg_split[1:])
> > +        if debug:
> > +            print "package: name: '%s', arch: '%s'" % (pkg_name, pkg_arch)
> > +        multimach_target_sys = "%s-%s" % (pkg_arch, target_suffix)
> > +        revlink = os.path.join(pkgdata_dir, multimach_target_sys, "runtime-reverse", pkg_name)
> > +        if debug:
> > +            print(revlink)
> > +        if not os.path.exists(revlink):
> > +            # [YOCTO #4227] try to drop -gnueabi from TARGET_OS
> > +            multimach_target_sys = '-'.join(multimach_target_sys.split('-')[:-1])
> > +            revlink = os.path.join(pkgdata_dir, multimach_target_sys, "runtime-reverse", pkg_name)
> > +            if debug:
> > +                print(revlink)
> > +        if os.path.exists(revlink):
> > +            mappedpkg = os.path.basename(os.readlink(revlink))
> > +            qvar = var
> > +            if qvar == "PKGSIZE":
> > +                # append packagename
> > +                qvar = "%s_%s" % (var, mappedpkg)
> > +            print(readvar(revlink, qvar))
> >
> >   # Too lazy to use getopt
> >   debug = False
> > @@ -173,6 +222,8 @@ if len(args) < 1:
> >
> >   if args[0] == "glob":
> >       glob(args[1:])
> > +if args[0] == "read_values":
> > +    read_values(args[1:])
> >   else:
> >       usage()
> >       sys.exit(1)
> >

Patch

diff --git a/meta/classes/buildhistory.bbclass b/meta/classes/buildhistory.bbclass
index 36e7fe1..1ebe68e 100644
--- a/meta/classes/buildhistory.bbclass
+++ b/meta/classes/buildhistory.bbclass
@@ -317,6 +317,8 @@  buildhistory_get_installed() {
 	# Get list of installed packages
 	pkgcache="$1/installed-packages.tmp"
 	list_installed_packages file | sort > $pkgcache
+	pkgcachearch="$1/installed-packages-arch.tmp"
+	list_installed_packages arch | sort > $pkgcachearch
 
 	cat $pkgcache | awk '{ print $1 }' > $1/installed-package-names.txt
 	if [ -s $pkgcache ] ; then
@@ -338,18 +340,17 @@  buildhistory_get_installed() {
 
 	# Produce installed package sizes list
 	printf "" > $1/installed-package-sizes.tmp
-	cat $pkgcache | while read pkg pkgfile
+	cat $pkgcachearch | while read pkg arch
 	do
-		if [ -f $pkgfile ] ; then
-			pkgsize=`du -k $pkgfile | head -n1 | awk '{ print $1 }'`
-			echo $pkgsize $pkg >> $1/installed-package-sizes.tmp
-		fi
+		size=`oe-pkgdata-util read_values ${TMPDIR}/pkgdata ${TARGET_VENDOR}-${TARGET_OS} "PKGSIZE" ${pkg}_${arch}`
+		echo "$size $pkg" >> $1/installed-package-sizes.tmp
 	done
 	cat $1/installed-package-sizes.tmp | sort -n -r | awk '{print $1 "\tKiB " $2}' > $1/installed-package-sizes.txt
 	rm $1/installed-package-sizes.tmp
 
 	# We're now done with the cache, delete it
 	rm $pkgcache
+	rm $pkgcachearch
 
 	if [ "$2" != "sdk" ] ; then
 		# Produce some cut-down graphs (for readability)
diff --git a/scripts/oe-pkgdata-util b/scripts/oe-pkgdata-util
index 629b2d5..88b8122 100755
--- a/scripts/oe-pkgdata-util
+++ b/scripts/oe-pkgdata-util
@@ -20,9 +20,12 @@ 
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 #
 #
-# Currently only has one function - mapping of packages to their dev/dbg/doc/locale etc. 
-# counterparts ("glob" command). Could be extended in future to perform other useful querying
-# functions on the pkgdata though.
+# Currently only has two functions 
+# 1) glob - mapping of packages to their dev/dbg/doc/locale etc. counterparts.
+# 2) read_values - mapping of packagenames to their location in
+#    pkgdata and then returns value of selected variable (e.g. PKGSIZE)
+# Could be extended in future to perform other useful querying functions on the
+# pkgdata though.
 #
 
 import sys
@@ -32,7 +35,8 @@  import fnmatch
 import re
 
 def usage():
-    print("syntax: pkgdata-util glob [-d] <pkgdatadir> <vendor-os> <pkglist> \"<globs>\"");
+    print("syntax: pkgdata-util glob [-d] <pkgdatadir> <vendor-os> <pkglist> \"<globs>\"\n \
+                    read_values [-d] <pkgdatadir> <vendor-os> <variable-name> \"<ipk filenames>\"");
 
 
 
@@ -151,7 +155,52 @@  def glob(args):
 
     print("\n".join(mappedpkgs))
 
+def read_values(args):
+    if len(args) < 4:
+        usage()
+        sys.exit(1)
+
+    pkgdata_dir = args[0]
+    target_suffix = args[1]
+    var = args[2]
+    packages = args[3].split()
 
+    if target_suffix.startswith("-"):
+        target_suffix = target_suffix[1:]
+
+    def readvar(pkgdata_file, var):
+        val = ""
+        with open(pkgdata_file, 'r') as f:
+            for line in f:
+                if line.startswith(var + ":"):
+                    val = line.split(': ')[1].rstrip()
+        return val
+
+    if debug:
+        print "read_values('%s', '%s', '%s' '%s'" % (pkgdata_dir, target_suffix, var, packages)
+    for package in packages:
+        pkg_split = package.split('_')
+        pkg_name = pkg_split[0]
+        pkg_arch = '_'.join(pkg_split[1:])
+        if debug:
+            print "package: name: '%s', arch: '%s'" % (pkg_name, pkg_arch)
+        multimach_target_sys = "%s-%s" % (pkg_arch, target_suffix)
+        revlink = os.path.join(pkgdata_dir, multimach_target_sys, "runtime-reverse", pkg_name)
+        if debug:
+            print(revlink)
+        if not os.path.exists(revlink):
+            # [YOCTO #4227] try to drop -gnueabi from TARGET_OS
+            multimach_target_sys = '-'.join(multimach_target_sys.split('-')[:-1])
+            revlink = os.path.join(pkgdata_dir, multimach_target_sys, "runtime-reverse", pkg_name)
+            if debug:
+                print(revlink)
+        if os.path.exists(revlink):
+            mappedpkg = os.path.basename(os.readlink(revlink))
+            qvar = var
+            if qvar == "PKGSIZE":
+                # append packagename
+                qvar = "%s_%s" % (var, mappedpkg)
+            print(readvar(revlink, qvar))
 
 # Too lazy to use getopt
 debug = False
@@ -173,6 +222,8 @@  if len(args) < 1:
 
 if args[0] == "glob":
     glob(args[1:])
+if args[0] == "read_values":
+    read_values(args[1:])
 else:
     usage()
     sys.exit(1)