Patchwork gitpkgv.bbclass: cache GITPKGV result

login
register
mail settings
Submitter Enrico Scholz
Date April 26, 2011, 10:45 a.m.
Message ID <1303814757-9940-1-git-send-email-enrico.scholz@sigma-chemnitz.de>
Download mbox | patch
Permalink /patch/2879/
State New, archived
Headers show

Comments

Enrico Scholz - April 26, 2011, 10:45 a.m.
gitpkgv runs the 'git rev-list | wc -l' several times when processing a
package using GITPKGV.  This takes ages for packages like the linux
kernel which has a) a large repository and b) lots of subpackages.

This patch caches the result of 'git rev-list' into the sources cache
directory and uses it on the next run.  Because collisions of the sha1
hash are very unlikely, the git revision is used directly as the key.

Signed-off-by: Enrico Scholz <enrico.scholz@sigma-chemnitz.de>
---
 classes/gitpkgv.bbclass |   29 +++++++++++++++++++++++------
 1 files changed, 23 insertions(+), 6 deletions(-)
Otavio Salvador - April 26, 2011, 4:55 p.m.
Acked-by: Otavio Salvador <otavio@ossystems.com.br>

On Tue, Apr 26, 2011 at 07:45, Enrico Scholz
<enrico.scholz@sigma-chemnitz.de> wrote:
> gitpkgv runs the 'git rev-list | wc -l' several times when processing a
> package using GITPKGV.  This takes ages for packages like the linux
> kernel which has a) a large repository and b) lots of subpackages.
>
> This patch caches the result of 'git rev-list' into the sources cache
> directory and uses it on the next run.  Because collisions of the sha1
> hash are very unlikely, the git revision is used directly as the key.
>
> Signed-off-by: Enrico Scholz <enrico.scholz@sigma-chemnitz.de>
> ---
>  classes/gitpkgv.bbclass |   29 +++++++++++++++++++++++------
>  1 files changed, 23 insertions(+), 6 deletions(-)
>
> diff --git a/classes/gitpkgv.bbclass b/classes/gitpkgv.bbclass
> index bedceb9..76c660b 100644
> --- a/classes/gitpkgv.bbclass
> +++ b/classes/gitpkgv.bbclass
> @@ -50,8 +50,11 @@ def gitpkgv_drop_tag_prefix(version):
>  def get_git_pkgv(d, use_tags):
>        import os
>        import bb
> +       from pipes import quote
>
>        urls = bb.data.getVar('SRC_URI', d, 1).split()
> +       cachedir = bb.data.expand("${DL_DIR}/gitpkgv", d)
> +       bb.mkdirhier(cachedir)
>
>        for url in urls:
>                (type, host, path, user, pswd, parm) = bb.decodeurl(bb.data.expand(url, d))
> @@ -63,22 +66,36 @@ def get_git_pkgv(d, use_tags):
>                                return None
>
>                        rev = bb.fetch.get_srcrev(d).split('+')[1]
> +                       rev_file = os.path.join(cachedir, rev)
>
> -                       cwd = os.getcwd()
> -                       os.chdir(repodir)
> +                       vars = { 'repodir' : quote(repodir),
> +                                'rev' : quote(rev),
> +                              }
>
> -                       commits = bb.fetch.runfetchcmd("git rev-list %s -- 2> /dev/null | wc -l" % rev, d, quiet=True).strip()
> +                       if not os.path.exists(rev_file) or os.path.getsize(rev_file)==0:
> +                               commits = bb.fetch.runfetchcmd(
> +                                      "cd %(repodir)s && "
> +                                      "git rev-list %(rev)s -- 2> /dev/null | wc -l" % vars,
> +                                      d, quiet=True).strip()
> +
> +                               if commits != "0":
> +                                       oe.path.remove(rev_file, recurse=False)
> +                                       open(rev_file, "w").write("%d\n" % commits)
> +                       else:
> +                               commits = open(rev_file, "r").readline(128).strip()
>
>                        if use_tags:
>                                try:
> -                                       ver = gitpkgv_drop_tag_prefix(bb.fetch.runfetchcmd("git describe %s 2>/dev/null" % rev, d, quiet=True).strip())
> +                                       tmp = bb.fetch.runfetchcmd(
> +                                       "cd %(repodir)s && "
> +                                       "git describe %(rev)s 2>/dev/null" % vars,
> +                                       d, quiet=True).strip()
> +                                       ver = gitpkgv_drop_tag_prefix(tmp)
>                                except Exception:
>                                        ver = "0.0-%s-g%s" % (commits, rev[:7])
>                        else:
>                                ver = "%s+%s" % (commits, rev[:7])
>
> -                       os.chdir(cwd)
> -
>                        return ver
>
>        return "0+0"
> --
> 1.7.4.4
>
>
> _______________________________________________
> Openembedded-devel mailing list
> Openembedded-devel@lists.openembedded.org
> http://lists.linuxtogo.org/cgi-bin/mailman/listinfo/openembedded-devel
>

Patch

diff --git a/classes/gitpkgv.bbclass b/classes/gitpkgv.bbclass
index bedceb9..76c660b 100644
--- a/classes/gitpkgv.bbclass
+++ b/classes/gitpkgv.bbclass
@@ -50,8 +50,11 @@  def gitpkgv_drop_tag_prefix(version):
 def get_git_pkgv(d, use_tags):
 	import os
 	import bb
+	from pipes import quote
 
 	urls = bb.data.getVar('SRC_URI', d, 1).split()
+	cachedir = bb.data.expand("${DL_DIR}/gitpkgv", d)
+	bb.mkdirhier(cachedir)
 
 	for url in urls:
 		(type, host, path, user, pswd, parm) = bb.decodeurl(bb.data.expand(url, d))
@@ -63,22 +66,36 @@  def get_git_pkgv(d, use_tags):
 				return None
 
 			rev = bb.fetch.get_srcrev(d).split('+')[1]
+			rev_file = os.path.join(cachedir, rev)
 
-			cwd = os.getcwd()
-			os.chdir(repodir)
+			vars = { 'repodir' : quote(repodir),
+				 'rev' : quote(rev),
+			       }
 
-			commits = bb.fetch.runfetchcmd("git rev-list %s -- 2> /dev/null | wc -l" % rev, d, quiet=True).strip()
+			if not os.path.exists(rev_file) or os.path.getsize(rev_file)==0:
+				commits = bb.fetch.runfetchcmd(
+				       "cd %(repodir)s && "
+				       "git rev-list %(rev)s -- 2> /dev/null | wc -l" % vars,
+				       d, quiet=True).strip()
+
+				if commits != "0":
+					oe.path.remove(rev_file, recurse=False)
+					open(rev_file, "w").write("%d\n" % commits)
+			else:
+				commits = open(rev_file, "r").readline(128).strip()
 
 			if use_tags:
 				try:
-					ver = gitpkgv_drop_tag_prefix(bb.fetch.runfetchcmd("git describe %s 2>/dev/null" % rev, d, quiet=True).strip())
+					tmp = bb.fetch.runfetchcmd(
+					"cd %(repodir)s && "
+					"git describe %(rev)s 2>/dev/null" % vars,
+					d, quiet=True).strip()
+					ver = gitpkgv_drop_tag_prefix(tmp)
 				except Exception:
 					ver = "0.0-%s-g%s" % (commits, rev[:7])
 			else:
 				ver = "%s+%s" % (commits, rev[:7])
 
-			os.chdir(cwd)
-
 			return ver
 
 	return "0+0"