gitpkgv.bbclass: cache GITPKGV result

Submitted by Enrico Scholz on April 26, 2011, 10:45 a.m.

Details

Message ID 1303814757-9940-1-git-send-email-enrico.scholz@sigma-chemnitz.de
State New, archived
Headers show

Commit Message

Enrico Scholz April 26, 2011, 10:45 a.m.
gitpkgv runs the 'git rev-list | wc -l' several times when processing a
package using GITPKGV.  This takes ages for packages like the linux
kernel which has a) a large repository and b) lots of subpackages.

This patch caches the result of 'git rev-list' into the sources cache
directory and uses it on the next run.  Because collisions of the sha1
hash are very unlikely, the git revision is used directly as the key.

Signed-off-by: Enrico Scholz <enrico.scholz@sigma-chemnitz.de>
---
 classes/gitpkgv.bbclass |   29 +++++++++++++++++++++++------
 1 files changed, 23 insertions(+), 6 deletions(-)

Patch hide | download patch | download mbox

diff --git a/classes/gitpkgv.bbclass b/classes/gitpkgv.bbclass
index bedceb9..76c660b 100644
--- a/classes/gitpkgv.bbclass
+++ b/classes/gitpkgv.bbclass
@@ -50,8 +50,11 @@  def gitpkgv_drop_tag_prefix(version):
 def get_git_pkgv(d, use_tags):
 	import os
 	import bb
+	from pipes import quote
 
 	urls = bb.data.getVar('SRC_URI', d, 1).split()
+	cachedir = bb.data.expand("${DL_DIR}/gitpkgv", d)
+	bb.mkdirhier(cachedir)
 
 	for url in urls:
 		(type, host, path, user, pswd, parm) = bb.decodeurl(bb.data.expand(url, d))
@@ -63,22 +66,36 @@  def get_git_pkgv(d, use_tags):
 				return None
 
 			rev = bb.fetch.get_srcrev(d).split('+')[1]
+			rev_file = os.path.join(cachedir, rev)
 
-			cwd = os.getcwd()
-			os.chdir(repodir)
+			vars = { 'repodir' : quote(repodir),
+				 'rev' : quote(rev),
+			       }
 
-			commits = bb.fetch.runfetchcmd("git rev-list %s -- 2> /dev/null | wc -l" % rev, d, quiet=True).strip()
+			if not os.path.exists(rev_file) or os.path.getsize(rev_file)==0:
+				commits = bb.fetch.runfetchcmd(
+				       "cd %(repodir)s && "
+				       "git rev-list %(rev)s -- 2> /dev/null | wc -l" % vars,
+				       d, quiet=True).strip()
+
+				if commits != "0":
+					oe.path.remove(rev_file, recurse=False)
+					open(rev_file, "w").write("%d\n" % commits)
+			else:
+				commits = open(rev_file, "r").readline(128).strip()
 
 			if use_tags:
 				try:
-					ver = gitpkgv_drop_tag_prefix(bb.fetch.runfetchcmd("git describe %s 2>/dev/null" % rev, d, quiet=True).strip())
+					tmp = bb.fetch.runfetchcmd(
+					"cd %(repodir)s && "
+					"git describe %(rev)s 2>/dev/null" % vars,
+					d, quiet=True).strip()
+					ver = gitpkgv_drop_tag_prefix(tmp)
 				except Exception:
 					ver = "0.0-%s-g%s" % (commits, rev[:7])
 			else:
 				ver = "%s+%s" % (commits, rev[:7])
 
-			os.chdir(cwd)
-
 			return ver
 
 	return "0+0"

Comments

Otavio Salvador April 26, 2011, 4:55 p.m.
Acked-by: Otavio Salvador <otavio@ossystems.com.br>

On Tue, Apr 26, 2011 at 07:45, Enrico Scholz
<enrico.scholz@sigma-chemnitz.de> wrote:
> gitpkgv runs the 'git rev-list | wc -l' several times when processing a
> package using GITPKGV.  This takes ages for packages like the linux
> kernel which has a) a large repository and b) lots of subpackages.
>
> This patch caches the result of 'git rev-list' into the sources cache
> directory and uses it on the next run.  Because collisions of the sha1
> hash are very unlikely, the git revision is used directly as the key.
>
> Signed-off-by: Enrico Scholz <enrico.scholz@sigma-chemnitz.de>
> ---
>  classes/gitpkgv.bbclass |   29 +++++++++++++++++++++++------
>  1 files changed, 23 insertions(+), 6 deletions(-)
>
> diff --git a/classes/gitpkgv.bbclass b/classes/gitpkgv.bbclass
> index bedceb9..76c660b 100644
> --- a/classes/gitpkgv.bbclass
> +++ b/classes/gitpkgv.bbclass
> @@ -50,8 +50,11 @@ def gitpkgv_drop_tag_prefix(version):
>  def get_git_pkgv(d, use_tags):
>        import os
>        import bb
> +       from pipes import quote
>
>        urls = bb.data.getVar('SRC_URI', d, 1).split()
> +       cachedir = bb.data.expand("${DL_DIR}/gitpkgv", d)
> +       bb.mkdirhier(cachedir)
>
>        for url in urls:
>                (type, host, path, user, pswd, parm) = bb.decodeurl(bb.data.expand(url, d))
> @@ -63,22 +66,36 @@ def get_git_pkgv(d, use_tags):
>                                return None
>
>                        rev = bb.fetch.get_srcrev(d).split('+')[1]
> +                       rev_file = os.path.join(cachedir, rev)
>
> -                       cwd = os.getcwd()
> -                       os.chdir(repodir)
> +                       vars = { 'repodir' : quote(repodir),
> +                                'rev' : quote(rev),
> +                              }
>
> -                       commits = bb.fetch.runfetchcmd("git rev-list %s -- 2> /dev/null | wc -l" % rev, d, quiet=True).strip()
> +                       if not os.path.exists(rev_file) or os.path.getsize(rev_file)==0:
> +                               commits = bb.fetch.runfetchcmd(
> +                                      "cd %(repodir)s && "
> +                                      "git rev-list %(rev)s -- 2> /dev/null | wc -l" % vars,
> +                                      d, quiet=True).strip()
> +
> +                               if commits != "0":
> +                                       oe.path.remove(rev_file, recurse=False)
> +                                       open(rev_file, "w").write("%d\n" % commits)
> +                       else:
> +                               commits = open(rev_file, "r").readline(128).strip()
>
>                        if use_tags:
>                                try:
> -                                       ver = gitpkgv_drop_tag_prefix(bb.fetch.runfetchcmd("git describe %s 2>/dev/null" % rev, d, quiet=True).strip())
> +                                       tmp = bb.fetch.runfetchcmd(
> +                                       "cd %(repodir)s && "
> +                                       "git describe %(rev)s 2>/dev/null" % vars,
> +                                       d, quiet=True).strip()
> +                                       ver = gitpkgv_drop_tag_prefix(tmp)
>                                except Exception:
>                                        ver = "0.0-%s-g%s" % (commits, rev[:7])
>                        else:
>                                ver = "%s+%s" % (commits, rev[:7])
>
> -                       os.chdir(cwd)
> -
>                        return ver
>
>        return "0+0"
> --
> 1.7.4.4
>
>
> _______________________________________________
> Openembedded-devel mailing list
> Openembedded-devel@lists.openembedded.org
> http://lists.linuxtogo.org/cgi-bin/mailman/listinfo/openembedded-devel
>