[bitbake-devel] bb.fetch2: added support for gclient

Submitted by Zoltan Kuscsik on Nov. 3, 2015, 4:10 p.m. | Patch ID: 106639

Details

Message ID 1446567045-31179-1-git-send-email-zoltan.kuscsik@linaro.org
State New
Headers show

Commit Message

Zoltan Kuscsik Nov. 3, 2015, 4:10 p.m.
Gclient is tool for updating source code from
multiple SCM repository locations.

Signed-off-by: Zoltan Kuscsik <zoltan.kuscsik@linaro.org>
---
 lib/bb/fetch2/__init__.py |   3 +-
 lib/bb/fetch2/gclient.py  | 151 ++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 153 insertions(+), 1 deletion(-)
 create mode 100644 lib/bb/fetch2/gclient.py

Patch hide | download patch | download mbox

diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py
index 288a1c8..5ca3bc7 100644
--- a/lib/bb/fetch2/__init__.py
+++ b/lib/bb/fetch2/__init__.py
@@ -1529,7 +1529,6 @@  class Fetch(object):
         fn = d.getVar('FILE', True)
         if cache and fn and fn in urldata_cache:
             self.ud = urldata_cache[fn]
-
         for url in urls:
             if url not in self.ud:
                 try:
@@ -1775,6 +1774,7 @@  from . import hg
 from . import osc
 from . import repo
 from . import clearcase
+from . import gclient
 
 methods.append(local.Local())
 methods.append(wget.Wget())
@@ -1791,3 +1791,4 @@  methods.append(hg.Hg())
 methods.append(osc.Osc())
 methods.append(repo.Repo())
 methods.append(clearcase.ClearCase())
+methods.append(gclient.Gclient())
diff --git a/lib/bb/fetch2/gclient.py b/lib/bb/fetch2/gclient.py
new file mode 100644
index 0000000..de06eb5
--- /dev/null
+++ b/lib/bb/fetch2/gclient.py
@@ -0,0 +1,151 @@ 
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+"""
+BitBake "Fetch" gclient implementation
+
+gclient fetcher support the SRCU_URI in the following format:
+
+    SRC_URI = "gclient://some.host/path/;optionA=xxx,optionB=xy"
+
+Supported SRC_UIR options are:
+
+- branch
+  The git branch of the root gclient project. The default is
+  "master".
+
+- tag
+  Git tag for gclient to retrieve.
+
+- protocol
+  The method to use for accessing the gclient core project. The default
+   is https.
+
+- project-name
+  Passing the --name command line argument to gclient. Not default is empty.
+
+- rev
+  Specify git commit id for gclient. Default is empty.
+
+- deps-file
+  Set the --deps-file command line argument for gclient.
+
+"""
+
+# Copyright (C) 2015 Zoltan Kuscsik <zoltan.kuscsik@linaro.org>
+#
+# Based on git.py which is:
+# Copyright (C) 2005 Richard Purdie
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import os
+import bb
+from   bb    import data
+from   bb.fetch2 import FetchMethod
+from   bb.fetch2 import runfetchcmd
+import string
+
+class Gclient(FetchMethod):
+    """Class to fetch a module or modules using gclient"""
+
+    def __fetch_url(self, ud):
+        return "%s://%s%s" % (ud.proto, ud.host, ud.path)
+
+    def supports(self, ud, d):
+        """
+        Check to see if a given url can be fetched with repo.
+        """
+        return ud.type in ["gclient"]
+
+    def urldata_init(self, ud, d):
+        """
+        Init gclient
+        """
+        ud.proto = ud.parm.get('protocol', 'https')
+
+        ud.branch = ud.parm.get('branch', 'master')
+        ud.tag = ud.parm.get('tag','')
+        ud.rev = ud.parm.get('rev','')
+        
+        ud.deps_file = ud.parm.get('deps-file', '')
+        ud.project_name = ud.parm.get('project-name', '')
+
+        if( ( ('tag' in ud.parm) + ('branch' in ud.parm) + ('res' in ud.parm)) > 1):
+            raise bb.fetch2.ParameterError("Confliting revision setting SRC_URI. Setting only one of the options: 'branch', 'tag', 'res'.", ud.url)
+
+        dldir = d.getVar("DL_DIR", True) 
+    
+        gclient_src_name = 'gclient_%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.'))
+
+        ud.syncdir = os.path.join(dldir, gclient_src_name)
+        ud.localfile = ud.syncdir
+
+        if ud.tag:
+            ud.revision =  "refs/tags/%s" % ud.tag
+        elif ud.rev:
+            ud.revision = ud.rev
+        else:
+            ud.revision = "refs/heads/%s" % ud.branch
+
+    def download(self, ud, d):
+        """Fetch url"""
+
+        if not os.path.exists(ud.localpath):
+            bb.utils.mkdirhier(ud.localpath)
+        os.chdir(ud.localpath)
+        gclient_url = self.__fetch_url(ud)
+
+
+        # Add optional parameters
+        extra_params=[]
+        if(ud.deps_file):
+            extra_params.append("--deps-file=%s" % ud.deps_file)
+
+        if(ud.project_name):
+            extra_params.append("--name=%s" % ud.project_name)
+
+        extra_args = string.join(extra_params, sep = " ")
+
+        runfetchcmd("gclient config %s %s " % (gclient_url, extra_args), d)
+
+        # Gclient parameters used
+        # -f                    - force sync
+        # -nohooks              - skip hooks execution at this time
+        # --with_branch_heads   - gclient can't find a commit ID if it is not in a branch
+        #
+        # Note: We should enable --no-history after some more testing 
+
+        runfetchcmd("gclient sync --with_branch_heads -f  --nohooks --revision %s" % ud.revision, d)
+
+    def unpack(self, ud, destdir, d):
+        """ Unpack the src to destdir """
+        destsuffix = "gclient/"
+        destdir = ud.destdir = os.path.join(destdir, destsuffix)
+        # Use rsync for copying. Creating a tar package here would be very
+        # time consuming.
+        runfetchcmd("rsync -av %s/ %s/" %(ud.syncdir, destdir), d)
+        return True
+
+    def need_update(self, ud, d):
+        return True
+
+    def clean(self, ud, d):
+        """ Clean the gclient directory """
+        bb.utils.remove(ud.localpath, True)
+
+    def localpath(self, ud, dr):
+        return ud.syncdir
+
+    def supports_srcrev(self):
+        return False

Comments

Peter Kjellerstedt Nov. 4, 2015, 12:36 a.m.
> -----Original Message-----
> From: bitbake-devel-bounces@lists.openembedded.org [mailto:bitbake-
> devel-bounces@lists.openembedded.org] On Behalf Of Zoltan Kuscsik
> Sent: den 3 november 2015 17:11
> To: bitbake-devel@lists.openembedded.org
> Subject: [bitbake-devel] [PATCH] bb.fetch2: added support for gclient
> 
> Gclient is tool for updating source code from
> multiple SCM repository locations.
> 
> Signed-off-by: Zoltan Kuscsik <zoltan.kuscsik@linaro.org>
> ---
>  lib/bb/fetch2/__init__.py |   3 +-
>  lib/bb/fetch2/gclient.py  | 151
> ++++++++++++++++++++++++++++++++++++++++++++++
>  2 files changed, 153 insertions(+), 1 deletion(-)
>  create mode 100644 lib/bb/fetch2/gclient.py
> 
> diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py
> index 288a1c8..5ca3bc7 100644
> --- a/lib/bb/fetch2/__init__.py
> +++ b/lib/bb/fetch2/__init__.py
> @@ -1529,7 +1529,6 @@ class Fetch(object):
>          fn = d.getVar('FILE', True)
>          if cache and fn and fn in urldata_cache:
>              self.ud = urldata_cache[fn]
> -

Restore the empty line.

>          for url in urls:
>              if url not in self.ud:
>                  try:
> @@ -1775,6 +1774,7 @@ from . import hg
>  from . import osc
>  from . import repo
>  from . import clearcase
> +from . import gclient
> 
>  methods.append(local.Local())
>  methods.append(wget.Wget())
> @@ -1791,3 +1791,4 @@ methods.append(hg.Hg())
>  methods.append(osc.Osc())
>  methods.append(repo.Repo())
>  methods.append(clearcase.ClearCase())
> +methods.append(gclient.Gclient())
> diff --git a/lib/bb/fetch2/gclient.py b/lib/bb/fetch2/gclient.py
> new file mode 100644
> index 0000000..de06eb5
> --- /dev/null
> +++ b/lib/bb/fetch2/gclient.py
> @@ -0,0 +1,151 @@
> +# ex:ts=4:sw=4:sts=4:et
> +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
> +"""
> +BitBake "Fetch" gclient implementation
> +
> +gclient fetcher support the SRCU_URI in the following format:
                               ^^^^^^^^
Correct the spelling of SRC_URI.

> +
> +    SRC_URI = "gclient://some.host/path/;optionA=xxx,optionB=xy"
> +
> +Supported SRC_UIR options are:
             ^^^^^^^
Correct the spelling of SRC_URI.

> +
> +- branch
> +  The git branch of the root gclient project. The default is
> +  "master".
> +
> +- tag
> +  Git tag for gclient to retrieve.
> +
> +- protocol
> +  The method to use for accessing the gclient core project. The default
> +   is https.
> +
> +- project-name
> +  Passing the --name command line argument to gclient. Not default is empty.

Change "Passing" to "Pass" and "Not" to "The".

> +
> +- rev
> +  Specify git commit id for gclient. Default is empty.
> +
> +- deps-file
> +  Set the --deps-file command line argument for gclient.
> +
> +"""
> +
> +# Copyright (C) 2015 Zoltan Kuscsik <zoltan.kuscsik@linaro.org>
> +#
> +# Based on git.py which is:
> +# Copyright (C) 2005 Richard Purdie
> +#
> +# This program is free software; you can redistribute it and/or modify
> +# it under the terms of the GNU General Public License version 2 as
> +# published by the Free Software Foundation.
> +#
> +# This program is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> +# GNU General Public License for more details.
> +#
> +# You should have received a copy of the GNU General Public License along
> +# with this program; if not, write to the Free Software Foundation, Inc.,
> +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
> +
> +import os
> +import bb
> +from   bb    import data
> +from   bb.fetch2 import FetchMethod
> +from   bb.fetch2 import runfetchcmd
> +import string
> +
> +class Gclient(FetchMethod):
> +    """Class to fetch a module or modules using gclient"""
> +
> +    def __fetch_url(self, ud):
> +        return "%s://%s%s" % (ud.proto, ud.host, ud.path)
> +
> +    def supports(self, ud, d):
> +        """
> +        Check to see if a given url can be fetched with repo.
                                                           ^^^^
Repo? I thought this was for gclient...

> +        """
> +        return ud.type in ["gclient"]
> +
> +    def urldata_init(self, ud, d):
> +        """
> +        Init gclient
> +        """
> +        ud.proto = ud.parm.get('protocol', 'https')
> +
> +        ud.branch = ud.parm.get('branch', 'master')
> +        ud.tag = ud.parm.get('tag','')
> +        ud.rev = ud.parm.get('rev','')
> +
> +        ud.deps_file = ud.parm.get('deps-file', '')
> +        ud.project_name = ud.parm.get('project-name', '')
> +
> +        if( ( ('tag' in ud.parm) + ('branch' in ud.parm) + ('res' in ud.parm)) > 1):

Rewrite as:

        if ('tag' in ud.parm) + ('branch' in ud.parm) + ('res' in ud.parm) > 1:

> +            raise bb.fetch2.ParameterError("Confliting revision setting SRC_URI. Setting only one of the options: 'branch', 'tag', 'res'.", ud.url)
                                               ^^^^^^^^^^
Correct the spelling of "Conflicting". Change "revision setting SRC_URI" 
to "revisions specified in SRC_URI" and "Setting only" to "Specify only".

> +
> +        dldir = d.getVar("DL_DIR", True)
> +
> +        gclient_src_name = 'gclient_%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.'))
> +
> +        ud.syncdir = os.path.join(dldir, gclient_src_name)
> +        ud.localfile = ud.syncdir
> +
> +        if ud.tag:
> +            ud.revision =  "refs/tags/%s" % ud.tag
                             ^
Remove unnecessary space.

> +        elif ud.rev:
> +            ud.revision = ud.rev
> +        else:
> +            ud.revision = "refs/heads/%s" % ud.branch
> +
> +    def download(self, ud, d):
> +        """Fetch url"""
> +
> +        if not os.path.exists(ud.localpath):
> +            bb.utils.mkdirhier(ud.localpath)
> +        os.chdir(ud.localpath)
> +        gclient_url = self.__fetch_url(ud)
> +
> +
> +        # Add optional parameters
> +        extra_params=[]
> +        if(ud.deps_file):

Rewrite as:

        if ud.deps_file:

> +            extra_params.append("--deps-file=%s" % ud.deps_file)
> +
> +        if(ud.project_name):

Rewrite as:

        if ud.project_name:

> +            extra_params.append("--name=%s" % ud.project_name)
> +
> +        extra_args = string.join(extra_params, sep = " ")
> +
> +        runfetchcmd("gclient config %s %s " % (gclient_url, extra_args), d)
> +
> +        # Gclient parameters used
> +        # -f                    - force sync
> +        # -nohooks              - skip hooks execution at this time
> +        # --with_branch_heads   - gclient can't find a commit ID if it is not in a branch
> +        #
> +        # Note: We should enable --no-history after some more testing
> +
> +        runfetchcmd("gclient sync --with_branch_heads -f  --nohooks --revision %s" % ud.revision, d)
> +
> +    def unpack(self, ud, destdir, d):
> +        """ Unpack the src to destdir """
> +        destsuffix = "gclient/"
> +        destdir = ud.destdir = os.path.join(destdir, destsuffix)
> +        # Use rsync for copying. Creating a tar package here would be very
> +        # time consuming.
> +        runfetchcmd("rsync -av %s/ %s/" %(ud.syncdir, destdir), d)
> +        return True
> +
> +    def need_update(self, ud, d):
> +        return True
> +
> +    def clean(self, ud, d):
> +        """ Clean the gclient directory """
> +        bb.utils.remove(ud.localpath, True)
> +
> +    def localpath(self, ud, dr):
> +        return ud.syncdir
> +
> +    def supports_srcrev(self):
> +        return False
> --
> 1.9.1

//Peter
Zoltan Kuscsik Nov. 4, 2015, 4:07 a.m.
Thanks Peter, all comments addressed in PATCHv2.
I also created a test for the patch:

https://github.com/kuscsik/meta-testing/tree/master/recipes-core/fetcher

Br,

Z

On 4 November 2015 at 01:36, Peter Kjellerstedt <peter.kjellerstedt@axis.com
> wrote:

> > -----Original Message-----
> > From: bitbake-devel-bounces@lists.openembedded.org [mailto:bitbake-
> > devel-bounces@lists.openembedded.org] On Behalf Of Zoltan Kuscsik
> > Sent: den 3 november 2015 17:11
> > To: bitbake-devel@lists.openembedded.org
> > Subject: [bitbake-devel] [PATCH] bb.fetch2: added support for gclient
> >
> > Gclient is tool for updating source code from
> > multiple SCM repository locations.
> >
> > Signed-off-by: Zoltan Kuscsik <zoltan.kuscsik@linaro.org>
> > ---
> >  lib/bb/fetch2/__init__.py |   3 +-
> >  lib/bb/fetch2/gclient.py  | 151
> > ++++++++++++++++++++++++++++++++++++++++++++++
> >  2 files changed, 153 insertions(+), 1 deletion(-)
> >  create mode 100644 lib/bb/fetch2/gclient.py
> >
> > diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py
> > index 288a1c8..5ca3bc7 100644
> > --- a/lib/bb/fetch2/__init__.py
> > +++ b/lib/bb/fetch2/__init__.py
> > @@ -1529,7 +1529,6 @@ class Fetch(object):
> >          fn = d.getVar('FILE', True)
> >          if cache and fn and fn in urldata_cache:
> >              self.ud = urldata_cache[fn]
> > -
>
> Restore the empty line.
>
> >          for url in urls:
> >              if url not in self.ud:
> >                  try:
> > @@ -1775,6 +1774,7 @@ from . import hg
> >  from . import osc
> >  from . import repo
> >  from . import clearcase
> > +from . import gclient
> >
> >  methods.append(local.Local())
> >  methods.append(wget.Wget())
> > @@ -1791,3 +1791,4 @@ methods.append(hg.Hg())
> >  methods.append(osc.Osc())
> >  methods.append(repo.Repo())
> >  methods.append(clearcase.ClearCase())
> > +methods.append(gclient.Gclient())
> > diff --git a/lib/bb/fetch2/gclient.py b/lib/bb/fetch2/gclient.py
> > new file mode 100644
> > index 0000000..de06eb5
> > --- /dev/null
> > +++ b/lib/bb/fetch2/gclient.py
> > @@ -0,0 +1,151 @@
> > +# ex:ts=4:sw=4:sts=4:et
> > +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
> > +"""
> > +BitBake "Fetch" gclient implementation
> > +
> > +gclient fetcher support the SRCU_URI in the following format:
>                                ^^^^^^^^
> Correct the spelling of SRC_URI.
>
> > +
> > +    SRC_URI = "gclient://some.host/path/;optionA=xxx,optionB=xy"
> > +
> > +Supported SRC_UIR options are:
>              ^^^^^^^
> Correct the spelling of SRC_URI.
>
> > +
> > +- branch
> > +  The git branch of the root gclient project. The default is
> > +  "master".
> > +
> > +- tag
> > +  Git tag for gclient to retrieve.
> > +
> > +- protocol
> > +  The method to use for accessing the gclient core project. The default
> > +   is https.
> > +
> > +- project-name
> > +  Passing the --name command line argument to gclient. Not default is
> empty.
>
> Change "Passing" to "Pass" and "Not" to "The".
>
> > +
> > +- rev
> > +  Specify git commit id for gclient. Default is empty.
> > +
> > +- deps-file
> > +  Set the --deps-file command line argument for gclient.
> > +
> > +"""
> > +
> > +# Copyright (C) 2015 Zoltan Kuscsik <zoltan.kuscsik@linaro.org>
> > +#
> > +# Based on git.py which is:
> > +# Copyright (C) 2005 Richard Purdie
> > +#
> > +# This program is free software; you can redistribute it and/or modify
> > +# it under the terms of the GNU General Public License version 2 as
> > +# published by the Free Software Foundation.
> > +#
> > +# This program is distributed in the hope that it will be useful,
> > +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> > +# GNU General Public License for more details.
> > +#
> > +# You should have received a copy of the GNU General Public License
> along
> > +# with this program; if not, write to the Free Software Foundation,
> Inc.,
> > +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
> > +
> > +import os
> > +import bb
> > +from   bb    import data
> > +from   bb.fetch2 import FetchMethod
> > +from   bb.fetch2 import runfetchcmd
> > +import string
> > +
> > +class Gclient(FetchMethod):
> > +    """Class to fetch a module or modules using gclient"""
> > +
> > +    def __fetch_url(self, ud):
> > +        return "%s://%s%s" % (ud.proto, ud.host, ud.path)
> > +
> > +    def supports(self, ud, d):
> > +        """
> > +        Check to see if a given url can be fetched with repo.
>                                                            ^^^^
> Repo? I thought this was for gclient...
>
> > +        """
> > +        return ud.type in ["gclient"]
> > +
> > +    def urldata_init(self, ud, d):
> > +        """
> > +        Init gclient
> > +        """
> > +        ud.proto = ud.parm.get('protocol', 'https')
> > +
> > +        ud.branch = ud.parm.get('branch', 'master')
> > +        ud.tag = ud.parm.get('tag','')
> > +        ud.rev = ud.parm.get('rev','')
> > +
> > +        ud.deps_file = ud.parm.get('deps-file', '')
> > +        ud.project_name = ud.parm.get('project-name', '')
> > +
> > +        if( ( ('tag' in ud.parm) + ('branch' in ud.parm) + ('res' in
> ud.parm)) > 1):
>
> Rewrite as:
>
>         if ('tag' in ud.parm) + ('branch' in ud.parm) + ('res' in ud.parm)
> > 1:
>
> > +            raise bb.fetch2.ParameterError("Confliting revision setting
> SRC_URI. Setting only one of the options: 'branch', 'tag', 'res'.", ud.url)
>                                                ^^^^^^^^^^
> Correct the spelling of "Conflicting". Change "revision setting SRC_URI"
> to "revisions specified in SRC_URI" and "Setting only" to "Specify only".
>
> > +
> > +        dldir = d.getVar("DL_DIR", True)
> > +
> > +        gclient_src_name = 'gclient_%s%s' % (ud.host.replace(':', '.'),
> ud.path.replace('/', '.').replace('*', '.'))
> > +
> > +        ud.syncdir = os.path.join(dldir, gclient_src_name)
> > +        ud.localfile = ud.syncdir
> > +
> > +        if ud.tag:
> > +            ud.revision =  "refs/tags/%s" % ud.tag
>                              ^
> Remove unnecessary space.
>
> > +        elif ud.rev:
> > +            ud.revision = ud.rev
> > +        else:
> > +            ud.revision = "refs/heads/%s" % ud.branch
> > +
> > +    def download(self, ud, d):
> > +        """Fetch url"""
> > +
> > +        if not os.path.exists(ud.localpath):
> > +            bb.utils.mkdirhier(ud.localpath)
> > +        os.chdir(ud.localpath)
> > +        gclient_url = self.__fetch_url(ud)
> > +
> > +
> > +        # Add optional parameters
> > +        extra_params=[]
> > +        if(ud.deps_file):
>
> Rewrite as:
>
>         if ud.deps_file:
>
> > +            extra_params.append("--deps-file=%s" % ud.deps_file)
> > +
> > +        if(ud.project_name):
>
> Rewrite as:
>
>         if ud.project_name:
>
> > +            extra_params.append("--name=%s" % ud.project_name)
> > +
> > +        extra_args = string.join(extra_params, sep = " ")
> > +
> > +        runfetchcmd("gclient config %s %s " % (gclient_url,
> extra_args), d)
> > +
> > +        # Gclient parameters used
> > +        # -f                    - force sync
> > +        # -nohooks              - skip hooks execution at this time
> > +        # --with_branch_heads   - gclient can't find a commit ID if it
> is not in a branch
> > +        #
> > +        # Note: We should enable --no-history after some more testing
> > +
> > +        runfetchcmd("gclient sync --with_branch_heads -f  --nohooks
> --revision %s" % ud.revision, d)
> > +
> > +    def unpack(self, ud, destdir, d):
> > +        """ Unpack the src to destdir """
> > +        destsuffix = "gclient/"
> > +        destdir = ud.destdir = os.path.join(destdir, destsuffix)
> > +        # Use rsync for copying. Creating a tar package here would be
> very
> > +        # time consuming.
> > +        runfetchcmd("rsync -av %s/ %s/" %(ud.syncdir, destdir), d)
> > +        return True
> > +
> > +    def need_update(self, ud, d):
> > +        return True
> > +
> > +    def clean(self, ud, d):
> > +        """ Clean the gclient directory """
> > +        bb.utils.remove(ud.localpath, True)
> > +
> > +    def localpath(self, ud, dr):
> > +        return ud.syncdir
> > +
> > +    def supports_srcrev(self):
> > +        return False
> > --
> > 1.9.1
>
> //Peter
>
>
Peter Kjellerstedt Nov. 4, 2015, 11:44 a.m.
You seem to have missed a couple of comments.

//Peter

From: Zoltan Kuscsik [mailto:zoltan.kuscsik@linaro.org]

Sent: den 4 november 2015 05:08
To: Peter Kjellerstedt
Cc: bitbake-devel@lists.openembedded.org
Subject: Re: [bitbake-devel] [PATCH] bb.fetch2: added support for gclient

Thanks Peter, all comments addressed in PATCHv2.
I also created a test for the patch:

https://github.com/kuscsik/meta-testing/tree/master/recipes-core/fetcher

Br,

Z

On 4 November 2015 at 01:36, Peter Kjellerstedt <peter.kjellerstedt@axis.com<mailto:peter.kjellerstedt@axis.com>> wrote:
> -----Original Message-----

> From: bitbake-devel-bounces@lists.openembedded.org<mailto:bitbake-devel-bounces@lists.openembedded.org> [mailto:bitbake-<mailto:bitbake->

> devel-bounces@lists.openembedded.org<mailto:devel-bounces@lists.openembedded.org>] On Behalf Of Zoltan Kuscsik

> Sent: den 3 november 2015 17:11

> To: bitbake-devel@lists.openembedded.org<mailto:bitbake-devel@lists.openembedded.org>

> Subject: [bitbake-devel] [PATCH] bb.fetch2: added support for gclient

>

> Gclient is tool for updating source code from

> multiple SCM repository locations.

>

> Signed-off-by: Zoltan Kuscsik <zoltan.kuscsik@linaro.org<mailto:zoltan.kuscsik@linaro.org>>

> ---

>  lib/bb/fetch2/__init__.py |   3 +-

>  lib/bb/fetch2/gclient.py  | 151

> ++++++++++++++++++++++++++++++++++++++++++++++

>  2 files changed, 153 insertions(+), 1 deletion(-)

>  create mode 100644 lib/bb/fetch2/gclient.py

>

> diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py

> index 288a1c8..5ca3bc7 100644

> --- a/lib/bb/fetch2/__init__.py

> +++ b/lib/bb/fetch2/__init__.py

> @@ -1529,7 +1529,6 @@ class Fetch(object):

>          fn = d.getVar('FILE', True)

>          if cache and fn and fn in urldata_cache:

>              self.ud = urldata_cache[fn]

> -


Restore the empty line.

>          for url in urls:

>              if url not in self.ud:

>                  try:

> @@ -1775,6 +1774,7 @@ from . import hg

>  from . import osc

>  from . import repo

>  from . import clearcase

> +from . import gclient

>

>  methods.append(local.Local())

>  methods.append(wget.Wget())

> @@ -1791,3 +1791,4 @@ methods.append(hg.Hg())

>  methods.append(osc.Osc())

>  methods.append(repo.Repo())

>  methods.append(clearcase.ClearCase())

> +methods.append(gclient.Gclient())

> diff --git a/lib/bb/fetch2/gclient.py b/lib/bb/fetch2/gclient.py

> new file mode 100644

> index 0000000..de06eb5

> --- /dev/null

> +++ b/lib/bb/fetch2/gclient.py

> @@ -0,0 +1,151 @@

> +# ex:ts=4:sw=4:sts=4:et

> +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-

> +"""

> +BitBake "Fetch" gclient implementation

> +

> +gclient fetcher support the SRCU_URI in the following format:

                               ^^^^^^^^
Correct the spelling of SRC_URI.

> +

> +    SRC_URI = "gclient://some.host/path/;optionA=xxx,optionB=xy"

> +

> +Supported SRC_UIR options are:

             ^^^^^^^
Correct the spelling of SRC_URI.

> +

> +- branch

> +  The git branch of the root gclient project. The default is

> +  "master".

> +

> +- tag

> +  Git tag for gclient to retrieve.

> +

> +- protocol

> +  The method to use for accessing the gclient core project. The default

> +   is https.

> +

> +- project-name

> +  Passing the --name command line argument to gclient. Not default is empty.


Change "Passing" to "Pass" and "Not" to "The".

> +

> +- rev

> +  Specify git commit id for gclient. Default is empty.

> +

> +- deps-file

> +  Set the --deps-file command line argument for gclient.

> +

> +"""

> +

> +# Copyright (C) 2015 Zoltan Kuscsik <zoltan.kuscsik@linaro.org<mailto:zoltan.kuscsik@linaro.org>>

> +#

> +# Based on git.py which is:

> +# Copyright (C) 2005 Richard Purdie

> +#

> +# This program is free software; you can redistribute it and/or modify

> +# it under the terms of the GNU General Public License version 2 as

> +# published by the Free Software Foundation.

> +#

> +# This program is distributed in the hope that it will be useful,

> +# but WITHOUT ANY WARRANTY; without even the implied warranty of

> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

> +# GNU General Public License for more details.

> +#

> +# You should have received a copy of the GNU General Public License along

> +# with this program; if not, write to the Free Software Foundation, Inc.,

> +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

> +

> +import os

> +import bb

> +from   bb    import data

> +from   bb.fetch2 import FetchMethod

> +from   bb.fetch2 import runfetchcmd

> +import string

> +

> +class Gclient(FetchMethod):

> +    """Class to fetch a module or modules using gclient"""

> +

> +    def __fetch_url(self, ud):

> +        return "%s://%s%s" % (ud.proto, ud.host, ud.path)

> +

> +    def supports(self, ud, d):

> +        """

> +        Check to see if a given url can be fetched with repo.

                                                           ^^^^
Repo? I thought this was for gclient...

> +        """

> +        return ud.type in ["gclient"]

> +

> +    def urldata_init(self, ud, d):

> +        """

> +        Init gclient

> +        """

> +        ud.proto = ud.parm.get('protocol', 'https')

> +

> +        ud.branch = ud.parm.get('branch', 'master')

> +        ud.tag = ud.parm.get('tag','')

> +        ud.rev = ud.parm.get('rev','')

> +

> +        ud.deps_file = ud.parm.get('deps-file', '')

> +        ud.project_name = ud.parm.get('project-name', '')

> +

> +        if( ( ('tag' in ud.parm) + ('branch' in ud.parm) + ('res' in ud.parm)) > 1):


Rewrite as:

        if ('tag' in ud.parm) + ('branch' in ud.parm) + ('res' in ud.parm) > 1:

> +            raise bb.fetch2.ParameterError("Confliting revision setting SRC_URI. Setting only one of the options: 'branch', 'tag', 'res'.", ud.url)

                                               ^^^^^^^^^^
Correct the spelling of "Conflicting". Change "revision setting SRC_URI"
to "revisions specified in SRC_URI" and "Setting only" to "Specify only".

> +

> +        dldir = d.getVar("DL_DIR", True)

> +

> +        gclient_src_name = 'gclient_%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.'))

> +

> +        ud.syncdir = os.path.join(dldir, gclient_src_name)

> +        ud.localfile = ud.syncdir

> +

> +        if ud.tag:

> +            ud.revision =  "refs/tags/%s" % ud.tag

                             ^
Remove unnecessary space.

> +        elif ud.rev:

> +            ud.revision = ud.rev

> +        else:

> +            ud.revision = "refs/heads/%s" % ud.branch

> +

> +    def download(self, ud, d):

> +        """Fetch url"""

> +

> +        if not os.path.exists(ud.localpath):

> +            bb.utils.mkdirhier(ud.localpath)

> +        os.chdir(ud.localpath)

> +        gclient_url = self.__fetch_url(ud)

> +

> +

> +        # Add optional parameters

> +        extra_params=[]

> +        if(ud.deps_file):


Rewrite as:

        if ud.deps_file:

> +            extra_params.append("--deps-file=%s" % ud.deps_file)

> +

> +        if(ud.project_name):


Rewrite as:

        if ud.project_name:

> +            extra_params.append("--name=%s" % ud.project_name)

> +

> +        extra_args = string.join(extra_params, sep = " ")

> +

> +        runfetchcmd("gclient config %s %s " % (gclient_url, extra_args), d)

> +

> +        # Gclient parameters used

> +        # -f                    - force sync

> +        # -nohooks              - skip hooks execution at this time

> +        # --with_branch_heads   - gclient can't find a commit ID if it is not in a branch

> +        #

> +        # Note: We should enable --no-history after some more testing

> +

> +        runfetchcmd("gclient sync --with_branch_heads -f  --nohooks --revision %s" % ud.revision, d)

> +

> +    def unpack(self, ud, destdir, d):

> +        """ Unpack the src to destdir """

> +        destsuffix = "gclient/"

> +        destdir = ud.destdir = os.path.join(destdir, destsuffix)

> +        # Use rsync for copying. Creating a tar package here would be very

> +        # time consuming.

> +        runfetchcmd("rsync -av %s/ %s/" %(ud.syncdir, destdir), d)

> +        return True

> +

> +    def need_update(self, ud, d):

> +        return True

> +

> +    def clean(self, ud, d):

> +        """ Clean the gclient directory """

> +        bb.utils.remove(ud.localpath, True)

> +

> +    def localpath(self, ud, dr):

> +        return ud.syncdir

> +

> +    def supports_srcrev(self):

> +        return False

> --

> 1.9.1

//Peter



--
Zoltan Kuscsik

Senior Software Developer, Linaro Digital Home Group
email: zoltan.kuscsik@linaro.org<mailto:zoltan.kuscsik@linaro.org>
IRC: zoli_ on irc.linaro.org<http://irc.linaro.org/>
Zoltan Kuscsik Nov. 4, 2015, 1:02 p.m.
Thanks for noticing it.
I double checked the comments. I believe all of them are processed in v3.

Z

On 4 November 2015 at 12:44, Peter Kjellerstedt <peter.kjellerstedt@axis.com
> wrote:

> You seem to have missed a couple of comments.
>
>
>
> //Peter
>
>
>
> *From:* Zoltan Kuscsik [mailto:zoltan.kuscsik@linaro.org]
> *Sent:* den 4 november 2015 05:08
> *To:* Peter Kjellerstedt
> *Cc:* bitbake-devel@lists.openembedded.org
> *Subject:* Re: [bitbake-devel] [PATCH] bb.fetch2: added support for
> gclient
>
>
>
> Thanks Peter, all comments addressed in PATCHv2.
>
> I also created a test for the patch:
>
>
>
> https://github.com/kuscsik/meta-testing/tree/master/recipes-core/fetcher
>
>
>
> Br,
>
>
>
> Z
>
>
>
> On 4 November 2015 at 01:36, Peter Kjellerstedt <
> peter.kjellerstedt@axis.com> wrote:
>
> > -----Original Message-----
> > From: bitbake-devel-bounces@lists.openembedded.org [mailto:bitbake-
> > devel-bounces@lists.openembedded.org] On Behalf Of Zoltan Kuscsik
> > Sent: den 3 november 2015 17:11
> > To: bitbake-devel@lists.openembedded.org
> > Subject: [bitbake-devel] [PATCH] bb.fetch2: added support for gclient
> >
> > Gclient is tool for updating source code from
> > multiple SCM repository locations.
> >
> > Signed-off-by: Zoltan Kuscsik <zoltan.kuscsik@linaro.org>
> > ---
> >  lib/bb/fetch2/__init__.py |   3 +-
> >  lib/bb/fetch2/gclient.py  | 151
> > ++++++++++++++++++++++++++++++++++++++++++++++
> >  2 files changed, 153 insertions(+), 1 deletion(-)
> >  create mode 100644 lib/bb/fetch2/gclient.py
> >
> > diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py
> > index 288a1c8..5ca3bc7 100644
> > --- a/lib/bb/fetch2/__init__.py
> > +++ b/lib/bb/fetch2/__init__.py
> > @@ -1529,7 +1529,6 @@ class Fetch(object):
> >          fn = d.getVar('FILE', True)
> >          if cache and fn and fn in urldata_cache:
> >              self.ud = urldata_cache[fn]
> > -
>
> Restore the empty line.
>
>
> >          for url in urls:
> >              if url not in self.ud:
> >                  try:
> > @@ -1775,6 +1774,7 @@ from . import hg
> >  from . import osc
> >  from . import repo
> >  from . import clearcase
> > +from . import gclient
> >
> >  methods.append(local.Local())
> >  methods.append(wget.Wget())
> > @@ -1791,3 +1791,4 @@ methods.append(hg.Hg())
> >  methods.append(osc.Osc())
> >  methods.append(repo.Repo())
> >  methods.append(clearcase.ClearCase())
> > +methods.append(gclient.Gclient())
> > diff --git a/lib/bb/fetch2/gclient.py b/lib/bb/fetch2/gclient.py
> > new file mode 100644
> > index 0000000..de06eb5
> > --- /dev/null
> > +++ b/lib/bb/fetch2/gclient.py
> > @@ -0,0 +1,151 @@
> > +# ex:ts=4:sw=4:sts=4:et
> > +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
> > +"""
> > +BitBake "Fetch" gclient implementation
> > +
> > +gclient fetcher support the SRCU_URI in the following format:
>
>                                ^^^^^^^^
> Correct the spelling of SRC_URI.
>
> > +
> > +    SRC_URI = "gclient://some.host/path/;optionA=xxx,optionB=xy"
> > +
> > +Supported SRC_UIR options are:
>              ^^^^^^^
> Correct the spelling of SRC_URI.
>
> > +
> > +- branch
> > +  The git branch of the root gclient project. The default is
> > +  "master".
> > +
> > +- tag
> > +  Git tag for gclient to retrieve.
> > +
> > +- protocol
> > +  The method to use for accessing the gclient core project. The default
> > +   is https.
> > +
> > +- project-name
> > +  Passing the --name command line argument to gclient. Not default is
> empty.
>
> Change "Passing" to "Pass" and "Not" to "The".
>
>
> > +
> > +- rev
> > +  Specify git commit id for gclient. Default is empty.
> > +
> > +- deps-file
> > +  Set the --deps-file command line argument for gclient.
> > +
> > +"""
> > +
> > +# Copyright (C) 2015 Zoltan Kuscsik <zoltan.kuscsik@linaro.org>
> > +#
> > +# Based on git.py which is:
> > +# Copyright (C) 2005 Richard Purdie
> > +#
> > +# This program is free software; you can redistribute it and/or modify
> > +# it under the terms of the GNU General Public License version 2 as
> > +# published by the Free Software Foundation.
> > +#
> > +# This program is distributed in the hope that it will be useful,
> > +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> > +# GNU General Public License for more details.
> > +#
> > +# You should have received a copy of the GNU General Public License
> along
> > +# with this program; if not, write to the Free Software Foundation,
> Inc.,
> > +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
> > +
> > +import os
> > +import bb
> > +from   bb    import data
> > +from   bb.fetch2 import FetchMethod
> > +from   bb.fetch2 import runfetchcmd
> > +import string
> > +
> > +class Gclient(FetchMethod):
> > +    """Class to fetch a module or modules using gclient"""
> > +
> > +    def __fetch_url(self, ud):
> > +        return "%s://%s%s" % (ud.proto, ud.host, ud.path)
> > +
> > +    def supports(self, ud, d):
> > +        """
> > +        Check to see if a given url can be fetched with repo.
>
>                                                            ^^^^
> Repo? I thought this was for gclient...
>
> > +        """
> > +        return ud.type in ["gclient"]
> > +
> > +    def urldata_init(self, ud, d):
> > +        """
> > +        Init gclient
> > +        """
> > +        ud.proto = ud.parm.get('protocol', 'https')
> > +
> > +        ud.branch = ud.parm.get('branch', 'master')
> > +        ud.tag = ud.parm.get('tag','')
> > +        ud.rev = ud.parm.get('rev','')
> > +
> > +        ud.deps_file = ud.parm.get('deps-file', '')
> > +        ud.project_name = ud.parm.get('project-name', '')
> > +
> > +        if( ( ('tag' in ud.parm) + ('branch' in ud.parm) + ('res' in
> ud.parm)) > 1):
>
> Rewrite as:
>
>         if ('tag' in ud.parm) + ('branch' in ud.parm) + ('res' in ud.parm)
> > 1:
>
> > +            raise bb.fetch2.ParameterError("Confliting revision setting
> SRC_URI. Setting only one of the options: 'branch', 'tag', 'res'.", ud.url)
>                                                ^^^^^^^^^^
> Correct the spelling of "Conflicting". Change "revision setting SRC_URI"
> to "revisions specified in SRC_URI" and "Setting only" to "Specify only".
>
> > +
> > +        dldir = d.getVar("DL_DIR", True)
> > +
> > +        gclient_src_name = 'gclient_%s%s' % (ud.host.replace(':', '.'),
> ud.path.replace('/', '.').replace('*', '.'))
> > +
> > +        ud.syncdir = os.path.join(dldir, gclient_src_name)
> > +        ud.localfile = ud.syncdir
> > +
> > +        if ud.tag:
> > +            ud.revision =  "refs/tags/%s" % ud.tag
>                              ^
> Remove unnecessary space.
>
> > +        elif ud.rev:
> > +            ud.revision = ud.rev
> > +        else:
> > +            ud.revision = "refs/heads/%s" % ud.branch
> > +
> > +    def download(self, ud, d):
> > +        """Fetch url"""
> > +
> > +        if not os.path.exists(ud.localpath):
> > +            bb.utils.mkdirhier(ud.localpath)
> > +        os.chdir(ud.localpath)
> > +        gclient_url = self.__fetch_url(ud)
> > +
> > +
> > +        # Add optional parameters
> > +        extra_params=[]
> > +        if(ud.deps_file):
>
> Rewrite as:
>
>         if ud.deps_file:
>
> > +            extra_params.append("--deps-file=%s" % ud.deps_file)
> > +
> > +        if(ud.project_name):
>
> Rewrite as:
>
>         if ud.project_name:
>
>
> > +            extra_params.append("--name=%s" % ud.project_name)
> > +
> > +        extra_args = string.join(extra_params, sep = " ")
> > +
> > +        runfetchcmd("gclient config %s %s " % (gclient_url,
> extra_args), d)
> > +
> > +        # Gclient parameters used
> > +        # -f                    - force sync
> > +        # -nohooks              - skip hooks execution at this time
> > +        # --with_branch_heads   - gclient can't find a commit ID if it
> is not in a branch
> > +        #
> > +        # Note: We should enable --no-history after some more testing
> > +
> > +        runfetchcmd("gclient sync --with_branch_heads -f  --nohooks
> --revision %s" % ud.revision, d)
> > +
> > +    def unpack(self, ud, destdir, d):
> > +        """ Unpack the src to destdir """
> > +        destsuffix = "gclient/"
> > +        destdir = ud.destdir = os.path.join(destdir, destsuffix)
> > +        # Use rsync for copying. Creating a tar package here would be
> very
> > +        # time consuming.
> > +        runfetchcmd("rsync -av %s/ %s/" %(ud.syncdir, destdir), d)
> > +        return True
> > +
> > +    def need_update(self, ud, d):
> > +        return True
> > +
> > +    def clean(self, ud, d):
> > +        """ Clean the gclient directory """
> > +        bb.utils.remove(ud.localpath, True)
> > +
> > +    def localpath(self, ud, dr):
> > +        return ud.syncdir
> > +
> > +    def supports_srcrev(self):
> > +        return False
> > --
> > 1.9.1
>
> //Peter
>
>
>
>
>
> --
>
> Zoltan Kuscsik
>
>
>
> Senior Software Developer, Linaro Digital Home Group
>
> email: zoltan.kuscsik@linaro.org
>
> IRC: zoli_ on irc.linaro.org
>