[bitbake-devel,v2] fetch2/gitsm.py: Change the URI construction logic

Submitted by Krystian Garliński on Jan. 8, 2019, 7:47 a.m. | Patch ID: 157618

Details

Message ID 20190108074737.15427-1-krystian.garlinski@comarch.pl
State Master Next
Commit 834baba8f01ac99751ff63f16345e350b5d4d6c1
Headers show

Commit Message

Krystian Garliński Jan. 8, 2019, 7:47 a.m.
Git allows to use both the proper URI's and SCP-like short style syntax
for the SSH protocol. This commit changes construction logic to detect which
one of the two is used in the submodule specification and to build a correct
URI out of it.

Signed-off-by: Krystian Garliński <krystian.garlinski@comarch.pl>
---
 lib/bb/fetch2/gitsm.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

Patch hide | download patch | download mbox

diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py
index 35729dbc..95d5f652 100644
--- a/lib/bb/fetch2/gitsm.py
+++ b/lib/bb/fetch2/gitsm.py
@@ -32,6 +32,7 @@  NOTE: Switching a SRC_URI from "git://" to "gitsm://" requires a clean of your r
 import os
 import bb
 import copy
+import re
 from   bb.fetch2.git import Git
 from   bb.fetch2 import runfetchcmd
 from   bb.fetch2 import logger
@@ -39,6 +40,8 @@  from   bb.fetch2 import Fetch
 from   bb.fetch2 import BBFetchException
 
 class GitSM(Git):
+    scp_regex = re.compile(r'^([a-zA-Z0-9_]+)@([a-zA-Z0-9._-]+):(.+)$')
+
     def supports(self, ud, d):
         """
         Check to see if a given url can be fetched with git.
@@ -88,8 +91,17 @@  class GitSM(Git):
             module_hash = module_hash.split()[2]
 
             # Build new SRC_URI
-            proto = uris[module].split(':', 1)[0]
-            url = uris[module].replace('%s:' % proto, 'gitsm:', 1)
+            uri = uris[module]
+            match = GitSM.scp_regex.match(uri)
+
+            if match:
+                # this URI follows the short SCP-like syntax
+                url = 'gitsm://{}@{}/{}'.format(match.group(1), match.group(2), match.group(3))
+                proto = 'ssh'
+            else:
+                proto = uri.split(':', 1)[0]
+                url = 'gitsm' + uri[len(proto):]
+
             url += ';protocol=%s' % proto
             url += ";name=%s" % module
             url += ";bareclone=1;nocheckout=1;nobranch=1"

Comments

Linus Ziegert Jan. 8, 2019, 4:35 p.m.
Hello people,

I am working as well on fetch2/gitsm.py. We have a privat repo with
submodules but the current gitsm.py does not support ssh.

I tested the patch by Krystian but it did not worked for me. I got the
following ERROR:

ERROR: <recipe> do_fetch: The URL: 'gitsm:git@github.com/<user>/<submodule
_name1>.git;protocol=ssh;name=<submodule_name2>;bareclone=1;nocheckout=1;
nobranch=1' is invalid and cannot be interpreted


The .gitmodules file used looks like this:

[submodule "submodule_name1"]
path = submodule_name1
        url = git@github.com/<user>/<submodule_name1>.git
[submodule "submodule_name2"]
path = submodule_name2
        url = git@github.com/<user>/<submodule_name2>.git

Attached you find my proposal.


Am Di., 8. Jan. 2019 um 08:48 Uhr schrieb Krystian Garliński <krystian.
garlinski@comarch.pl>:

> Git allows to use both the proper URI's and SCP-like short style syntax
> for the SSH protocol. This commit changes construction logic to detect
> which
> one of the two is used in the submodule specification and to build a
> correct
> URI out of it.
>
> Signed-off-by: Krystian Garliński <krystian.garlinski@comarch.pl>
> ---
>  lib/bb/fetch2/gitsm.py | 16 ++++++++++++++--
>  1 file changed, 14 insertions(+), 2 deletions(-)
>
> diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py
> index 35729dbc..95d5f652 100644
> --- a/lib/bb/fetch2/gitsm.py
> +++ b/lib/bb/fetch2/gitsm.py
> @@ -32,6 +32,7 @@ NOTE: Switching a SRC_URI from "git://" to "gitsm://"
> requires a clean of your r
>  import os
>  import bb
>  import copy
> +import re
>  from   bb.fetch2.git import Git
>  from   bb.fetch2 import runfetchcmd
>  from   bb.fetch2 import logger
> @@ -39,6 +40,8 @@ from   bb.fetch2 import Fetch
>  from   bb.fetch2 import BBFetchException
>
>  class GitSM(Git):
> +    scp_regex = re.compile(r'^([a-zA-Z0-9_]+)@([a-zA-Z0-9._-]+):(.+)$')
> +
>      def supports(self, ud, d):
>          """
>          Check to see if a given url can be fetched with git.
> @@ -88,8 +91,17 @@ class GitSM(Git):
>              module_hash = module_hash.split()[2]
>
>              # Build new SRC_URI
> -            proto = uris[module].split(':', 1)[0]
> -            url = uris[module].replace('%s:' % proto, 'gitsm:', 1)
> +            uri = uris[module]
> +            match = GitSM.scp_regex.match(uri)
> +
> +            if match:
> +                # this URI follows the short SCP-like syntax
> +                url = 'gitsm://{}@{}/{}'.format(match.group(1),
> match.group(2), match.group(3))
> +                proto = 'ssh'
> +            else:
> +                proto = uri.split(':', 1)[0]
> +                url = 'gitsm' + uri[len(proto):]
> +
>              url += ';protocol=%s' % proto
>              url += ";name=%s" % module
>              url += ";bareclone=1;nocheckout=1;nobranch=1"
> --
> 2.19.1
>
> --
> _______________________________________________
> bitbake-devel mailing list
> bitbake-devel@lists.openembedded.org
> http://lists.openembedded.org/mailman/listinfo/bitbake-devel
>
Mark Hatle Jan. 8, 2019, 7:05 p.m.
The patch here and the other that didn't quite work either are close but not
completely correct.

There are apparently three (ssh) formats that git supports:

url format - ssh://user:password@host/path
ssh format - user@host:/path
(ssh)? format - user@host/path

Each have a different regex or splitting format required.  I'm working on a
specific reproducer test case for each of these, and then will work to combine
or tailor the various patches that have been submitted.  If anyone is aware of
any other odd formats please let me know and I'll add appropriate test cases for
those as well.

I'm guessing it'll be a couple of days for me to work through this, and then I
ask that people with the problem test this and verify it works in their specific
case.

--Mark

On 1/8/19 10:35 AM, Linus Ziegert wrote:
> Hello people,
> 
> I am working as well on fetch2/gitsm.py. We have a privat repo with submodules
> but the current gitsm.py does not support ssh.
> 
> I tested the patch by Krystian but it did not worked for me. I got the following
> ERROR:
> 
>     ERROR: <recipe> do_fetch: The URL:
>     'gitsm:git@github.com/<user>/<submodule_name1>.git;protocol=ssh;name=<submodule_name2>;bareclone=1;nocheckout=1;nobranch=1'
>     is invalid and cannot be interpreted
> 
> 
> The .gitmodules file used looks like this:
> 
> [submodule "submodule_name1"]
> path = submodule_name1
>         url = git@github.com/<user>/<submodule_name1>.git 
> [submodule "submodule_name2"]
> path = submodule_name2
>         url = git@github.com/<user>/<submodule_name2>.git 
> 
> Attached you find my proposal.
> 
> 
> Am Di., 8. Jan. 2019 um 08:48 Uhr schrieb Krystian Garliński
> <krystian.garlinski@comarch.pl>:
> 
>     Git allows to use both the proper URI's and SCP-like short style syntax
>     for the SSH protocol. This commit changes construction logic to detect which
>     one of the two is used in the submodule specification and to build a correct
>     URI out of it.
> 
>     Signed-off-by: Krystian Garliński <krystian.garlinski@comarch.pl
>     <mailto:krystian.garlinski@comarch.pl>>
>     ---
>      lib/bb/fetch2/gitsm.py | 16 ++++++++++++++--
>      1 file changed, 14 insertions(+), 2 deletions(-)
> 
>     diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py
>     index 35729dbc..95d5f652 100644
>     --- a/lib/bb/fetch2/gitsm.py
>     +++ b/lib/bb/fetch2/gitsm.py
>     @@ -32,6 +32,7 @@ NOTE: Switching a SRC_URI from "git://" to "gitsm://"
>     requires a clean of your r
>      import os
>      import bb
>      import copy
>     +import re
>      from   bb.fetch2.git import Git
>      from   bb.fetch2 import runfetchcmd
>      from   bb.fetch2 import logger
>     @@ -39,6 +40,8 @@ from   bb.fetch2 import Fetch
>      from   bb.fetch2 import BBFetchException
> 
>      class GitSM(Git):
>     +    scp_regex = re.compile(r'^([a-zA-Z0-9_]+)@([a-zA-Z0-9._-]+):(.+)$')
>     +
>          def supports(self, ud, d):
>              """
>              Check to see if a given url can be fetched with git.
>     @@ -88,8 +91,17 @@ class GitSM(Git):
>                  module_hash = module_hash.split()[2]
> 
>                  # Build new SRC_URI
>     -            proto = uris[module].split(':', 1)[0]
>     -            url = uris[module].replace('%s:' % proto, 'gitsm:', 1)
>     +            uri = uris[module]
>     +            match = GitSM.scp_regex.match(uri)
>     +
>     +            if match:
>     +                # this URI follows the short SCP-like syntax
>     +                url = 'gitsm://{}@{}/{}'.format(match.group(1),
>     match.group(2), match.group(3))
>     +                proto = 'ssh'
>     +            else:
>     +                proto = uri.split(':', 1)[0]
>     +                url = 'gitsm' + uri[len(proto):]
>     +
>                  url += ';protocol=%s' % proto
>                  url += ";name=%s" % module
>                  url += ";bareclone=1;nocheckout=1;nobranch=1"
>     -- 
>     2.19.1
> 
>     -- 
>     _______________________________________________
>     bitbake-devel mailing list
>     bitbake-devel@lists.openembedded.org
>     <mailto:bitbake-devel@lists.openembedded.org>
>     http://lists.openembedded.org/mailman/listinfo/bitbake-devel
> 
> 
> 
> -- 
> 
> Cheers, 
> Linus
>
Mark Hatle Jan. 8, 2019, 9:19 p.m.
Upon further work, I'm getting this:

[git-submodule-test]$ git submodule add git@git.yoctoproject.org/bitbake-gitsm-test3
repo URL: 'git@git.yoctoproject.org/bitbake-gitsm-test3' must be absolute or
begin with ./|../

This tells me that the format mentioned below is not actually valid.

I'm getting the message from git versions:

git version 1.8.3.1
git version 2.16.0.rc2

If I change it to the following, it does work -- and this is the format we're
already expecting...

git submodule add git@git.yoctoproject.org:/bitbake-gitsm-test3

I'll try to manually modify the .gitmodules for this format, but technically
speaking it doesn't appear to be legal.

--Mark

On 1/8/19 10:35 AM, Linus Ziegert wrote:
> Hello people,
> 
> I am working as well on fetch2/gitsm.py. We have a privat repo with submodules
> but the current gitsm.py does not support ssh.
> 
> I tested the patch by Krystian but it did not worked for me. I got the following
> ERROR:
> 
>     ERROR: <recipe> do_fetch: The URL:
>     'gitsm:git@github.com/<user>/<submodule_name1>.git;protocol=ssh;name=<submodule_name2>;bareclone=1;nocheckout=1;nobranch=1'
>     is invalid and cannot be interpreted
> 
> 
> The .gitmodules file used looks like this:
> 
> [submodule "submodule_name1"]
> path = submodule_name1
>         url = git@github.com/<user>/<submodule_name1>.git 
> [submodule "submodule_name2"]
> path = submodule_name2
>         url = git@github.com/<user>/<submodule_name2>.git 
> 
> Attached you find my proposal.
> 
> 
> Am Di., 8. Jan. 2019 um 08:48 Uhr schrieb Krystian Garliński
> <krystian.garlinski@comarch.pl>:
> 
>     Git allows to use both the proper URI's and SCP-like short style syntax
>     for the SSH protocol. This commit changes construction logic to detect which
>     one of the two is used in the submodule specification and to build a correct
>     URI out of it.
> 
>     Signed-off-by: Krystian Garliński <krystian.garlinski@comarch.pl
>     <mailto:krystian.garlinski@comarch.pl>>
>     ---
>      lib/bb/fetch2/gitsm.py | 16 ++++++++++++++--
>      1 file changed, 14 insertions(+), 2 deletions(-)
> 
>     diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py
>     index 35729dbc..95d5f652 100644
>     --- a/lib/bb/fetch2/gitsm.py
>     +++ b/lib/bb/fetch2/gitsm.py
>     @@ -32,6 +32,7 @@ NOTE: Switching a SRC_URI from "git://" to "gitsm://"
>     requires a clean of your r
>      import os
>      import bb
>      import copy
>     +import re
>      from   bb.fetch2.git import Git
>      from   bb.fetch2 import runfetchcmd
>      from   bb.fetch2 import logger
>     @@ -39,6 +40,8 @@ from   bb.fetch2 import Fetch
>      from   bb.fetch2 import BBFetchException
> 
>      class GitSM(Git):
>     +    scp_regex = re.compile(r'^([a-zA-Z0-9_]+)@([a-zA-Z0-9._-]+):(.+)$')
>     +
>          def supports(self, ud, d):
>              """
>              Check to see if a given url can be fetched with git.
>     @@ -88,8 +91,17 @@ class GitSM(Git):
>                  module_hash = module_hash.split()[2]
> 
>                  # Build new SRC_URI
>     -            proto = uris[module].split(':', 1)[0]
>     -            url = uris[module].replace('%s:' % proto, 'gitsm:', 1)
>     +            uri = uris[module]
>     +            match = GitSM.scp_regex.match(uri)
>     +
>     +            if match:
>     +                # this URI follows the short SCP-like syntax
>     +                url = 'gitsm://{}@{}/{}'.format(match.group(1),
>     match.group(2), match.group(3))
>     +                proto = 'ssh'
>     +            else:
>     +                proto = uri.split(':', 1)[0]
>     +                url = 'gitsm' + uri[len(proto):]
>     +
>                  url += ';protocol=%s' % proto
>                  url += ";name=%s" % module
>                  url += ";bareclone=1;nocheckout=1;nobranch=1"
>     -- 
>     2.19.1
> 
>     -- 
>     _______________________________________________
>     bitbake-devel mailing list
>     bitbake-devel@lists.openembedded.org
>     <mailto:bitbake-devel@lists.openembedded.org>
>     http://lists.openembedded.org/mailman/listinfo/bitbake-devel
> 
> 
> 
> -- 
> 
> Cheers, 
> Linus
>
Mark Hatle Jan. 8, 2019, 9:24 p.m.
On 1/8/19 3:19 PM, Mark Hatle wrote:
> Upon further work, I'm getting this:
> 
> [git-submodule-test]$ git submodule add git@git.yoctoproject.org/bitbake-gitsm-test3
> repo URL: 'git@git.yoctoproject.org/bitbake-gitsm-test3' must be absolute or
> begin with ./|../
> 
> This tells me that the format mentioned below is not actually valid.
> 
> I'm getting the message from git versions:
> 
> git version 1.8.3.1
> git version 2.16.0.rc2
> 
> If I change it to the following, it does work -- and this is the format we're
> already expecting...
> 
> git submodule add git@git.yoctoproject.org:/bitbake-gitsm-test3
> 
> I'll try to manually modify the .gitmodules for this format, but technically
> speaking it doesn't appear to be legal.

I did finally try it.  Without the ':' git is looking at the local hard disk and
assuming that is a path and failing:


fatal: repository 'git@git.yoctoproject.org/bitbake-gitsm-test3' does not exist
Clone of 'git@git.yoctoproject.org/bitbake-gitsm-test3' into submodule path
'bitbake-gitsm-test3' failed

So I'm not sure how the stuff below is even valid.  I need a reproducer to be
able to code for it and test things.  Until then, everything I see says that
it's one of three formats:

proto://user:pass@host/path  (url style)
user@host:path (ssh style)
/path or ./path or ../path  (local path)

--Mark

> --Mark
> 
> On 1/8/19 10:35 AM, Linus Ziegert wrote:
>> Hello people,
>>
>> I am working as well on fetch2/gitsm.py. We have a privat repo with submodules
>> but the current gitsm.py does not support ssh.
>>
>> I tested the patch by Krystian but it did not worked for me. I got the following
>> ERROR:
>>
>>     ERROR: <recipe> do_fetch: The URL:
>>     'gitsm:git@github.com/<user>/<submodule_name1>.git;protocol=ssh;name=<submodule_name2>;bareclone=1;nocheckout=1;nobranch=1'
>>     is invalid and cannot be interpreted
>>
>>
>> The .gitmodules file used looks like this:
>>
>> [submodule "submodule_name1"]
>> path = submodule_name1
>>         url = git@github.com/<user>/<submodule_name1>.git 
>> [submodule "submodule_name2"]
>> path = submodule_name2
>>         url = git@github.com/<user>/<submodule_name2>.git 
>>
>> Attached you find my proposal.
>>
>>
>> Am Di., 8. Jan. 2019 um 08:48 Uhr schrieb Krystian Garliński
>> <krystian.garlinski@comarch.pl>:
>>
>>     Git allows to use both the proper URI's and SCP-like short style syntax
>>     for the SSH protocol. This commit changes construction logic to detect which
>>     one of the two is used in the submodule specification and to build a correct
>>     URI out of it.
>>
>>     Signed-off-by: Krystian Garliński <krystian.garlinski@comarch.pl
>>     <mailto:krystian.garlinski@comarch.pl>>
>>     ---
>>      lib/bb/fetch2/gitsm.py | 16 ++++++++++++++--
>>      1 file changed, 14 insertions(+), 2 deletions(-)
>>
>>     diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py
>>     index 35729dbc..95d5f652 100644
>>     --- a/lib/bb/fetch2/gitsm.py
>>     +++ b/lib/bb/fetch2/gitsm.py
>>     @@ -32,6 +32,7 @@ NOTE: Switching a SRC_URI from "git://" to "gitsm://"
>>     requires a clean of your r
>>      import os
>>      import bb
>>      import copy
>>     +import re
>>      from   bb.fetch2.git import Git
>>      from   bb.fetch2 import runfetchcmd
>>      from   bb.fetch2 import logger
>>     @@ -39,6 +40,8 @@ from   bb.fetch2 import Fetch
>>      from   bb.fetch2 import BBFetchException
>>
>>      class GitSM(Git):
>>     +    scp_regex = re.compile(r'^([a-zA-Z0-9_]+)@([a-zA-Z0-9._-]+):(.+)$')
>>     +
>>          def supports(self, ud, d):
>>              """
>>              Check to see if a given url can be fetched with git.
>>     @@ -88,8 +91,17 @@ class GitSM(Git):
>>                  module_hash = module_hash.split()[2]
>>
>>                  # Build new SRC_URI
>>     -            proto = uris[module].split(':', 1)[0]
>>     -            url = uris[module].replace('%s:' % proto, 'gitsm:', 1)
>>     +            uri = uris[module]
>>     +            match = GitSM.scp_regex.match(uri)
>>     +
>>     +            if match:
>>     +                # this URI follows the short SCP-like syntax
>>     +                url = 'gitsm://{}@{}/{}'.format(match.group(1),
>>     match.group(2), match.group(3))
>>     +                proto = 'ssh'
>>     +            else:
>>     +                proto = uri.split(':', 1)[0]
>>     +                url = 'gitsm' + uri[len(proto):]
>>     +
>>                  url += ';protocol=%s' % proto
>>                  url += ";name=%s" % module
>>                  url += ";bareclone=1;nocheckout=1;nobranch=1"
>>     -- 
>>     2.19.1
>>
>>     -- 
>>     _______________________________________________
>>     bitbake-devel mailing list
>>     bitbake-devel@lists.openembedded.org
>>     <mailto:bitbake-devel@lists.openembedded.org>
>>     http://lists.openembedded.org/mailman/listinfo/bitbake-devel
>>
>>
>>
>> -- 
>>
>> Cheers, 
>> Linus
>>
>