| Message ID | 20230804155424.1502010-1-JPEWhacker@gmail.com |
|---|---|
| State | New |
| Headers | show |
| Series | [bitbake-devel] fetch2: git: Check if directory is a bare git repo | expand |
Hello Joshua, I'm fairly sure this causes the following bitbake-selftest failure: https://autobuilder.yoctoproject.org/typhoon/#/builders/56/builds/2307/steps/11/logs/stdio On 04/08/2023 09:54:24-0600, Joshua Watt wrote: > If the clone target directory exists, but isn't the valid top level of a > bare git repo, it needs to be erased and re-cloned. One example of how > this can happen is if a clone creates the directory, but then fails to > actual clone and make it a git repository. This left-over directory can > be particularly problematic if the download directory is a descent of > some top layer git repo (e.g. the default with poky), as the commands > that operate on the remote later will then mangle the layers git > repository instead of the download git repo. > > Signed-off-by: Joshua Watt <JPEWhacker@gmail.com> > --- > bitbake/lib/bb/fetch2/git.py | 24 +++++++++++++++++++++++- > 1 file changed, 23 insertions(+), 1 deletion(-) > > diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py > index 2a3c06fe4e9..112bed294f0 100644 > --- a/bitbake/lib/bb/fetch2/git.py > +++ b/bitbake/lib/bb/fetch2/git.py > @@ -65,6 +65,7 @@ import fnmatch > import os > import re > import shlex > +import shutil > import subprocess > import tempfile > import bb > @@ -365,8 +366,29 @@ class Git(FetchMethod): > runfetchcmd(fetch_cmd, d, workdir=ud.clonedir) > repourl = self._get_repo_url(ud) > > + needs_clone = False > + if os.path.exists(ud.clonedir): > + # The directory may exist, but not be the top level of a bare git > + # repository in which case it needs to be deleted and re-cloned. > + try: > + # Since clones are bare, use --absolute-git-dir instead of --show-toplevel > + output = runfetchcmd("LANG=C %s rev-parse --absolute-git-dir" % ud.basecmd, d, workdir=ud.clonedir) > + except bb.fetch2.FetchError as e: > + logger.warning("Unable to get top level for %s (not a git directory?): %s", ud.clonedir, e) > + needs_clone = True > + else: > + toplevel = output.rstrip() > + if os.path.abspath(toplevel) != os.path.abspath(ud.clonedir): > + logger.warning("Top level directory '%s' doesn't match expected '%s'. Re-cloning", toplevel, ud.clonedir) > + needs_clone = True > + > + if needs_clone: > + shutil.rmtree(ud.clonedir) > + else: > + needs_clone = True > + > # If the repo still doesn't exist, fallback to cloning it > - if not os.path.exists(ud.clonedir): > + if needs_clone: > # We do this since git will use a "-l" option automatically for local urls where possible, > # but it doesn't work when git/objects is a symlink, only works when it is a directory. > if repourl.startswith("file://"): > -- > 2.33.0 > > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#14915): https://lists.openembedded.org/g/bitbake-devel/message/14915 > Mute This Topic: https://lists.openembedded.org/mt/100548945/3617179 > Group Owner: bitbake-devel+owner@lists.openembedded.org > Unsubscribe: https://lists.openembedded.org/g/bitbake-devel/unsub [alexandre.belloni@bootlin.com] > -=-=-=-=-=-=-=-=-=-=-=- >
Alexandre, I'm not sure that's quite correct; I was able to run the test fine locally, and that failure looks like a 502 error from the server, which really shouldn't have anything to do with this patch? On Tue, Aug 8, 2023 at 7:52 PM Alexandre Belloni <alexandre.belloni@bootlin.com> wrote: > > Hello Joshua, > > I'm fairly sure this causes the following bitbake-selftest failure: > > https://autobuilder.yoctoproject.org/typhoon/#/builders/56/builds/2307/steps/11/logs/stdio > > On 04/08/2023 09:54:24-0600, Joshua Watt wrote: > > If the clone target directory exists, but isn't the valid top level of a > > bare git repo, it needs to be erased and re-cloned. One example of how > > this can happen is if a clone creates the directory, but then fails to > > actual clone and make it a git repository. This left-over directory can > > be particularly problematic if the download directory is a descent of > > some top layer git repo (e.g. the default with poky), as the commands > > that operate on the remote later will then mangle the layers git > > repository instead of the download git repo. > > > > Signed-off-by: Joshua Watt <JPEWhacker@gmail.com> > > --- > > bitbake/lib/bb/fetch2/git.py | 24 +++++++++++++++++++++++- > > 1 file changed, 23 insertions(+), 1 deletion(-) > > > > diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py > > index 2a3c06fe4e9..112bed294f0 100644 > > --- a/bitbake/lib/bb/fetch2/git.py > > +++ b/bitbake/lib/bb/fetch2/git.py > > @@ -65,6 +65,7 @@ import fnmatch > > import os > > import re > > import shlex > > +import shutil > > import subprocess > > import tempfile > > import bb > > @@ -365,8 +366,29 @@ class Git(FetchMethod): > > runfetchcmd(fetch_cmd, d, workdir=ud.clonedir) > > repourl = self._get_repo_url(ud) > > > > + needs_clone = False > > + if os.path.exists(ud.clonedir): > > + # The directory may exist, but not be the top level of a bare git > > + # repository in which case it needs to be deleted and re-cloned. > > + try: > > + # Since clones are bare, use --absolute-git-dir instead of --show-toplevel > > + output = runfetchcmd("LANG=C %s rev-parse --absolute-git-dir" % ud.basecmd, d, workdir=ud.clonedir) > > + except bb.fetch2.FetchError as e: > > + logger.warning("Unable to get top level for %s (not a git directory?): %s", ud.clonedir, e) > > + needs_clone = True > > + else: > > + toplevel = output.rstrip() > > + if os.path.abspath(toplevel) != os.path.abspath(ud.clonedir): > > + logger.warning("Top level directory '%s' doesn't match expected '%s'. Re-cloning", toplevel, ud.clonedir) > > + needs_clone = True > > + > > + if needs_clone: > > + shutil.rmtree(ud.clonedir) > > + else: > > + needs_clone = True > > + > > # If the repo still doesn't exist, fallback to cloning it > > - if not os.path.exists(ud.clonedir): > > + if needs_clone: > > # We do this since git will use a "-l" option automatically for local urls where possible, > > # but it doesn't work when git/objects is a symlink, only works when it is a directory. > > if repourl.startswith("file://"): > > -- > > 2.33.0 > > > > > > > -=-=-=-=-=-=-=-=-=-=-=- > > Links: You receive all messages sent to this group. > > View/Reply Online (#14915): https://lists.openembedded.org/g/bitbake-devel/message/14915 > > Mute This Topic: https://lists.openembedded.org/mt/100548945/3617179 > > Group Owner: bitbake-devel+owner@lists.openembedded.org > > Unsubscribe: https://lists.openembedded.org/g/bitbake-devel/unsub [alexandre.belloni@bootlin.com] > > -=-=-=-=-=-=-=-=-=-=-=- > > > > > -- > Alexandre Belloni, co-owner and COO, Bootlin > Embedded Linux and Kernel engineering > https://bootlin.com
On 17/08/2023 13:19:56-0600, Joshua Watt wrote: > Alexandre, > > I'm not sure that's quite correct; I was able to run the test fine > locally, and that failure looks like a 502 error from the server, > which really shouldn't have anything to do with this patch? > I'm pretty sure you are right... > On Tue, Aug 8, 2023 at 7:52 PM Alexandre Belloni > <alexandre.belloni@bootlin.com> wrote: > > > > Hello Joshua, > > > > I'm fairly sure this causes the following bitbake-selftest failure: > > > > https://autobuilder.yoctoproject.org/typhoon/#/builders/56/builds/2307/steps/11/logs/stdio > > > > On 04/08/2023 09:54:24-0600, Joshua Watt wrote: > > > If the clone target directory exists, but isn't the valid top level of a > > > bare git repo, it needs to be erased and re-cloned. One example of how > > > this can happen is if a clone creates the directory, but then fails to > > > actual clone and make it a git repository. This left-over directory can > > > be particularly problematic if the download directory is a descent of > > > some top layer git repo (e.g. the default with poky), as the commands > > > that operate on the remote later will then mangle the layers git > > > repository instead of the download git repo. > > > > > > Signed-off-by: Joshua Watt <JPEWhacker@gmail.com> > > > --- > > > bitbake/lib/bb/fetch2/git.py | 24 +++++++++++++++++++++++- > > > 1 file changed, 23 insertions(+), 1 deletion(-) > > > > > > diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py > > > index 2a3c06fe4e9..112bed294f0 100644 > > > --- a/bitbake/lib/bb/fetch2/git.py > > > +++ b/bitbake/lib/bb/fetch2/git.py > > > @@ -65,6 +65,7 @@ import fnmatch > > > import os > > > import re > > > import shlex > > > +import shutil > > > import subprocess > > > import tempfile > > > import bb > > > @@ -365,8 +366,29 @@ class Git(FetchMethod): > > > runfetchcmd(fetch_cmd, d, workdir=ud.clonedir) > > > repourl = self._get_repo_url(ud) > > > > > > + needs_clone = False > > > + if os.path.exists(ud.clonedir): > > > + # The directory may exist, but not be the top level of a bare git > > > + # repository in which case it needs to be deleted and re-cloned. > > > + try: > > > + # Since clones are bare, use --absolute-git-dir instead of --show-toplevel > > > + output = runfetchcmd("LANG=C %s rev-parse --absolute-git-dir" % ud.basecmd, d, workdir=ud.clonedir) > > > + except bb.fetch2.FetchError as e: > > > + logger.warning("Unable to get top level for %s (not a git directory?): %s", ud.clonedir, e) > > > + needs_clone = True > > > + else: > > > + toplevel = output.rstrip() > > > + if os.path.abspath(toplevel) != os.path.abspath(ud.clonedir): > > > + logger.warning("Top level directory '%s' doesn't match expected '%s'. Re-cloning", toplevel, ud.clonedir) > > > + needs_clone = True > > > + > > > + if needs_clone: > > > + shutil.rmtree(ud.clonedir) > > > + else: > > > + needs_clone = True > > > + > > > # If the repo still doesn't exist, fallback to cloning it > > > - if not os.path.exists(ud.clonedir): > > > + if needs_clone: > > > # We do this since git will use a "-l" option automatically for local urls where possible, > > > # but it doesn't work when git/objects is a symlink, only works when it is a directory. > > > if repourl.startswith("file://"): > > > -- > > > 2.33.0 > > > > > > > > > > > -=-=-=-=-=-=-=-=-=-=-=- > > > Links: You receive all messages sent to this group. > > > View/Reply Online (#14915): https://lists.openembedded.org/g/bitbake-devel/message/14915 > > > Mute This Topic: https://lists.openembedded.org/mt/100548945/3617179 > > > Group Owner: bitbake-devel+owner@lists.openembedded.org > > > Unsubscribe: https://lists.openembedded.org/g/bitbake-devel/unsub [alexandre.belloni@bootlin.com] > > > -=-=-=-=-=-=-=-=-=-=-=- > > > > > > > > > -- > > Alexandre Belloni, co-owner and COO, Bootlin > > Embedded Linux and Kernel engineering > > https://bootlin.com
This was because I didn't point to the correct build error:
======================================================================
ERROR: test_mirror_commit_exists (bb.tests.fetch.FetchPremirroronlyLocalTest.test_mirror_commit_exists)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/bitbake/lib/bb/tests/fetch.py", line 3075, in test_mirror_commit_exists
fetcher.download()
File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/bitbake/lib/bb/fetch2/__init__.py", line 1746, in download
done = m.try_mirrors(self, ud, self.d, mirrors)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/bitbake/lib/bb/fetch2/__init__.py", line 1612, in try_mirrors
return bool(try_mirrors(fetch, d, urldata, mirrors, check))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/bitbake/lib/bb/fetch2/__init__.py", line 1136, in try_mirrors
ret = try_mirror_url(fetch, origud, uds[index], ld, check)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/bitbake/lib/bb/fetch2/__init__.py", line 1065, in try_mirror_url
origud.method.download(origud, ld)
File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/bitbake/lib/bb/fetch2/git.py", line 401, in download
bb.fetch2.check_network_access(d, clone_cmd, ud.url)
File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/bitbake/lib/bb/fetch2/__init__.py", line 944, in check_network_access
raise NetworkAccess(url, info)
bb.fetch2.NetworkAccess: Network access disabled through BB_NO_NETWORK (or set indirectly due to use of BB_FETCH_PREMIRRORONLY) but access requested with command LANG=C git -c gc.autoDetach=false -c core.pager=cat clone --bare --mirror https://git.fake.repo/bitbake /tmp/bitbake-fetch-xeon98ff/download/git2/git.fake.repo.bitbake --progress (for url git://git.fake.repo/bitbake;branch=master;protocol=https)
https://autobuilder.yoctoproject.org/typhoon/#/builders/86/builds/5619/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/80/builds/5577/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/127/builds/1935/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/87/builds/5653/steps/11/logs/stdio
On 17/08/2023 22:14:34+0200, Alexandre Belloni via lists.openembedded.org wrote:
> On 17/08/2023 13:19:56-0600, Joshua Watt wrote:
> > Alexandre,
> >
> > I'm not sure that's quite correct; I was able to run the test fine
> > locally, and that failure looks like a 502 error from the server,
> > which really shouldn't have anything to do with this patch?
> >
>
> I'm pretty sure you are right...
>
> > On Tue, Aug 8, 2023 at 7:52 PM Alexandre Belloni
> > <alexandre.belloni@bootlin.com> wrote:
> > >
> > > Hello Joshua,
> > >
> > > I'm fairly sure this causes the following bitbake-selftest failure:
> > >
> > > https://autobuilder.yoctoproject.org/typhoon/#/builders/56/builds/2307/steps/11/logs/stdio
> > >
> > > On 04/08/2023 09:54:24-0600, Joshua Watt wrote:
> > > > If the clone target directory exists, but isn't the valid top level of a
> > > > bare git repo, it needs to be erased and re-cloned. One example of how
> > > > this can happen is if a clone creates the directory, but then fails to
> > > > actual clone and make it a git repository. This left-over directory can
> > > > be particularly problematic if the download directory is a descent of
> > > > some top layer git repo (e.g. the default with poky), as the commands
> > > > that operate on the remote later will then mangle the layers git
> > > > repository instead of the download git repo.
> > > >
> > > > Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
> > > > ---
> > > > bitbake/lib/bb/fetch2/git.py | 24 +++++++++++++++++++++++-
> > > > 1 file changed, 23 insertions(+), 1 deletion(-)
> > > >
> > > > diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
> > > > index 2a3c06fe4e9..112bed294f0 100644
> > > > --- a/bitbake/lib/bb/fetch2/git.py
> > > > +++ b/bitbake/lib/bb/fetch2/git.py
> > > > @@ -65,6 +65,7 @@ import fnmatch
> > > > import os
> > > > import re
> > > > import shlex
> > > > +import shutil
> > > > import subprocess
> > > > import tempfile
> > > > import bb
> > > > @@ -365,8 +366,29 @@ class Git(FetchMethod):
> > > > runfetchcmd(fetch_cmd, d, workdir=ud.clonedir)
> > > > repourl = self._get_repo_url(ud)
> > > >
> > > > + needs_clone = False
> > > > + if os.path.exists(ud.clonedir):
> > > > + # The directory may exist, but not be the top level of a bare git
> > > > + # repository in which case it needs to be deleted and re-cloned.
> > > > + try:
> > > > + # Since clones are bare, use --absolute-git-dir instead of --show-toplevel
> > > > + output = runfetchcmd("LANG=C %s rev-parse --absolute-git-dir" % ud.basecmd, d, workdir=ud.clonedir)
> > > > + except bb.fetch2.FetchError as e:
> > > > + logger.warning("Unable to get top level for %s (not a git directory?): %s", ud.clonedir, e)
> > > > + needs_clone = True
> > > > + else:
> > > > + toplevel = output.rstrip()
> > > > + if os.path.abspath(toplevel) != os.path.abspath(ud.clonedir):
> > > > + logger.warning("Top level directory '%s' doesn't match expected '%s'. Re-cloning", toplevel, ud.clonedir)
> > > > + needs_clone = True
> > > > +
> > > > + if needs_clone:
> > > > + shutil.rmtree(ud.clonedir)
> > > > + else:
> > > > + needs_clone = True
> > > > +
> > > > # If the repo still doesn't exist, fallback to cloning it
> > > > - if not os.path.exists(ud.clonedir):
> > > > + if needs_clone:
> > > > # We do this since git will use a "-l" option automatically for local urls where possible,
> > > > # but it doesn't work when git/objects is a symlink, only works when it is a directory.
> > > > if repourl.startswith("file://"):
> > > > --
> > > > 2.33.0
> > > >
> > >
> > > >
> > > >
> > > >
> > >
> > >
> > > --
> > > Alexandre Belloni, co-owner and COO, Bootlin
> > > Embedded Linux and Kernel engineering
> > > https://bootlin.com
>
> --
> Alexandre Belloni, co-owner and COO, Bootlin
> Embedded Linux and Kernel engineering
> https://bootlin.com
>
> -=-=-=-=-=-=-=-=-=-=-=-
> Links: You receive all messages sent to this group.
> View/Reply Online (#14952): https://lists.openembedded.org/g/bitbake-devel/message/14952
> Mute This Topic: https://lists.openembedded.org/mt/100548945/3617179
> Group Owner: bitbake-devel+owner@lists.openembedded.org
> Unsubscribe: https://lists.openembedded.org/g/bitbake-devel/unsub [alexandre.belloni@bootlin.com]
> -=-=-=-=-=-=-=-=-=-=-=-
>
On Sat, Aug 19, 2023 at 6:21 AM Alexandre Belloni <alexandre.belloni@bootlin.com> wrote: > > This was because I didn't point to the correct build error: Yep, fixed in V2 (and I validated that the tests pass locally also) > > ====================================================================== > ERROR: test_mirror_commit_exists (bb.tests.fetch.FetchPremirroronlyLocalTest.test_mirror_commit_exists) > ---------------------------------------------------------------------- > Traceback (most recent call last): > File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/bitbake/lib/bb/tests/fetch.py", line 3075, in test_mirror_commit_exists > fetcher.download() > File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/bitbake/lib/bb/fetch2/__init__.py", line 1746, in download > done = m.try_mirrors(self, ud, self.d, mirrors) > ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ > File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/bitbake/lib/bb/fetch2/__init__.py", line 1612, in try_mirrors > return bool(try_mirrors(fetch, d, urldata, mirrors, check)) > ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ > File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/bitbake/lib/bb/fetch2/__init__.py", line 1136, in try_mirrors > ret = try_mirror_url(fetch, origud, uds[index], ld, check) > ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ > File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/bitbake/lib/bb/fetch2/__init__.py", line 1065, in try_mirror_url > origud.method.download(origud, ld) > File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/bitbake/lib/bb/fetch2/git.py", line 401, in download > bb.fetch2.check_network_access(d, clone_cmd, ud.url) > File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/bitbake/lib/bb/fetch2/__init__.py", line 944, in check_network_access > raise NetworkAccess(url, info) > bb.fetch2.NetworkAccess: Network access disabled through BB_NO_NETWORK (or set indirectly due to use of BB_FETCH_PREMIRRORONLY) but access requested with command LANG=C git -c gc.autoDetach=false -c core.pager=cat clone --bare --mirror https://git.fake.repo/bitbake /tmp/bitbake-fetch-xeon98ff/download/git2/git.fake.repo.bitbake --progress (for url git://git.fake.repo/bitbake;branch=master;protocol=https) > > > https://autobuilder.yoctoproject.org/typhoon/#/builders/86/builds/5619/steps/11/logs/stdio > https://autobuilder.yoctoproject.org/typhoon/#/builders/80/builds/5577/steps/11/logs/stdio > https://autobuilder.yoctoproject.org/typhoon/#/builders/127/builds/1935/steps/11/logs/stdio > https://autobuilder.yoctoproject.org/typhoon/#/builders/87/builds/5653/steps/11/logs/stdio > > > On 17/08/2023 22:14:34+0200, Alexandre Belloni via lists.openembedded.org wrote: > > On 17/08/2023 13:19:56-0600, Joshua Watt wrote: > > > Alexandre, > > > > > > I'm not sure that's quite correct; I was able to run the test fine > > > locally, and that failure looks like a 502 error from the server, > > > which really shouldn't have anything to do with this patch? > > > > > > > I'm pretty sure you are right... > > > > > On Tue, Aug 8, 2023 at 7:52 PM Alexandre Belloni > > > <alexandre.belloni@bootlin.com> wrote: > > > > > > > > Hello Joshua, > > > > > > > > I'm fairly sure this causes the following bitbake-selftest failure: > > > > > > > > https://autobuilder.yoctoproject.org/typhoon/#/builders/56/builds/2307/steps/11/logs/stdio > > > > > > > > On 04/08/2023 09:54:24-0600, Joshua Watt wrote: > > > > > If the clone target directory exists, but isn't the valid top level of a > > > > > bare git repo, it needs to be erased and re-cloned. One example of how > > > > > this can happen is if a clone creates the directory, but then fails to > > > > > actual clone and make it a git repository. This left-over directory can > > > > > be particularly problematic if the download directory is a descent of > > > > > some top layer git repo (e.g. the default with poky), as the commands > > > > > that operate on the remote later will then mangle the layers git > > > > > repository instead of the download git repo. > > > > > > > > > > Signed-off-by: Joshua Watt <JPEWhacker@gmail.com> > > > > > --- > > > > > bitbake/lib/bb/fetch2/git.py | 24 +++++++++++++++++++++++- > > > > > 1 file changed, 23 insertions(+), 1 deletion(-) > > > > > > > > > > diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py > > > > > index 2a3c06fe4e9..112bed294f0 100644 > > > > > --- a/bitbake/lib/bb/fetch2/git.py > > > > > +++ b/bitbake/lib/bb/fetch2/git.py > > > > > @@ -65,6 +65,7 @@ import fnmatch > > > > > import os > > > > > import re > > > > > import shlex > > > > > +import shutil > > > > > import subprocess > > > > > import tempfile > > > > > import bb > > > > > @@ -365,8 +366,29 @@ class Git(FetchMethod): > > > > > runfetchcmd(fetch_cmd, d, workdir=ud.clonedir) > > > > > repourl = self._get_repo_url(ud) > > > > > > > > > > + needs_clone = False > > > > > + if os.path.exists(ud.clonedir): > > > > > + # The directory may exist, but not be the top level of a bare git > > > > > + # repository in which case it needs to be deleted and re-cloned. > > > > > + try: > > > > > + # Since clones are bare, use --absolute-git-dir instead of --show-toplevel > > > > > + output = runfetchcmd("LANG=C %s rev-parse --absolute-git-dir" % ud.basecmd, d, workdir=ud.clonedir) > > > > > + except bb.fetch2.FetchError as e: > > > > > + logger.warning("Unable to get top level for %s (not a git directory?): %s", ud.clonedir, e) > > > > > + needs_clone = True > > > > > + else: > > > > > + toplevel = output.rstrip() > > > > > + if os.path.abspath(toplevel) != os.path.abspath(ud.clonedir): > > > > > + logger.warning("Top level directory '%s' doesn't match expected '%s'. Re-cloning", toplevel, ud.clonedir) > > > > > + needs_clone = True > > > > > + > > > > > + if needs_clone: > > > > > + shutil.rmtree(ud.clonedir) > > > > > + else: > > > > > + needs_clone = True > > > > > + > > > > > # If the repo still doesn't exist, fallback to cloning it > > > > > - if not os.path.exists(ud.clonedir): > > > > > + if needs_clone: > > > > > # We do this since git will use a "-l" option automatically for local urls where possible, > > > > > # but it doesn't work when git/objects is a symlink, only works when it is a directory. > > > > > if repourl.startswith("file://"): > > > > > -- > > > > > 2.33.0 > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > -- > > > > Alexandre Belloni, co-owner and COO, Bootlin > > > > Embedded Linux and Kernel engineering > > > > https://bootlin.com > > > > -- > > Alexandre Belloni, co-owner and COO, Bootlin > > Embedded Linux and Kernel engineering > > https://bootlin.com > > > > > -=-=-=-=-=-=-=-=-=-=-=- > > Links: You receive all messages sent to this group. > > View/Reply Online (#14952): https://lists.openembedded.org/g/bitbake-devel/message/14952 > > Mute This Topic: https://lists.openembedded.org/mt/100548945/3617179 > > Group Owner: bitbake-devel+owner@lists.openembedded.org > > Unsubscribe: https://lists.openembedded.org/g/bitbake-devel/unsub [alexandre.belloni@bootlin.com] > > -=-=-=-=-=-=-=-=-=-=-=- > > > > > -- > Alexandre Belloni, co-owner and COO, Bootlin > Embedded Linux and Kernel engineering > https://bootlin.com
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py index 2a3c06fe4e9..112bed294f0 100644 --- a/bitbake/lib/bb/fetch2/git.py +++ b/bitbake/lib/bb/fetch2/git.py @@ -65,6 +65,7 @@ import fnmatch import os import re import shlex +import shutil import subprocess import tempfile import bb @@ -365,8 +366,29 @@ class Git(FetchMethod): runfetchcmd(fetch_cmd, d, workdir=ud.clonedir) repourl = self._get_repo_url(ud) + needs_clone = False + if os.path.exists(ud.clonedir): + # The directory may exist, but not be the top level of a bare git + # repository in which case it needs to be deleted and re-cloned. + try: + # Since clones are bare, use --absolute-git-dir instead of --show-toplevel + output = runfetchcmd("LANG=C %s rev-parse --absolute-git-dir" % ud.basecmd, d, workdir=ud.clonedir) + except bb.fetch2.FetchError as e: + logger.warning("Unable to get top level for %s (not a git directory?): %s", ud.clonedir, e) + needs_clone = True + else: + toplevel = output.rstrip() + if os.path.abspath(toplevel) != os.path.abspath(ud.clonedir): + logger.warning("Top level directory '%s' doesn't match expected '%s'. Re-cloning", toplevel, ud.clonedir) + needs_clone = True + + if needs_clone: + shutil.rmtree(ud.clonedir) + else: + needs_clone = True + # If the repo still doesn't exist, fallback to cloning it - if not os.path.exists(ud.clonedir): + if needs_clone: # We do this since git will use a "-l" option automatically for local urls where possible, # but it doesn't work when git/objects is a symlink, only works when it is a directory. if repourl.startswith("file://"):
If the clone target directory exists, but isn't the valid top level of a bare git repo, it needs to be erased and re-cloned. One example of how this can happen is if a clone creates the directory, but then fails to actual clone and make it a git repository. This left-over directory can be particularly problematic if the download directory is a descent of some top layer git repo (e.g. the default with poky), as the commands that operate on the remote later will then mangle the layers git repository instead of the download git repo. Signed-off-by: Joshua Watt <JPEWhacker@gmail.com> --- bitbake/lib/bb/fetch2/git.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-)