diff mbox series

[1/1] fetch2/git: Fix File name too long error

Message ID 02074341b8914bf7d1bf44291ec50a77853a4c5b.1712565610.git.liezhi.yang@windriver.com
State New
Headers show
Series [1/1] fetch2/git: Fix File name too long error | expand

Commit Message

Robert Yang April 8, 2024, 8:42 a.m. UTC 
From: Robert Yang <liezhi.yang@windriver.com>

There are "File name too long" errors when local PREMIRROR/MIRROR
is in a deep directory (len(path) > 255 (The NAME_MAX)).

This is becuase ud.path.replace() changes the path to filename, and
the errors when happens when len(path) > 255.

Use hashlib.md5(ud.path.encode('utf-8')).hexdigest() can fix the problem and
avoid potential file name conflicts, now the files are like:

$ bitbake mtd-utils-native -cfetch
$ ls downloads/git2
1625d71bc771b3c4d479337faeef324f.git.infradead.org.mtd-utils.git
git.infradead.org.mtd-utils.git -> /path/to/build/downloads/git2/1625d71bc771b3c4d479337faeef324f.git.infradead.org.mtd-utils.git
git.infradead.org.mtd-utils.git.done

The 1625d71bc771b3c4d479337faeef324f.git.infradead.org.mtd-utils.git is gitsrcname.

Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
---
 bitbake/lib/bb/fetch2/git.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

Comments

Alexandre Belloni April 10, 2024, 5:01 p.m. UTC | #1 
I believe this causes the following error:

======================================================================
FAIL: test_shallow_submodules (bb.tests.fetch.GitShallowTest.test_shallow_submodules)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/bitbake/lib/bb/tests/fetch.py", line 1946, in test_shallow_submodules
    self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule'))
  File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/bitbake/lib/bb/tests/fetch.py", line 1728, in assertRevCount
    self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count))
AssertionError: 1 != 2 : Object count `2` is not the expected `1`

https://autobuilder.yoctoproject.org/typhoon/#/builders/86/builds/6634/steps/11/logs/stdio

On 08/04/2024 01:42:58-0700, Robert Yang via lists.openembedded.org wrote:
> From: Robert Yang <liezhi.yang@windriver.com>
> 
> There are "File name too long" errors when local PREMIRROR/MIRROR
> is in a deep directory (len(path) > 255 (The NAME_MAX)).
> 
> This is becuase ud.path.replace() changes the path to filename, and
> the errors when happens when len(path) > 255.
> 
> Use hashlib.md5(ud.path.encode('utf-8')).hexdigest() can fix the problem and
> avoid potential file name conflicts, now the files are like:
> 
> $ bitbake mtd-utils-native -cfetch
> $ ls downloads/git2
> 1625d71bc771b3c4d479337faeef324f.git.infradead.org.mtd-utils.git
> git.infradead.org.mtd-utils.git -> /path/to/build/downloads/git2/1625d71bc771b3c4d479337faeef324f.git.infradead.org.mtd-utils.git
> git.infradead.org.mtd-utils.git.done
> 
> The 1625d71bc771b3c4d479337faeef324f.git.infradead.org.mtd-utils.git is gitsrcname.
> 
> Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
> ---
>  bitbake/lib/bb/fetch2/git.py | 6 +++++-
>  1 file changed, 5 insertions(+), 1 deletion(-)
> 
> diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
> index c7ff769fdfe..7ed51af336d 100644
> --- a/bitbake/lib/bb/fetch2/git.py
> +++ b/bitbake/lib/bb/fetch2/git.py
> @@ -83,6 +83,7 @@ import subprocess
>  import tempfile
>  import bb
>  import bb.progress
> +import hashlib
>  from contextlib import contextmanager
>  from   bb.fetch2 import FetchMethod
>  from   bb.fetch2 import runfetchcmd
> @@ -277,7 +278,10 @@ class Git(FetchMethod):
>                      ud.unresolvedrev[name] = ud.revisions[name]
>                  ud.revisions[name] = self.latest_revision(ud, d, name)
>  
> -        gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_').replace('(', '_').replace(')', '_'))
> +        if ud.proto == "file":
> +            gitsrcname = '%s.%s' % (hashlib.md5(ud.path.encode('utf-8')).hexdigest(), os.path.basename(ud.path))
> +        else:
> +            gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_').replace('(', '_').replace(')', '_'))
>          if gitsrcname.startswith('.'):
>              gitsrcname = gitsrcname[1:]
>  
> -- 
> 2.42.0
> 

> 
> -=-=-=-=-=-=-=-=-=-=-=-
> Links: You receive all messages sent to this group.
> View/Reply Online (#16064): https://lists.openembedded.org/g/bitbake-devel/message/16064
> Mute This Topic: https://lists.openembedded.org/mt/105397591/3617179
> Group Owner: bitbake-devel+owner@lists.openembedded.org
> Unsubscribe: https://lists.openembedded.org/g/bitbake-devel/unsub [alexandre.belloni@bootlin.com]
> -=-=-=-=-=-=-=-=-=-=-=-
>
diff mbox series

Patch

diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index c7ff769fdfe..7ed51af336d 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -83,6 +83,7 @@  import subprocess
 import tempfile
 import bb
 import bb.progress
+import hashlib
 from contextlib import contextmanager
 from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import runfetchcmd
@@ -277,7 +278,10 @@  class Git(FetchMethod):
                     ud.unresolvedrev[name] = ud.revisions[name]
                 ud.revisions[name] = self.latest_revision(ud, d, name)
 
-        gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_').replace('(', '_').replace(')', '_'))
+        if ud.proto == "file":
+            gitsrcname = '%s.%s' % (hashlib.md5(ud.path.encode('utf-8')).hexdigest(), os.path.basename(ud.path))
+        else:
+            gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_').replace('(', '_').replace(')', '_'))
         if gitsrcname.startswith('.'):
             gitsrcname = gitsrcname[1:]