| Message ID | 1340018907-8814-1-git-send-email-rep.dot.nop@gmail.com |
|---|---|
| State | New |
| Headers | show |
diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py index 324eef2..0c0d60d 100644 --- a/lib/bb/fetch2/__init__.py +++ b/lib/bb/fetch2/__init__.py @@ -601,6 +601,7 @@ def get_file_checksums(filelist, pn): """ def checksum_file(f): + if not os.path.isfile(f): return None try: checksum = _checksum_cache.get_checksum(f) except OSError as e:
On Mon, 2012-06-18 at 13:28 +0200, Bernhard Reutner-Fischer wrote: > computing the md5 of a socket does not work too well. > > Traceback (most recent call last): > File "/scratch/src/oe/bitbake/build/lib/bb/siggen.py", line 172, in SignatureGeneratorOEBasic.get_taskhash(fn='/scratch/src/oe/openembedded-core/meta/recipes-core/uclibc/uclibc_git.bb', task='do_fetch', deps=[], dataCache=<bb.cache.CacheData object at 0xd95890>): > if task in dataCache.file_checksums[fn]: > > checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename) > for (f,cs) in checksums: > File "/scratch/src/oe/bitbake/build/lib/bb/fetch2/__init__.py", line 627, in get_file_checksums(filelist='/scratch/src/uClibc/ /scratch/src/oe/openembedded-core/meta/recipes-core/uclibc/uclibc-git/uClibc.machine /scratch/src/oe/openembedded-core/meta/recipes-core/uclibc/uclibc-git/uClibc.distro', pn='uclibc'): > fullpth = os.path.join(root, name) > > checksum = checksum_file(fullpth) > if checksum: > File "/scratch/src/oe/bitbake/build/lib/bb/fetch2/__init__.py", line 605, in checksum_file(f='/scratch/src/uClibc/test/inet/socktest'): > try: > > checksum = _checksum_cache.get_checksum(f) > except OSError as e: > File "/scratch/src/oe/bitbake/build/lib/bb/checksum.py", line 78, in FileChecksumCache.get_checksum(f='/scratch/src/uClibc/test/inet/socktest'): > > > hashval = bb.utils.md5_file(f) > self.cachedata_extras[0][f] = (cmtime, hashval) > File "/scratch/src/oe/bitbake/build/lib/bb/utils.py", line 362, in md5_file(filename='/scratch/src/uClibc/test/inet/socktest'): > > > for line in open(filename): > m.update(line) > IOError: [Errno 6] No such device or address: '/scratch/src/uClibc/test/inet/socktest' Can you give some details on how to reproduce this? I'm guessing this is due to some kind of proxy configuration? What is worrying me a bit is that we shouldn't be calling the checksum code for git repositories at all. Whilst this fix no doubt avoids the problem, I'm worried there might be something deeper going on... Cheers, Richard > Signed-off-by: Bernhard Reutner-Fischer <rep.dot.nop@gmail.com> > --- > lib/bb/fetch2/__init__.py | 1 + > 1 file changed, 1 insertion(+) > > diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py > index 324eef2..0c0d60d 100644 > --- a/lib/bb/fetch2/__init__.py > +++ b/lib/bb/fetch2/__init__.py > @@ -601,6 +601,7 @@ def get_file_checksums(filelist, pn): > """ > > def checksum_file(f): > + if not os.path.isfile(f): return None > try: > checksum = _checksum_cache.get_checksum(f) > except OSError as e:
On 21 June 2012 14:16, Richard Purdie <richard.purdie@linuxfoundation.org> wrote: > On Mon, 2012-06-18 at 13:28 +0200, Bernhard Reutner-Fischer wrote: >> computing the md5 of a socket does not work too well. >> >> Traceback (most recent call last): >> File "/scratch/src/oe/bitbake/build/lib/bb/siggen.py", line 172, in SignatureGeneratorOEBasic.get_taskhash(fn='/scratch/src/oe/openembedded-core/meta/recipes-core/uclibc/uclibc_git.bb', task='do_fetch', deps=[], dataCache=<bb.cache.CacheData object at 0xd95890>): >> if task in dataCache.file_checksums[fn]: >> > checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename) >> for (f,cs) in checksums: >> File "/scratch/src/oe/bitbake/build/lib/bb/fetch2/__init__.py", line 627, in get_file_checksums(filelist='/scratch/src/uClibc/ /scratch/src/oe/openembedded-core/meta/recipes-core/uclibc/uclibc-git/uClibc.machine /scratch/src/oe/openembedded-core/meta/recipes-core/uclibc/uclibc-git/uClibc.distro', pn='uclibc'): >> fullpth = os.path.join(root, name) >> > checksum = checksum_file(fullpth) >> if checksum: >> File "/scratch/src/oe/bitbake/build/lib/bb/fetch2/__init__.py", line 605, in checksum_file(f='/scratch/src/uClibc/test/inet/socktest'): >> try: >> > checksum = _checksum_cache.get_checksum(f) >> except OSError as e: >> File "/scratch/src/oe/bitbake/build/lib/bb/checksum.py", line 78, in FileChecksumCache.get_checksum(f='/scratch/src/uClibc/test/inet/socktest'): >> >> > hashval = bb.utils.md5_file(f) >> self.cachedata_extras[0][f] = (cmtime, hashval) >> File "/scratch/src/oe/bitbake/build/lib/bb/utils.py", line 362, in md5_file(filename='/scratch/src/uClibc/test/inet/socktest'): >> >> > for line in open(filename): >> m.update(line) >> IOError: [Errno 6] No such device or address: '/scratch/src/uClibc/test/inet/socktest' > > Can you give some details on how to reproduce this? I'm guessing this is > due to some kind of proxy configuration? I'm using an rsync of my local clone, like (The only change i have in an otherwise pristine oe-core): $ sed -e "/#/d;/^[[:space:]]*$/d" meta/recipes-core/uclibc/uclibc_git.bb require uclibc.inc DEFAULT_PREFERENCE = "9" PV = "0.9.33+git0" PR = "${INC_PR}.1" PROVIDES += "virtual/${TARGET_PREFIX}libc-for-gcc" FILESPATH = "${@base_set_filespath([ '${FILE_DIRNAME}/uclibc-git' ], d)}" SRC_URI = "file:///scratch/src/uClibc/;branch=master;protocol=rsync;rebaseable=1 \ file://uClibc.machine \ file://uClibc.distro \ " S = "${WORKDIR}/uClibc" > > What is worrying me a bit is that we shouldn't be calling the checksum > code for git repositories at all. Whilst this fix no doubt avoids the > problem, I'm worried there might be something deeper going on... Could it be that the logic to deduce the fetcher is somehow broken? Indeed, this was failing for me a couple of weeks ago with a similar SRC_URI. Back then i could not use file:///mydir;protocol=git (i.e. create a clone from a clone living on the local filesystem) so had to resort back to protocol=rsync. There was a socket lying around in that directory, a leftover from some testsuite run. Good point of yours but i do not currently have time to dig into this, sorry :-( so please ignore the patch! cheers, > > Cheers, > > Richard > >> Signed-off-by: Bernhard Reutner-Fischer <rep.dot.nop@gmail.com> >> --- >> lib/bb/fetch2/__init__.py | 1 + >> 1 file changed, 1 insertion(+) >> >> diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py >> index 324eef2..0c0d60d 100644 >> --- a/lib/bb/fetch2/__init__.py >> +++ b/lib/bb/fetch2/__init__.py >> @@ -601,6 +601,7 @@ def get_file_checksums(filelist, pn): >> """ >> >> def checksum_file(f): >> + if not os.path.isfile(f): return None >> try: >> checksum = _checksum_cache.get_checksum(f) >> except OSError as e: > >
On Thu, 2012-06-21 at 15:08 +0200, Bernhard Reutner-Fischer wrote: > On 21 June 2012 14:16, Richard Purdie > <richard.purdie@linuxfoundation.org> wrote: > > On Mon, 2012-06-18 at 13:28 +0200, Bernhard Reutner-Fischer wrote: > >> computing the md5 of a socket does not work too well. > >> > >> Traceback (most recent call last): > >> File "/scratch/src/oe/bitbake/build/lib/bb/siggen.py", line 172, in SignatureGeneratorOEBasic.get_taskhash(fn='/scratch/src/oe/openembedded-core/meta/recipes-core/uclibc/uclibc_git.bb', task='do_fetch', deps=[], dataCache=<bb.cache.CacheData object at 0xd95890>): > >> if task in dataCache.file_checksums[fn]: > >> > checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename) > >> for (f,cs) in checksums: > >> File "/scratch/src/oe/bitbake/build/lib/bb/fetch2/__init__.py", line 627, in get_file_checksums(filelist='/scratch/src/uClibc/ /scratch/src/oe/openembedded-core/meta/recipes-core/uclibc/uclibc-git/uClibc.machine /scratch/src/oe/openembedded-core/meta/recipes-core/uclibc/uclibc-git/uClibc.distro', pn='uclibc'): > >> fullpth = os.path.join(root, name) > >> > checksum = checksum_file(fullpth) > >> if checksum: > >> File "/scratch/src/oe/bitbake/build/lib/bb/fetch2/__init__.py", line 605, in checksum_file(f='/scratch/src/uClibc/test/inet/socktest'): > >> try: > >> > checksum = _checksum_cache.get_checksum(f) > >> except OSError as e: > >> File "/scratch/src/oe/bitbake/build/lib/bb/checksum.py", line 78, in FileChecksumCache.get_checksum(f='/scratch/src/uClibc/test/inet/socktest'): > >> > >> > hashval = bb.utils.md5_file(f) > >> self.cachedata_extras[0][f] = (cmtime, hashval) > >> File "/scratch/src/oe/bitbake/build/lib/bb/utils.py", line 362, in md5_file(filename='/scratch/src/uClibc/test/inet/socktest'): > >> > >> > for line in open(filename): > >> m.update(line) > >> IOError: [Errno 6] No such device or address: '/scratch/src/uClibc/test/inet/socktest' > > > > Can you give some details on how to reproduce this? I'm guessing this is > > due to some kind of proxy configuration? > > I'm using an rsync of my local clone, like (The only change i have in > an otherwise pristine oe-core): > $ sed -e "/#/d;/^[[:space:]]*$/d" meta/recipes-core/uclibc/uclibc_git.bb > require uclibc.inc > DEFAULT_PREFERENCE = "9" > PV = "0.9.33+git0" > PR = "${INC_PR}.1" > PROVIDES += "virtual/${TARGET_PREFIX}libc-for-gcc" > FILESPATH = "${@base_set_filespath([ '${FILE_DIRNAME}/uclibc-git' ], d)}" > SRC_URI = "file:///scratch/src/uClibc/;branch=master;protocol=rsync;rebaseable=1 > \ > file://uClibc.machine \ > file://uClibc.distro \ > " > S = "${WORKDIR}/uClibc" Ah, this makes more sense now. For what its worth, the "file://" url handler will simply strip off everything including and after the ";" so all it really sees is "file:///scratch/src/uClibc/" but this works since it will just copy that directory whole. It will then attempt to checksum everything in a file:// url which is why it finds the socket and then fails. So your change does point at a potentially valid issue. Paul might have a better idea on which level we might want to fix this at? > > What is worrying me a bit is that we shouldn't be calling the checksum > > code for git repositories at all. Whilst this fix no doubt avoids the > > problem, I'm worried there might be something deeper going on... > > Could it be that the logic to deduce the fetcher is somehow broken? > > Indeed, this was failing for me a couple of weeks ago with a similar SRC_URI. > Back then i could not use file:///mydir;protocol=git (i.e. create a > clone from a clone living on the local filesystem) so had to resort > back to protocol=rsync. > There was a socket lying around in that directory, a leftover from > some testsuite run. FWIW, the url you want is git:///mydir;protocol=file which should work. As of the patches today, this also works as mirror syntax so you could even add this in local.conf as a PREMIRROR and leave the recipe pristine. Cheers, Richard
On Thursday 21 June 2012 14:25:56 Richard Purdie wrote: > It will then attempt to checksum everything in a file:// url which is > why it finds the socket and then fails. So your change does point at a > potentially valid issue. Paul might have a better idea on which level we > might want to fix this at? Regarding the patch in question, I'm not opposed to a change to avoid errors like this, however it looks to me that if the file is missing the os.path.isfile() call will return false and bypass the warning that we currently show, which would not be desirable. I think it would need to do something like call os.stat() and then interpret the results, then we're just making one call. > > Indeed, this was failing for me a couple of weeks ago with a similar > > SRC_URI. Back then i could not use file:///mydir;protocol=git (i.e. > > create a clone from a clone living on the local filesystem) so had to > > resort back to protocol=rsync. > > There was a socket lying around in that directory, a leftover from > > some testsuite run. > > FWIW, the url you want is git:///mydir;protocol=file which should work. > As of the patches today, this also works as mirror syntax so you could > even add this in local.conf as a PREMIRROR and leave the recipe > pristine. Also, if what you want is to have a local source directory you can modify for development purposes, you may find the recently added externalsrc.bbclass is helpful since that's what it was designed to handle. I guess it depends on what you're trying to achieve and what your workflow is. Cheers, Paul
>> SRC_URI = "file:///scratch/src/uClibc/;branch=master;protocol=rsync;rebaseable=1 > Ah, this makes more sense now. For what its worth, the "file://" url > handler will simply strip off everything including and after the ";" so > all it really sees is "file:///scratch/src/uClibc/" but this works since > it will just copy that directory whole. > > It will then attempt to checksum everything in a file:// url which is > why it finds the socket and then fails. So your change does point at a > potentially valid issue. Paul might have a better idea on which level we > might want to fix this at? > >> > What is worrying me a bit is that we shouldn't be calling the checksum >> > code for git repositories at all. Whilst this fix no doubt avoids the >> > problem, I'm worried there might be something deeper going on... >> >> Could it be that the logic to deduce the fetcher is somehow broken? >> >> Indeed, this was failing for me a couple of weeks ago with a similar SRC_URI. >> Back then i could not use file:///mydir;protocol=git (i.e. create a >> clone from a clone living on the local filesystem) so had to resort >> back to protocol=rsync. >> There was a socket lying around in that directory, a leftover from >> some testsuite run. > > FWIW, the url you want is git:///mydir;protocol=file which should work. Oh, that's rather surprising! The format suggested to me that, iff specified, with "protocol=" i can select the fetcher, regardless of the actual path-spec. Shouldn't protocol have precedence over any eventual path-implied fetcher? file:///mydir;protocol=rsync rsync -a /mydir S file:///mydir;protocol=git git clone -s file:///mydir S > As of the patches today, this also works as mirror syntax so you could > even add this in local.conf as a PREMIRROR and leave the recipe > pristine. thanks for the hint, yes, i know. Changing the recipe reminds me that i have local changes and i can paste it to khem easily. cheers,
On Thu, 2012-06-21 at 15:57 +0200, Bernhard Reutner-Fischer wrote: > > > > FWIW, the url you want is git:///mydir;protocol=file which should work. > > Oh, that's rather surprising! > The format suggested to me that, iff specified, with "protocol=" i can > select the fetcher, regardless of the actual path-spec. > Shouldn't protocol have precedence over any eventual path-implied fetcher? > file:///mydir;protocol=rsync > rsync -a /mydir S > file:///mydir;protocol=git > git clone -s file:///mydir S No, things work the opposite way around, they always have done and trying to change that now would be hard. It makes most sense when you think about it that anything starting with "git://" gets passed to the git fetcher code, anything "file://" to the local fetcher and so on. Cheers, Richard
computing the md5 of a socket does not work too well. Traceback (most recent call last): File "/scratch/src/oe/bitbake/build/lib/bb/siggen.py", line 172, in SignatureGeneratorOEBasic.get_taskhash(fn='/scratch/src/oe/openembedded-core/meta/recipes-core/uclibc/uclibc_git.bb', task='do_fetch', deps=[], dataCache=<bb.cache.CacheData object at 0xd95890>): if task in dataCache.file_checksums[fn]: > checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename) for (f,cs) in checksums: File "/scratch/src/oe/bitbake/build/lib/bb/fetch2/__init__.py", line 627, in get_file_checksums(filelist='/scratch/src/uClibc/ /scratch/src/oe/openembedded-core/meta/recipes-core/uclibc/uclibc-git/uClibc.machine /scratch/src/oe/openembedded-core/meta/recipes-core/uclibc/uclibc-git/uClibc.distro', pn='uclibc'): fullpth = os.path.join(root, name) > checksum = checksum_file(fullpth) if checksum: File "/scratch/src/oe/bitbake/build/lib/bb/fetch2/__init__.py", line 605, in checksum_file(f='/scratch/src/uClibc/test/inet/socktest'): try: > checksum = _checksum_cache.get_checksum(f) except OSError as e: File "/scratch/src/oe/bitbake/build/lib/bb/checksum.py", line 78, in FileChecksumCache.get_checksum(f='/scratch/src/uClibc/test/inet/socktest'): > hashval = bb.utils.md5_file(f) self.cachedata_extras[0][f] = (cmtime, hashval) File "/scratch/src/oe/bitbake/build/lib/bb/utils.py", line 362, in md5_file(filename='/scratch/src/uClibc/test/inet/socktest'): > for line in open(filename): m.update(line) IOError: [Errno 6] No such device or address: '/scratch/src/uClibc/test/inet/socktest' Signed-off-by: Bernhard Reutner-Fischer <rep.dot.nop@gmail.com> --- lib/bb/fetch2/__init__.py | 1 + 1 file changed, 1 insertion(+)