Message ID | 20220119180015.1931903-1-richard.purdie@linuxfoundation.org |
---|---|
State | Accepted, archived |
Commit | 1f06f326fa8b47e2a4dce756d57a9369a2225201 |
Headers | show |
Series | fetch2: Add crate fetcher | expand |
Hi Richard, Am 19.01.2022 um 19:00 schrieb Richard Purdie via lists.openembedded.org: > This imports the crate fetcher from OE-Core to resolve various module issues > and adds some very very basic tests of that new fetcher. > > Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org> > --- > lib/bb/fetch2/__init__.py | 2 + > lib/bb/fetch2/crate.py | 137 ++++++++++++++++++++++++++++++++++++++ > lib/bb/tests/fetch.py | 35 ++++++++++ > 3 files changed, 174 insertions(+) > create mode 100644 lib/bb/fetch2/crate.py > > diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py > index 0b39ea6aaa..d37174185a 100644 > --- a/lib/bb/fetch2/__init__.py > +++ b/lib/bb/fetch2/__init__.py > @@ -1942,6 +1942,7 @@ from . import clearcase > from . import npm > from . import npmsw > from . import az > +from . import crate > > methods.append(local.Local()) > methods.append(wget.Wget()) > @@ -1962,3 +1963,4 @@ methods.append(clearcase.ClearCase()) > methods.append(npm.Npm()) > methods.append(npmsw.NpmShrinkWrap()) > methods.append(az.Az()) > +methods.append(crate.Crate()) > diff --git a/lib/bb/fetch2/crate.py b/lib/bb/fetch2/crate.py > new file mode 100644 > index 0000000000..f7e2354afb > --- /dev/null > +++ b/lib/bb/fetch2/crate.py > @@ -0,0 +1,137 @@ > +# ex:ts=4:sw=4:sts=4:et > +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- > +""" > +BitBake 'Fetch' implementation for crates.io > +""" > + > +# Copyright (C) 2016 Doug Goldstein > +# > +# SPDX-License-Identifier: GPL-2.0-only > +# > +# Based on functions from the base bb module, Copyright 2003 Holger Schurig > + > +import hashlib > +import json > +import os > +import shutil > +import subprocess > +import bb > +from bb.fetch2 import logger, subprocess_setup, UnpackError > +from bb.fetch2.wget import Wget > + > + > +class Crate(Wget): > + > + """Class to fetch crates via wget""" > + > + def _cargo_bitbake_path(self, rootdir): > + return os.path.join(rootdir, "cargo_home", "bitbake") > + > + def supports(self, ud, d): > + """ > + Check to see if a given url is for this fetcher > + """ > + return ud.type in ['crate'] > + > + def recommends_checksum(self, urldata): > + return False Why is this false? How the fetcher check the integrity? > + > + def urldata_init(self, ud, d): > + """ > + Sets up to download the respective crate from crates.io > + """ > + > + if ud.type == 'crate': > + self._crate_urldata_init(ud, d) > + > + super(Crate, self).urldata_init(ud, d) Why is this needed? The ud.type should always be 'crate'. > + > + def _crate_urldata_init(self, ud, d): > + """ > + Sets up the download for a crate > + """ > + > + # URL syntax is: crate://NAME/VERSION > + # break the URL apart by / Why does this fetcher use a '/' as separation? The npm fetcher use package and version options: npm://some.registry.url;package=${BPN];version=${PV} > + parts = ud.url.split('/') > + if len(parts) < 5: > + raise bb.fetch2.ParameterError("Invalid URL: Must be crate://HOST/NAME/VERSION", ud.url) > + > + # last field is version > + version = parts[len(parts) - 1] > + # second to last field is name > + name = parts[len(parts) - 2] > + # host (this is to allow custom crate registries to be specified > + host = '/'.join(parts[2:len(parts) - 2]) This is error-prone because it is impossible to detect a missing name or version if the real host url is used. > + > + # if using upstream just fix it up nicely > + if host == 'crates.io': > + host = 'crates.io/api/v1/crates' > + > + ud.url = "https://%s/%s/%s/download" % (host, name, version) > + ud.parm['downloadfilename'] = "%s-%s.crate" % (name, version) > + ud.parm['name'] = name > + > + logger.debug(2, "Fetching %s to %s" % (ud.url, ud.parm['downloadfilename'])) > + > + def unpack(self, ud, rootdir, d): > + """ > + Uses the crate to build the necessary paths for cargo to utilize it > + """ > + if ud.type == 'crate': > + return self._crate_unpack(ud, rootdir, d) > + else: > + super(Crate, self).unpack(ud, rootdir, d) Why is this needed? The ud.type should always be 'crate'. > + > + def _crate_unpack(self, ud, rootdir, d): > + """ > + Unpacks a crate > + """ > + thefile = ud.localpath > + > + # possible metadata we need to write out > + metadata = {} > + > + # change to the rootdir to unpack but save the old working dir > + save_cwd = os.getcwd() > + os.chdir(rootdir) > + > + pn = d.getVar('BPN') > + if pn == ud.parm.get('name'): > + cmd = "tar -xz --no-same-owner -f %s" % thefile > + else: > + cargo_bitbake = self._cargo_bitbake_path(rootdir) > + > + cmd = "tar -xz --no-same-owner -f %s -C %s" % (thefile, cargo_bitbake) > + > + # ensure we've got these paths made > + bb.utils.mkdirhier(cargo_bitbake) > + Why don't you use the origin unpack and set the subdir to the correct folder. This allows the usage of the wget fetcher for crate archives outside of a repository. > + # generate metadata necessary > + with open(thefile, 'rb') as f: > + # get the SHA256 of the original tarball > + tarhash = hashlib.sha256(f.read()).hexdigest() Why don't you reuse the checksum from the recipe. > + > + metadata['files'] = {} > + metadata['package'] = tarhash > + > + path = d.getVar('PATH') > + if path: > + cmd = "PATH=\"%s\" %s" % (path, cmd) > + bb.note("Unpacking %s to %s/" % (thefile, os.getcwd())) > + > + ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True) > + > + os.chdir(save_cwd) > + > + if ret != 0: > + raise UnpackError("Unpack command %s failed with return value %s" % (cmd, ret), ud.url) > + > + # if we have metadata to write out.. > + if len(metadata) > 0: > + cratepath = os.path.splitext(os.path.basename(thefile))[0] > + bbpath = self._cargo_bitbake_path(rootdir) > + mdfile = '.cargo-checksum.json' > + mdpath = os.path.join(bbpath, cratepath, mdfile) > + with open(mdpath, "w") as f: > + json.dump(metadata, f) [snip]
Hello Stefan, this is already in master, so the concerns you have are probably better addressed with followup patches. Alex On Thu, 27 Jan 2022 at 11:39, Stefan Herbrechtsmeier < stefan.herbrechtsmeier-oss@weidmueller.com> wrote: > Hi Richard, > > Am 19.01.2022 um 19:00 schrieb Richard Purdie via lists.openembedded.org: > > This imports the crate fetcher from OE-Core to resolve various module > issues > > and adds some very very basic tests of that new fetcher. > > > > Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org> > > --- > > lib/bb/fetch2/__init__.py | 2 + > > lib/bb/fetch2/crate.py | 137 ++++++++++++++++++++++++++++++++++++++ > > lib/bb/tests/fetch.py | 35 ++++++++++ > > 3 files changed, 174 insertions(+) > > create mode 100644 lib/bb/fetch2/crate.py > > > > diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py > > index 0b39ea6aaa..d37174185a 100644 > > --- a/lib/bb/fetch2/__init__.py > > +++ b/lib/bb/fetch2/__init__.py > > @@ -1942,6 +1942,7 @@ from . import clearcase > > from . import npm > > from . import npmsw > > from . import az > > +from . import crate > > > > methods.append(local.Local()) > > methods.append(wget.Wget()) > > @@ -1962,3 +1963,4 @@ methods.append(clearcase.ClearCase()) > > methods.append(npm.Npm()) > > methods.append(npmsw.NpmShrinkWrap()) > > methods.append(az.Az()) > > +methods.append(crate.Crate()) > > diff --git a/lib/bb/fetch2/crate.py b/lib/bb/fetch2/crate.py > > new file mode 100644 > > index 0000000000..f7e2354afb > > --- /dev/null > > +++ b/lib/bb/fetch2/crate.py > > @@ -0,0 +1,137 @@ > > +# ex:ts=4:sw=4:sts=4:et > > +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- > > +""" > > +BitBake 'Fetch' implementation for crates.io > > +""" > > + > > +# Copyright (C) 2016 Doug Goldstein > > +# > > +# SPDX-License-Identifier: GPL-2.0-only > > +# > > +# Based on functions from the base bb module, Copyright 2003 Holger > Schurig > > + > > +import hashlib > > +import json > > +import os > > +import shutil > > +import subprocess > > +import bb > > +from bb.fetch2 import logger, subprocess_setup, UnpackError > > +from bb.fetch2.wget import Wget > > + > > + > > +class Crate(Wget): > > + > > + """Class to fetch crates via wget""" > > + > > + def _cargo_bitbake_path(self, rootdir): > > + return os.path.join(rootdir, "cargo_home", "bitbake") > > + > > + def supports(self, ud, d): > > + """ > > + Check to see if a given url is for this fetcher > > + """ > > + return ud.type in ['crate'] > > + > > + def recommends_checksum(self, urldata): > > + return False > > Why is this false? How the fetcher check the integrity? > > > + > > + def urldata_init(self, ud, d): > > + """ > > + Sets up to download the respective crate from crates.io > > + """ > > + > > + if ud.type == 'crate': > > + self._crate_urldata_init(ud, d) > > + > > + super(Crate, self).urldata_init(ud, d) > > Why is this needed? The ud.type should always be 'crate'. > > > + > > + def _crate_urldata_init(self, ud, d): > > + """ > > + Sets up the download for a crate > > + """ > > + > > + # URL syntax is: crate://NAME/VERSION > > + # break the URL apart by / > > Why does this fetcher use a '/' as separation? The npm fetcher use > package and version options: > > npm://some.registry.url;package=${BPN];version=${PV} > > > + parts = ud.url.split('/') > > + if len(parts) < 5: > > + raise bb.fetch2.ParameterError("Invalid URL: Must be > crate://HOST/NAME/VERSION", ud.url) > > + > > + # last field is version > > + version = parts[len(parts) - 1] > > + # second to last field is name > > + name = parts[len(parts) - 2] > > + # host (this is to allow custom crate registries to be specified > > + host = '/'.join(parts[2:len(parts) - 2]) > > This is error-prone because it is impossible to detect a missing name or > version if the real host url is used. > > > + > > + # if using upstream just fix it up nicely > > + if host == 'crates.io': > > + host = 'crates.io/api/v1/crates' > > + > > + ud.url = "https://%s/%s/%s/download" % (host, name, version) > > + ud.parm['downloadfilename'] = "%s-%s.crate" % (name, version) > > + ud.parm['name'] = name > > + > > + logger.debug(2, "Fetching %s to %s" % (ud.url, > ud.parm['downloadfilename'])) > > + > > + def unpack(self, ud, rootdir, d): > > + """ > > + Uses the crate to build the necessary paths for cargo to > utilize it > > + """ > > + if ud.type == 'crate': > > + return self._crate_unpack(ud, rootdir, d) > > + else: > > + super(Crate, self).unpack(ud, rootdir, d) > > Why is this needed? The ud.type should always be 'crate'. > > > + > > + def _crate_unpack(self, ud, rootdir, d): > > + """ > > + Unpacks a crate > > + """ > > + thefile = ud.localpath > > + > > + # possible metadata we need to write out > > + metadata = {} > > + > > + # change to the rootdir to unpack but save the old working dir > > + save_cwd = os.getcwd() > > + os.chdir(rootdir) > > + > > + pn = d.getVar('BPN') > > + if pn == ud.parm.get('name'): > > + cmd = "tar -xz --no-same-owner -f %s" % thefile > > + else: > > + cargo_bitbake = self._cargo_bitbake_path(rootdir) > > + > > + cmd = "tar -xz --no-same-owner -f %s -C %s" % (thefile, > cargo_bitbake) > > + > > + # ensure we've got these paths made > > + bb.utils.mkdirhier(cargo_bitbake) > > + > > Why don't you use the origin unpack and set the subdir to the correct > folder. This allows the usage of the wget fetcher for crate archives > outside of a repository. > > > + # generate metadata necessary > > + with open(thefile, 'rb') as f: > > + # get the SHA256 of the original tarball > > + tarhash = hashlib.sha256(f.read()).hexdigest() > > Why don't you reuse the checksum from the recipe. > > > + > > + metadata['files'] = {} > > + metadata['package'] = tarhash > > + > > + path = d.getVar('PATH') > > + if path: > > + cmd = "PATH=\"%s\" %s" % (path, cmd) > > + bb.note("Unpacking %s to %s/" % (thefile, os.getcwd())) > > + > > + ret = subprocess.call(cmd, preexec_fn=subprocess_setup, > shell=True) > > + > > + os.chdir(save_cwd) > > + > > + if ret != 0: > > + raise UnpackError("Unpack command %s failed with return > value %s" % (cmd, ret), ud.url) > > + > > + # if we have metadata to write out.. > > + if len(metadata) > 0: > > + cratepath = os.path.splitext(os.path.basename(thefile))[0] > > + bbpath = self._cargo_bitbake_path(rootdir) > > + mdfile = '.cargo-checksum.json' > > + mdpath = os.path.join(bbpath, cratepath, mdfile) > > + with open(mdpath, "w") as f: > > + json.dump(metadata, f) > > [snip] > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#13287): > https://lists.openembedded.org/g/bitbake-devel/message/13287 > Mute This Topic: https://lists.openembedded.org/mt/88540302/1686489 > Group Owner: bitbake-devel+owner@lists.openembedded.org > Unsubscribe: https://lists.openembedded.org/g/bitbake-devel/unsub [ > alex.kanavin@gmail.com] > -=-=-=-=-=-=-=-=-=-=-=- > >
Hi Alex, Am 27.01.2022 um 11:53 schrieb Alexander Kanavin: > this is already in master, so the concerns you have are probably better > addressed with followup patches. Does this means the SRC_URI syntax is fixed and we have total different syntax to specific a repository, package name and version for npm and crate?
On Thu, 27 Jan 2022 at 13:00, Stefan Herbrechtsmeier < stefan.herbrechtsmeier-oss@weidmueller.com> wrote: > Am 27.01.2022 um 11:53 schrieb Alexander Kanavin: > > this is already in master, so the concerns you have are probably better > > addressed with followup patches. > > Does this means the SRC_URI syntax is fixed and we have total different > syntax to specific a repository, package name and version for npm and > crate? > No it means that patches to address the concerns are welcome, and they will be reviewed on their own merit. Nothing is fixed, and everything can be changed. Alex
diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py index 0b39ea6aaa..d37174185a 100644 --- a/lib/bb/fetch2/__init__.py +++ b/lib/bb/fetch2/__init__.py @@ -1942,6 +1942,7 @@ from . import clearcase from . import npm from . import npmsw from . import az +from . import crate methods.append(local.Local()) methods.append(wget.Wget()) @@ -1962,3 +1963,4 @@ methods.append(clearcase.ClearCase()) methods.append(npm.Npm()) methods.append(npmsw.NpmShrinkWrap()) methods.append(az.Az()) +methods.append(crate.Crate()) diff --git a/lib/bb/fetch2/crate.py b/lib/bb/fetch2/crate.py new file mode 100644 index 0000000000..f7e2354afb --- /dev/null +++ b/lib/bb/fetch2/crate.py @@ -0,0 +1,137 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'Fetch' implementation for crates.io +""" + +# Copyright (C) 2016 Doug Goldstein +# +# SPDX-License-Identifier: GPL-2.0-only +# +# Based on functions from the base bb module, Copyright 2003 Holger Schurig + +import hashlib +import json +import os +import shutil +import subprocess +import bb +from bb.fetch2 import logger, subprocess_setup, UnpackError +from bb.fetch2.wget import Wget + + +class Crate(Wget): + + """Class to fetch crates via wget""" + + def _cargo_bitbake_path(self, rootdir): + return os.path.join(rootdir, "cargo_home", "bitbake") + + def supports(self, ud, d): + """ + Check to see if a given url is for this fetcher + """ + return ud.type in ['crate'] + + def recommends_checksum(self, urldata): + return False + + def urldata_init(self, ud, d): + """ + Sets up to download the respective crate from crates.io + """ + + if ud.type == 'crate': + self._crate_urldata_init(ud, d) + + super(Crate, self).urldata_init(ud, d) + + def _crate_urldata_init(self, ud, d): + """ + Sets up the download for a crate + """ + + # URL syntax is: crate://NAME/VERSION + # break the URL apart by / + parts = ud.url.split('/') + if len(parts) < 5: + raise bb.fetch2.ParameterError("Invalid URL: Must be crate://HOST/NAME/VERSION", ud.url) + + # last field is version + version = parts[len(parts) - 1] + # second to last field is name + name = parts[len(parts) - 2] + # host (this is to allow custom crate registries to be specified + host = '/'.join(parts[2:len(parts) - 2]) + + # if using upstream just fix it up nicely + if host == 'crates.io': + host = 'crates.io/api/v1/crates' + + ud.url = "https://%s/%s/%s/download" % (host, name, version) + ud.parm['downloadfilename'] = "%s-%s.crate" % (name, version) + ud.parm['name'] = name + + logger.debug(2, "Fetching %s to %s" % (ud.url, ud.parm['downloadfilename'])) + + def unpack(self, ud, rootdir, d): + """ + Uses the crate to build the necessary paths for cargo to utilize it + """ + if ud.type == 'crate': + return self._crate_unpack(ud, rootdir, d) + else: + super(Crate, self).unpack(ud, rootdir, d) + + def _crate_unpack(self, ud, rootdir, d): + """ + Unpacks a crate + """ + thefile = ud.localpath + + # possible metadata we need to write out + metadata = {} + + # change to the rootdir to unpack but save the old working dir + save_cwd = os.getcwd() + os.chdir(rootdir) + + pn = d.getVar('BPN') + if pn == ud.parm.get('name'): + cmd = "tar -xz --no-same-owner -f %s" % thefile + else: + cargo_bitbake = self._cargo_bitbake_path(rootdir) + + cmd = "tar -xz --no-same-owner -f %s -C %s" % (thefile, cargo_bitbake) + + # ensure we've got these paths made + bb.utils.mkdirhier(cargo_bitbake) + + # generate metadata necessary + with open(thefile, 'rb') as f: + # get the SHA256 of the original tarball + tarhash = hashlib.sha256(f.read()).hexdigest() + + metadata['files'] = {} + metadata['package'] = tarhash + + path = d.getVar('PATH') + if path: + cmd = "PATH=\"%s\" %s" % (path, cmd) + bb.note("Unpacking %s to %s/" % (thefile, os.getcwd())) + + ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True) + + os.chdir(save_cwd) + + if ret != 0: + raise UnpackError("Unpack command %s failed with return value %s" % (cmd, ret), ud.url) + + # if we have metadata to write out.. + if len(metadata) > 0: + cratepath = os.path.splitext(os.path.basename(thefile))[0] + bbpath = self._cargo_bitbake_path(rootdir) + mdfile = '.cargo-checksum.json' + mdpath = os.path.join(bbpath, cratepath, mdfile) + with open(mdpath, "w") as f: + json.dump(metadata, f) diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py index 2a046d0ded..ec7d83c959 100644 --- a/lib/bb/tests/fetch.py +++ b/lib/bb/tests/fetch.py @@ -2260,6 +2260,41 @@ class GitURLWithSpacesTest(FetcherTest): self.assertEqual(ud.clonedir, os.path.join(self.dldir, "git2", ref['gitsrcname'])) self.assertEqual(ud.fullmirror, os.path.join(self.dldir, "git2_" + ref['gitsrcname'] + '.tar.gz')) +class CrateTest(FetcherTest): + def test_crate_url(self): + + uri = "crate://crates.io/glob/0.2.11" + self.d.setVar('SRC_URI', uri) + + uris = self.d.getVar('SRC_URI').split() + d = self.d + + fetcher = bb.fetch2.Fetch(uris, self.d) + fetcher.download() + fetcher.unpack(self.tempdir) + self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked']) + self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done']) + self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json")) + self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs")) + + def test_crate_url_multi(self): + + uri = "crate://crates.io/glob/0.2.11 crate://crates.io/time/0.1.35" + self.d.setVar('SRC_URI', uri) + + uris = self.d.getVar('SRC_URI').split() + d = self.d + + fetcher = bb.fetch2.Fetch(uris, self.d) + fetcher.download() + fetcher.unpack(self.tempdir) + self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked']) + self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done', 'time-0.1.35.crate', 'time-0.1.35.crate.done']) + self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json")) + self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs")) + self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/time-0.1.35/.cargo-checksum.json")) + self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/time-0.1.35/src/lib.rs")) + class NPMTest(FetcherTest): def skipIfNoNpm(): import shutil
This imports the crate fetcher from OE-Core to resolve various module issues and adds some very very basic tests of that new fetcher. Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org> --- lib/bb/fetch2/__init__.py | 2 + lib/bb/fetch2/crate.py | 137 ++++++++++++++++++++++++++++++++++++++ lib/bb/tests/fetch.py | 35 ++++++++++ 3 files changed, 174 insertions(+) create mode 100644 lib/bb/fetch2/crate.py