| Message ID | 20220914142120.53537-1-paulo@eero.com |
|---|---|
| State | New |
| Headers | show |
| Series | [meta-oe,kirkstone,add-npm-commit-from-main-to-kirkstone] nodejs-oe-cache-native: initial checkin | expand |
I've already sent this backport request yesterday. On Wed, Sep 14, 2022 at 4:26 PM Paulo Vitor Becker Ayrosa Monteiro de Andrade <paulo@eero.com> wrote: > [Edited Message Follows] > > This is a commit from the master branch which is already merged. > In order to get yocto's build working on kirkstone branch, this commit is > necessary on meta-openembedded as well. > I ran into npm issues when doing some work here and stumbled upon Enrico's > solution to this, and it worked perfectly. If this was already merged to > kirkstone branch, I believe more people would benefit. > > From: Enrico Scholz <enrico.scholz@sigma-chemnitz.de> > > This implements an 'npm cache add' like functionality but allows to > specify the key of the data and sets metadata which are required to > find the data. > > It is used to cache information as done during 'npm install'. > > Keyformat and metadata are nodejs version specific. > > Signed-off-by: Enrico Scholz <enrico.scholz@sigma-chemnitz.de> > Signed-off-by: Khem Raj <raj.khem@gmail.com> > --- > .../nodejs/nodejs-oe-cache-16.14/oe-npm-cache | 77 +++++++++++++++++++ > .../nodejs/nodejs-oe-cache-native_16.14.bb | 21 +++++ > 2 files changed, 98 insertions(+) > create mode 100755 > meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-16.14/oe-npm-cache > create mode 100644 meta-oe/recipes-devtools/nodejs/ > nodejs-oe-cache-native_16.14.bb > > diff --git > a/meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-16.14/oe-npm-cache > b/meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-16.14/oe-npm-cache > new file mode 100755 > index 000000000..f59620764 > --- /dev/null > +++ b/meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-16.14/oe-npm-cache > @@ -0,0 +1,77 @@ > +#!/usr/bin/env node > + > +/// Usage: oe-npm-cache <cache-dir> <type> <key> <file-name> > +/// <type> ... meta - metainformation about package > +/// tgz - tarball > + > +const process = require("node:process"); > + > +module.paths.unshift("@@libdir@@/node_modules/npm/node_modules"); > + > +const cacache = require('cacache') > +const fs = require('fs') > + > +// argv[0] is 'node', argv[1] is this script > +const cache_dir = process.argv[2] > +const type = process.argv[3] > +const key = process.argv[4] > +const file = process.argv[5] > + > +const data = fs.readFileSync(file) > + > +// metadata content is highly nodejs dependent; when cache entries are not > +// found, place debug statements in > 'make-fetch-happen/lib/cache/policy.js' > +// (CachePolicy::satisfies()) > +const xlate = { > + 'meta': { > + 'key_prefix': 'make-fetch-happen:request-cache:', > + 'metadata': function() { > + return { > + time: Date.now(), > + url: key, > + reqHeaders: { > + 'accept': 'application/vnd.npm.install-v1+json; q=1.0, > application/json; q=0.8, */*', > + }, > + resHeaders: { > + "content-type": "application/json", > + "status": 200, > + }, > + options: { > + compress: true, > + } > + }; > + }, > + }, > + > + 'tgz': { > + 'key_prefix': 'make-fetch-happen:request-cache:', > + 'metadata': function() { > + return { > + time: Date.now(), > + url: key, > + reqHeaders: { > + 'accept': '*/*', > + }, > + resHeaders: { > + "content-type": "application/octet-stream", > + "status": 200, > + }, > + options: { > + compress: true, > + }, > + }; > + }, > + }, > +}; > + > +const info = xlate[type]; > +let opts = {} > + > +if (info.metadata) { > + opts['metadata'] = info.metadata(); > +} > + > +cacache.put(cache_dir, info.key_prefix + key, data, opts) > + .then(integrity => { > + console.log(`Saved content of ${key} (${file}).`); > +}) > diff --git a/meta-oe/recipes-devtools/nodejs/ > nodejs-oe-cache-native_16.14.bb b/meta-oe/recipes-devtools/nodejs/ > nodejs-oe-cache-native_16.14.bb > new file mode 100644 > index 000000000..a61dd5018 > --- /dev/null > +++ b/meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-native_16.14.bb > @@ -0,0 +1,21 @@ > +DESCRIPTION = "OE helper for manipulating npm cache" > +LICENSE = "Apache-2.0" > +LIC_FILES_CHKSUM = > "file://${COMMON_LICENSE_DIR}/Apache-2.0;md5=89aea4e17d99a7cacdbeed46a0096b10" > + > +SRC_URI = "\ > + file://oe-npm-cache \ > +" > + > +inherit native > + > +B = "${WORKDIR}/build" > + > +do_configure() { > + sed -e 's!@@libdir@@!${libdir}!g' < '${WORKDIR}/oe-npm-cache' > > '${B}/oe-npm-cache' > +} > + > +do_install() { > + install -D -p -m 0755 ${B}/oe-npm-cache ${D}${bindir}/oe-npm-cache > +} > + > +RDEPENDS:${PN} = "nodejs-native" > -- > 2.17.1 > > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#98804): > https://lists.openembedded.org/g/openembedded-devel/message/98804 > Mute This Topic: https://lists.openembedded.org/mt/93678585/3617156 > Group Owner: openembedded-devel+owner@lists.openembedded.org > Unsubscribe: https://lists.openembedded.org/g/openembedded-devel/unsub [ > Martin.Jansa@gmail.com] > -=-=-=-=-=-=-=-=-=-=-=- > >
Awesome! I tried sending it earlier, but had some problems. Thanks a bunch, then! On Wed, Sep 14, 2022 at 10:27 AM Martin Jansa <martin.jansa@gmail.com> wrote: > I've already sent this backport request yesterday. > > On Wed, Sep 14, 2022 at 4:26 PM Paulo Vitor Becker Ayrosa Monteiro de > Andrade <paulo@eero.com> wrote: > >> [Edited Message Follows] >> >> This is a commit from the master branch which is already merged. >> In order to get yocto's build working on kirkstone branch, this commit is >> necessary on meta-openembedded as well. >> I ran into npm issues when doing some work here and stumbled upon >> Enrico's solution to this, and it worked perfectly. If this was already >> merged to kirkstone branch, I believe more people would benefit. >> >> From: Enrico Scholz <enrico.scholz@sigma-chemnitz.de> >> >> This implements an 'npm cache add' like functionality but allows to >> specify the key of the data and sets metadata which are required to >> find the data. >> >> It is used to cache information as done during 'npm install'. >> >> Keyformat and metadata are nodejs version specific. >> >> Signed-off-by: Enrico Scholz <enrico.scholz@sigma-chemnitz.de> >> Signed-off-by: Khem Raj <raj.khem@gmail.com> >> --- >> .../nodejs/nodejs-oe-cache-16.14/oe-npm-cache | 77 +++++++++++++++++++ >> .../nodejs/nodejs-oe-cache-native_16.14.bb | 21 +++++ >> 2 files changed, 98 insertions(+) >> create mode 100755 >> meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-16.14/oe-npm-cache >> create mode 100644 meta-oe/recipes-devtools/nodejs/ >> nodejs-oe-cache-native_16.14.bb >> >> diff --git >> a/meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-16.14/oe-npm-cache >> b/meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-16.14/oe-npm-cache >> new file mode 100755 >> index 000000000..f59620764 >> --- /dev/null >> +++ b/meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-16.14/oe-npm-cache >> @@ -0,0 +1,77 @@ >> +#!/usr/bin/env node >> + >> +/// Usage: oe-npm-cache <cache-dir> <type> <key> <file-name> >> +/// <type> ... meta - metainformation about package >> +/// tgz - tarball >> + >> +const process = require("node:process"); >> + >> +module.paths.unshift("@@libdir@@/node_modules/npm/node_modules"); >> + >> +const cacache = require('cacache') >> +const fs = require('fs') >> + >> +// argv[0] is 'node', argv[1] is this script >> +const cache_dir = process.argv[2] >> +const type = process.argv[3] >> +const key = process.argv[4] >> +const file = process.argv[5] >> + >> +const data = fs.readFileSync(file) >> + >> +// metadata content is highly nodejs dependent; when cache entries are >> not >> +// found, place debug statements in >> 'make-fetch-happen/lib/cache/policy.js' >> +// (CachePolicy::satisfies()) >> +const xlate = { >> + 'meta': { >> + 'key_prefix': 'make-fetch-happen:request-cache:', >> + 'metadata': function() { >> + return { >> + time: Date.now(), >> + url: key, >> + reqHeaders: { >> + 'accept': 'application/vnd.npm.install-v1+json; >> q=1.0, application/json; q=0.8, */*', >> + }, >> + resHeaders: { >> + "content-type": "application/json", >> + "status": 200, >> + }, >> + options: { >> + compress: true, >> + } >> + }; >> + }, >> + }, >> + >> + 'tgz': { >> + 'key_prefix': 'make-fetch-happen:request-cache:', >> + 'metadata': function() { >> + return { >> + time: Date.now(), >> + url: key, >> + reqHeaders: { >> + 'accept': '*/*', >> + }, >> + resHeaders: { >> + "content-type": "application/octet-stream", >> + "status": 200, >> + }, >> + options: { >> + compress: true, >> + }, >> + }; >> + }, >> + }, >> +}; >> + >> +const info = xlate[type]; >> +let opts = {} >> + >> +if (info.metadata) { >> + opts['metadata'] = info.metadata(); >> +} >> + >> +cacache.put(cache_dir, info.key_prefix + key, data, opts) >> + .then(integrity => { >> + console.log(`Saved content of ${key} (${file}).`); >> +}) >> diff --git a/meta-oe/recipes-devtools/nodejs/ >> nodejs-oe-cache-native_16.14.bb b/meta-oe/recipes-devtools/nodejs/ >> nodejs-oe-cache-native_16.14.bb >> new file mode 100644 >> index 000000000..a61dd5018 >> --- /dev/null >> +++ b/meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-native_16.14.bb >> @@ -0,0 +1,21 @@ >> +DESCRIPTION = "OE helper for manipulating npm cache" >> +LICENSE = "Apache-2.0" >> +LIC_FILES_CHKSUM = >> "file://${COMMON_LICENSE_DIR}/Apache-2.0;md5=89aea4e17d99a7cacdbeed46a0096b10" >> + >> +SRC_URI = "\ >> + file://oe-npm-cache \ >> +" >> + >> +inherit native >> + >> +B = "${WORKDIR}/build" >> + >> +do_configure() { >> + sed -e 's!@@libdir@@!${libdir}!g' < '${WORKDIR}/oe-npm-cache' > >> '${B}/oe-npm-cache' >> +} >> + >> +do_install() { >> + install -D -p -m 0755 ${B}/oe-npm-cache ${D}${bindir}/oe-npm-cache >> +} >> + >> +RDEPENDS:${PN} = "nodejs-native" >> -- >> 2.17.1 >> >> >> -=-=-=-=-=-=-=-=-=-=-=- >> Links: You receive all messages sent to this group. >> View/Reply Online (#98804): >> https://lists.openembedded.org/g/openembedded-devel/message/98804 >> Mute This Topic: https://lists.openembedded.org/mt/93678585/3617156 >> Group Owner: openembedded-devel+owner@lists.openembedded.org >> Unsubscribe: https://lists.openembedded.org/g/openembedded-devel/unsub [ >> Martin.Jansa@gmail.com] >> -=-=-=-=-=-=-=-=-=-=-=- >> >>
diff --git a/meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-16.14/oe-npm-cache b/meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-16.14/oe-npm-cache new file mode 100755 index 000000000..f59620764 --- /dev/null +++ b/meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-16.14/oe-npm-cache @@ -0,0 +1,77 @@ +#!/usr/bin/env node + +/// Usage: oe-npm-cache <cache-dir> <type> <key> <file-name> +/// <type> ... meta - metainformation about package +/// tgz - tarball + +const process = require("node:process"); + +module.paths.unshift("@@libdir@@/node_modules/npm/node_modules"); + +const cacache = require('cacache') +const fs = require('fs') + +// argv[0] is 'node', argv[1] is this script +const cache_dir = process.argv[2] +const type = process.argv[3] +const key = process.argv[4] +const file = process.argv[5] + +const data = fs.readFileSync(file) + +// metadata content is highly nodejs dependent; when cache entries are not +// found, place debug statements in 'make-fetch-happen/lib/cache/policy.js' +// (CachePolicy::satisfies()) +const xlate = { + 'meta': { + 'key_prefix': 'make-fetch-happen:request-cache:', + 'metadata': function() { + return { + time: Date.now(), + url: key, + reqHeaders: { + 'accept': 'application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8, */*', + }, + resHeaders: { + "content-type": "application/json", + "status": 200, + }, + options: { + compress: true, + } + }; + }, + }, + + 'tgz': { + 'key_prefix': 'make-fetch-happen:request-cache:', + 'metadata': function() { + return { + time: Date.now(), + url: key, + reqHeaders: { + 'accept': '*/*', + }, + resHeaders: { + "content-type": "application/octet-stream", + "status": 200, + }, + options: { + compress: true, + }, + }; + }, + }, +}; + +const info = xlate[type]; +let opts = {} + +if (info.metadata) { + opts['metadata'] = info.metadata(); +} + +cacache.put(cache_dir, info.key_prefix + key, data, opts) + .then(integrity => { + console.log(`Saved content of ${key} (${file}).`); +}) diff --git a/meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-native_16.14.bb b/meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-native_16.14.bb new file mode 100644 index 000000000..a61dd5018 --- /dev/null +++ b/meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-native_16.14.bb @@ -0,0 +1,21 @@ +DESCRIPTION = "OE helper for manipulating npm cache" +LICENSE = "Apache-2.0" +LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/Apache-2.0;md5=89aea4e17d99a7cacdbeed46a0096b10" + +SRC_URI = "\ + file://oe-npm-cache \ +" + +inherit native + +B = "${WORKDIR}/build" + +do_configure() { + sed -e 's!@@libdir@@!${libdir}!g' < '${WORKDIR}/oe-npm-cache' > '${B}/oe-npm-cache' +} + +do_install() { + install -D -p -m 0755 ${B}/oe-npm-cache ${D}${bindir}/oe-npm-cache +} + +RDEPENDS:${PN} = "nodejs-native"