From patchwork Thu May 19 10:05:51 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Enrico Scholz X-Patchwork-Id: 8263 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 405DEC433FE for ; Thu, 19 May 2022 10:06:28 +0000 (UTC) Received: from smtpout.cvg.de (smtpout.cvg.de [87.128.211.67]) by mx.groups.io with SMTP id smtpd.web08.5768.1652954775612988058 for ; Thu, 19 May 2022 03:06:19 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@sigma-chemnitz.de header.s=v2022040800 header.b=EIjBK+uS; spf=pass (domain: sigma-chemnitz.de, ip: 87.128.211.67, mailfrom: enrico.scholz@sigma-chemnitz.de) Received: from mail-mta-3.intern.sigma-chemnitz.de (mail-mta-3.intern.sigma-chemnitz.de [192.168.12.71]) by mail-out-3.intern.sigma-chemnitz.de (8.16.1/8.16.1) with ESMTPS id 24JA6CI8884853 (version=TLSv1.3 cipher=TLS_AES_256_GCM_SHA384 bits=256 verify=OK) for ; Thu, 19 May 2022 12:06:12 +0200 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=sigma-chemnitz.de; s=v2022040800; t=1652954772; bh=eDUd+ezmGlAwC941iMqD3OkuhR09WitcQwfxEAr7YeI=; l=2419; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=EIjBK+uSJPOkOsOJyWVnrAeVvJtGdDyI8kuFsZ/JT0MJ3WOHJUV3Xye2KzXmiUa0+ iJdoMFSYZBYae/EMDlmBov5IghGm+ccE7U1JrKtHjuO4eoiJUsnN/jS+WWmT0KYecN M9mwAEbKrsJiMKxOdCAynNpIXFbqJQUwboqOcnSWp4pzPPbV2ja3cC0H57o5TIQKbD KSFYzi7LNDwlle0p8MymwTM42drQcGzdt9OgtGuyfNfm1O/KSpcgl1ay/v7k1Tc6Gt PnWQEeyE51yqtzoR5i2YnWjrHxocWVTVsZFWFTfbKutQZIKhbkv8wc6iyQxAip7rHd 3AxHhNXWCfmVQ== Received: from reddoxx.intern.sigma-chemnitz.de (reddoxx.sigma.local [192.168.16.32]) by mail-mta-3.intern.sigma-chemnitz.de (8.16.1/8.16.1) with ESMTP id 24JA68j12313867 for from enrico.scholz@sigma-chemnitz.de; Thu, 19 May 2022 12:06:10 +0200 Received: from mail-msa-3.intern.sigma-chemnitz.de ( [192.168.12.73]) by reddoxx.intern.sigma-chemnitz.de (Reddoxx engine) with SMTP id 4671F054540; Thu, 19 May 2022 12:06:07 +0200 Received: from ensc-pc.intern.sigma-chemnitz.de (ensc-pc.intern.sigma-chemnitz.de [192.168.3.24]) by mail-msa-3.intern.sigma-chemnitz.de (8.15.2/8.15.2) with ESMTPS id 24JA66te856493 (version=TLSv1.3 cipher=TLS_AES_256_GCM_SHA384 bits=256 verify=NO); Thu, 19 May 2022 12:06:06 +0200 Received: from ensc by ensc-pc.intern.sigma-chemnitz.de with local (Exim 4.95) (envelope-from ) id 1nrd2k-004ImP-Hn; Thu, 19 May 2022 12:06:06 +0200 From: Enrico Scholz To: openembedded-core@lists.openembedded.org Cc: Enrico Scholz Subject: [PATCH 1/6] npm: replace 'npm pack' call by 'tar czf' Date: Thu, 19 May 2022 12:05:51 +0200 Message-Id: <42a8c94825cdcee944a274210a78e9e9940e6b63.1652954597.git.enrico.scholz@sigma-chemnitz.de> X-Mailer: git-send-email 2.36.1 In-Reply-To: References: MIME-Version: 1.0 Sender: Enrico Scholz List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Thu, 19 May 2022 10:06:28 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/165877 'npm pack' is a maintainer tool which tries to execute 'prepare' and similar scripts. This fails usually in OE because it requires completely installed 'node_modules'. Earlier nodejs versions supported an undocumented 'ignore-scripts' option. This has been removed in nodejs 16. We could patch 'package.json' and remove the unwanted scripts. But this might complicate local workflows (applying patches) and installed packages will contain the modified 'package.json'. Instead of, package it manually by 'tar czf'. As a sideeffect, 'do_configure' is running much faster now. Signed-off-by: Enrico Scholz --- meta/classes/npm.bbclass | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/meta/classes/npm.bbclass b/meta/classes/npm.bbclass index ba50fcac20..91f8f36b0d 100644 --- a/meta/classes/npm.bbclass +++ b/meta/classes/npm.bbclass @@ -57,13 +57,36 @@ def npm_global_configs(d): configs.append(("cache", d.getVar("NPM_CACHE"))) return configs +## 'npm pack' runs 'prepare' and 'prepack' scripts. Support for +## 'ignore-scripts' which prevents this behavior has been removed +## from nodejs 16. Use simple 'tar' instead of. def npm_pack(env, srcdir, workdir): - """Run 'npm pack' on a specified directory""" - import shlex - cmd = "npm pack %s" % shlex.quote(srcdir) - args = [("ignore-scripts", "true")] - tarball = env.run(cmd, args=args, workdir=workdir).strip("\n") - return os.path.join(workdir, tarball) + """Emulate 'npm pack' on a specified directory""" + import subprocess + import os + import json + + src = os.path.join(srcdir, 'package.json') + with open(src) as f: + j = json.load(f) + + # base does not really matter and is for documentation purposes + # only. But the 'version' part must exist because other parts of + # the bbclass rely on it. + base = j['name'].split('/')[-1] + tarball = os.path.join(workdir, "%s-%s.tgz" % (base, j['version'])); + + # TODO: real 'npm pack' does not include directories while 'tar' + # does. But this does not seem to matter... + subprocess.run(['tar', 'czf', tarball, + '--exclude', './node-modules', + '--exclude-vcs', + '--transform', 's,^\./,package/,', + '--mtime', '1985-10-26T08:15:00.000Z', + '.'], + check = True, cwd = srcdir) + + return tarball python npm_do_configure() { """ From patchwork Thu May 19 10:05:52 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Enrico Scholz X-Patchwork-Id: 8264 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 5E364C4321E for ; Thu, 19 May 2022 10:06:28 +0000 (UTC) Received: from smtpout.cvg.de (smtpout.cvg.de [87.128.211.67]) by mx.groups.io with SMTP id smtpd.web12.5794.1652954775950427984 for ; Thu, 19 May 2022 03:06:18 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@sigma-chemnitz.de header.s=v2022040800 header.b=R+00HQn9; spf=pass (domain: sigma-chemnitz.de, ip: 87.128.211.67, mailfrom: enrico.scholz@sigma-chemnitz.de) Received: from mail-mta-3.intern.sigma-chemnitz.de (mail-mta-3.intern.sigma-chemnitz.de [192.168.12.71]) by mail-out-2.intern.sigma-chemnitz.de (8.16.1/8.16.1) with ESMTPS id 24JA6Atw621378 (version=TLSv1.3 cipher=TLS_AES_256_GCM_SHA384 bits=256 verify=OK) for ; Thu, 19 May 2022 12:06:11 +0200 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=sigma-chemnitz.de; s=v2022040800; t=1652954771; bh=JGPM6hoyOtYMAuhEg+rktOd6b6XhEKy8/h3GZgAM4Gk=; l=1768; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=R+00HQn9Tf16Az5nvyu7Z724Gpi4iMChqfHUwIzohVBRWk457WEdSKsSWHoI58e9z Okoq+fJ3XX7ILHs3KlEAuzD3mneLigXkvUZqofG/kG81eVoOuReq55GpWlZh/2KaCP tiDjCArzSIe+YzkD4gCSJ6YikYWx940K6yCseNGL2Ku+XBB3W5kJZXtoWO885Tai6e QpV3wrhDfthOs+Ge3GC4y8pT9RmrtwaaXUHKDWWq8dU+VOy44zrEqn6mExAvuI/kVa CPeEkJMOCJjmZhZ5xlkHDXvBJiZg1g/fn2LhWDWUYFeYaWmokblOrN3PTUU1nhfOAP 182jX51I5vnEQ== Received: from reddoxx.intern.sigma-chemnitz.de (reddoxx.sigma.local [192.168.16.32]) by mail-mta-3.intern.sigma-chemnitz.de (8.16.1/8.16.1) with ESMTP id 24JA687C2313863 for from enrico.scholz@sigma-chemnitz.de; Thu, 19 May 2022 12:06:08 +0200 Received: from mail-msa-2.intern.sigma-chemnitz.de ( [192.168.12.72]) by reddoxx.intern.sigma-chemnitz.de (Reddoxx engine) with SMTP id 4E59FBF212B; Thu, 19 May 2022 12:06:06 +0200 Received: from ensc-pc.intern.sigma-chemnitz.de (ensc-pc.intern.sigma-chemnitz.de [192.168.3.24]) by mail-msa-2.intern.sigma-chemnitz.de (8.16.1/8.16.1) with ESMTPS id 24JA66OX596899 (version=TLSv1.3 cipher=TLS_AES_256_GCM_SHA384 bits=256 verify=NO); Thu, 19 May 2022 12:06:06 +0200 Received: from ensc by ensc-pc.intern.sigma-chemnitz.de with local (Exim 4.95) (envelope-from ) id 1nrd2k-004ImS-Is; Thu, 19 May 2022 12:06:06 +0200 From: Enrico Scholz To: openembedded-core@lists.openembedded.org Cc: Enrico Scholz Subject: [PATCH 2/6] npm: return content of 'package.json' in 'npm_pack' Date: Thu, 19 May 2022 12:05:52 +0200 Message-Id: X-Mailer: git-send-email 2.36.1 In-Reply-To: References: MIME-Version: 1.0 Sender: Enrico Scholz List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Thu, 19 May 2022 10:06:28 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/165873 We have to read 'package.json' to calculate the name of the tarball. This content is interesting for later patches. Signed-off-by: Enrico Scholz --- meta/classes/npm.bbclass | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/meta/classes/npm.bbclass b/meta/classes/npm.bbclass index 91f8f36b0d..014f793450 100644 --- a/meta/classes/npm.bbclass +++ b/meta/classes/npm.bbclass @@ -86,7 +86,7 @@ def npm_pack(env, srcdir, workdir): '.'], check = True, cwd = srcdir) - return tarball + return (tarball, j) python npm_do_configure() { """ @@ -186,7 +186,7 @@ python npm_do_configure() { with tempfile.TemporaryDirectory() as tmpdir: # Add the dependency to the npm cache destdir = os.path.join(d.getVar("S"), destsuffix) - tarball = npm_pack(env, destdir, tmpdir) + (tarball, pkg) = npm_pack(env, destdir, tmpdir) _npm_cache_add(tarball) # Add its signature to the cached shrinkwrap dep = _npmsw_dependency_dict(cached_shrinkwrap, deptree) @@ -207,7 +207,7 @@ python npm_do_configure() { # Configure the main package with tempfile.TemporaryDirectory() as tmpdir: - tarball = npm_pack(env, d.getVar("S"), tmpdir) + (tarball, _) = npm_pack(env, d.getVar("S"), tmpdir) npm_unpack(tarball, d.getVar("NPM_PACKAGE"), d) # Configure the cached manifest file and cached shrinkwrap file @@ -280,7 +280,7 @@ python npm_do_compile() { args.append(("build-from-source", "true")) # Pack and install the main package - tarball = npm_pack(env, d.getVar("NPM_PACKAGE"), tmpdir) + (tarball, _) = npm_pack(env, d.getVar("NPM_PACKAGE"), tmpdir) cmd = "npm install %s %s" % (shlex.quote(tarball), d.getVar("EXTRA_OENPM")) env.run(cmd, args=args) } From patchwork Thu May 19 10:05:53 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Enrico Scholz X-Patchwork-Id: 8261 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 4F664C43219 for ; Thu, 19 May 2022 10:06:28 +0000 (UTC) Received: from smtpout.cvg.de (smtpout.cvg.de [87.128.211.67]) by mx.groups.io with SMTP id smtpd.web10.5809.1652954777537820610 for ; Thu, 19 May 2022 03:06:18 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@sigma-chemnitz.de header.s=v2022040800 header.b=s6D2h4JD; spf=pass (domain: sigma-chemnitz.de, ip: 87.128.211.67, mailfrom: enrico.scholz@sigma-chemnitz.de) Received: from mail-mta-2.intern.sigma-chemnitz.de (mail-mta-2.intern.sigma-chemnitz.de [192.168.12.70]) by mail-out-3.intern.sigma-chemnitz.de (8.16.1/8.16.1) with ESMTPS id 24JA6EJ3884858 (version=TLSv1.3 cipher=TLS_AES_256_GCM_SHA384 bits=256 verify=OK) for ; Thu, 19 May 2022 12:06:14 +0200 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=sigma-chemnitz.de; s=v2022040800; t=1652954774; bh=9fL99t6vbwBkirIUh39Mw6WJVQTX21pT/cCk6141nvQ=; l=1289; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=s6D2h4JDmNdzbTcSTSEdUr3eWU508RihBN6ajpghXUncfow0bJ6y8hk7XZcaOtrrh B+JvI7wpgcOtKPrwAc6WIn6dPharXfHorWnvp2bIMWmB66UZV+K3UOGkqA+1qqMD6I TsIVXuJV6PHx+6kFrGgTZbPyOzbiqbi48uQ/hlKoTUPB6k4sUvqnvRwTGPVaETtnxi ilyb2Z0HY4rUmMe2U23LtkFDD7y99gBqIOWzCQzX/wd7xwwEZ1aAVK+4bGBwi9ZD1d DQw0vmDsDvMRr2M1wl6HrtVFRuKmJ4eNlrmmRam396d9+tZJjk7wkjCG6cjYcwBgh9 /79zOpDDfZZ8Q== Received: from reddoxx.intern.sigma-chemnitz.de (reddoxx.sigma.local [192.168.16.32]) by mail-mta-2.intern.sigma-chemnitz.de (8.16.1/8.16.1) with ESMTP id 24JA68wT791549 for from enrico.scholz@sigma-chemnitz.de; Thu, 19 May 2022 12:06:10 +0200 Received: from mail-msa-3.intern.sigma-chemnitz.de ( [192.168.12.73]) by reddoxx.intern.sigma-chemnitz.de (Reddoxx engine) with SMTP id 10A11FA07C7; Thu, 19 May 2022 12:06:07 +0200 Received: from ensc-pc.intern.sigma-chemnitz.de (ensc-pc.intern.sigma-chemnitz.de [192.168.3.24]) by mail-msa-3.intern.sigma-chemnitz.de (8.15.2/8.15.2) with ESMTPS id 24JA66XC856494 (version=TLSv1.3 cipher=TLS_AES_256_GCM_SHA384 bits=256 verify=NO); Thu, 19 May 2022 12:06:06 +0200 Received: from ensc by ensc-pc.intern.sigma-chemnitz.de with local (Exim 4.95) (envelope-from ) id 1nrd2k-004ImV-Jp; Thu, 19 May 2022 12:06:06 +0200 From: Enrico Scholz To: openembedded-core@lists.openembedded.org Cc: Enrico Scholz Subject: [PATCH 3/6] npm: take 'version' directly from 'package.json' Date: Thu, 19 May 2022 12:05:53 +0200 Message-Id: <37a1970bc8cacda99efefad7c6480cb694d9aec7.1652954597.git.enrico.scholz@sigma-chemnitz.de> X-Mailer: git-send-email 2.36.1 In-Reply-To: References: MIME-Version: 1.0 Sender: Enrico Scholz List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Thu, 19 May 2022 10:06:28 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/165875 We know the content of 'package.json' from earlier patches; there is no need to parse the tarball name to extract the version. Signed-off-by: Enrico Scholz --- meta/classes/npm.bbclass | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/meta/classes/npm.bbclass b/meta/classes/npm.bbclass index 014f793450..11c80a738e 100644 --- a/meta/classes/npm.bbclass +++ b/meta/classes/npm.bbclass @@ -125,11 +125,6 @@ python npm_do_configure() { sha512 = bb.utils.sha512_file(tarball) return "sha512-" + base64.b64encode(bytes.fromhex(sha512)).decode() - def _npm_version(tarball): - """Return the version of a specified tarball""" - regex = r"-(\d+\.\d+\.\d+(-.*)?(\+.*)?)\.tgz" - return re.search(regex, tarball).group(1) - def _npmsw_dependency_dict(orig, deptree): """ Return the sub dictionary in the 'orig' dictionary corresponding to the @@ -190,7 +185,7 @@ python npm_do_configure() { _npm_cache_add(tarball) # Add its signature to the cached shrinkwrap dep = _npmsw_dependency_dict(cached_shrinkwrap, deptree) - dep["version"] = _npm_version(tarball) + dep["version"] = pkg['version'] dep["integrity"] = _npm_integrity(tarball) if params.get("dev", False): dep["dev"] = True From patchwork Thu May 19 10:05:54 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Enrico Scholz X-Patchwork-Id: 8260 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 3F04FC433EF for ; Thu, 19 May 2022 10:06:28 +0000 (UTC) Received: from smtpout.cvg.de (smtpout.cvg.de [87.128.211.67]) by mx.groups.io with SMTP id smtpd.web12.5795.1652954775972947727 for ; Thu, 19 May 2022 03:06:18 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@sigma-chemnitz.de header.s=v2022040800 header.b=XucAH0D3; spf=pass (domain: sigma-chemnitz.de, ip: 87.128.211.67, mailfrom: enrico.scholz@sigma-chemnitz.de) Received: from mail-mta-2.intern.sigma-chemnitz.de (mail-mta-2.intern.sigma-chemnitz.de [192.168.12.70]) by mail-out-3.intern.sigma-chemnitz.de (8.16.1/8.16.1) with ESMTPS id 24JA6Ct3884852 (version=TLSv1.3 cipher=TLS_AES_256_GCM_SHA384 bits=256 verify=OK) for ; Thu, 19 May 2022 12:06:12 +0200 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=sigma-chemnitz.de; s=v2022040800; t=1652954772; bh=bEMiKxshjTrJvAX3jBlLulf4+9P2xSWMVLb/xoJy9WQ=; l=725; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=XucAH0D3bTwH/U4QLBoBJIQbD7LYTKx4FGmVinGi7oMRbrZoucfm96sG/GgC7xxb2 rjWD0QqQr44TGqteaMEpRUZBD+rjY9c9bR1eUp7wNWHibF8IyEA/ERMTuvin+iRz7P N5bTiJkkMTDQh/ACu/L4/RiaGif3MCJwM1rzZ+axwrWpL+TbOqZN0YFnGfArFRvULz yhUytXniG5pQBhvujrEiSpbmAq7v/TUAunlf8jcvXSIDWC/1cOOOswhaxTcuVT5rim hnjJqTHvhEJ/v2k286Jr59x4/6OxX9DLg+hhkJsMUYyyJyn+vgIsNSQJj/1qq4Q6IF BceuvqUbYOH7w== Received: from reddoxx.intern.sigma-chemnitz.de (reddoxx.sigma.local [192.168.16.32]) by mail-mta-2.intern.sigma-chemnitz.de (8.16.1/8.16.1) with ESMTP id 24JA68XR791548 for from enrico.scholz@sigma-chemnitz.de; Thu, 19 May 2022 12:06:10 +0200 Received: from mail-msa-3.intern.sigma-chemnitz.de ( [192.168.12.73]) by reddoxx.intern.sigma-chemnitz.de (Reddoxx engine) with SMTP id 14DF203DB70; Thu, 19 May 2022 12:06:07 +0200 Received: from ensc-pc.intern.sigma-chemnitz.de (ensc-pc.intern.sigma-chemnitz.de [192.168.3.24]) by mail-msa-3.intern.sigma-chemnitz.de (8.15.2/8.15.2) with ESMTPS id 24JA66MV856495 (version=TLSv1.3 cipher=TLS_AES_256_GCM_SHA384 bits=256 verify=NO); Thu, 19 May 2022 12:06:06 +0200 Received: from ensc by ensc-pc.intern.sigma-chemnitz.de with local (Exim 4.95) (envelope-from ) id 1nrd2k-004ImY-Kj; Thu, 19 May 2022 12:06:06 +0200 From: Enrico Scholz To: openembedded-core@lists.openembedded.org Cc: Enrico Scholz Subject: [PATCH 4/6] npm: disable 'audit' + 'fund' Date: Thu, 19 May 2022 12:05:54 +0200 Message-Id: <9dd04b7493db380bc08e0346df8083c716e505c3.1652954597.git.enrico.scholz@sigma-chemnitz.de> X-Mailer: git-send-email 2.36.1 In-Reply-To: References: MIME-Version: 1.0 Sender: Enrico Scholz List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Thu, 19 May 2022 10:06:28 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/165872 'audit' can cause extra network traffic; 'fund' is not needed. Signed-off-by: Enrico Scholz --- meta/classes/npm.bbclass | 2 ++ 1 file changed, 2 insertions(+) diff --git a/meta/classes/npm.bbclass b/meta/classes/npm.bbclass index 11c80a738e..1e41072116 100644 --- a/meta/classes/npm.bbclass +++ b/meta/classes/npm.bbclass @@ -53,6 +53,8 @@ def npm_global_configs(d): # Ensure no network access is done configs.append(("offline", "true")) configs.append(("proxy", "http://invalid")) + configs.append(("funds", False)) + configs.append(("audit", False)) # Configure the cache directory configs.append(("cache", d.getVar("NPM_CACHE"))) return configs From patchwork Thu May 19 10:05:55 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Enrico Scholz X-Patchwork-Id: 8265 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 50643C43217 for ; Thu, 19 May 2022 10:06:28 +0000 (UTC) Received: from smtpout.cvg.de (smtpout.cvg.de [87.128.211.67]) by mx.groups.io with SMTP id smtpd.web10.5808.1652954775985982885 for ; Thu, 19 May 2022 03:06:18 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@sigma-chemnitz.de header.s=v2022040800 header.b=aPiTJB3U; spf=pass (domain: sigma-chemnitz.de, ip: 87.128.211.67, mailfrom: enrico.scholz@sigma-chemnitz.de) Received: from mail-mta-2.intern.sigma-chemnitz.de (mail-mta-2.intern.sigma-chemnitz.de [192.168.12.70]) by mail-out-3.intern.sigma-chemnitz.de (8.16.1/8.16.1) with ESMTPS id 24JA6BdK884849 (version=TLSv1.3 cipher=TLS_AES_256_GCM_SHA384 bits=256 verify=OK) for ; Thu, 19 May 2022 12:06:11 +0200 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=sigma-chemnitz.de; s=v2022040800; t=1652954771; bh=LAbtJG27b/FuMnf9Up6jEiGHILcTrMVuC2S1mCceVL8=; l=5185; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=aPiTJB3UuqZk0y8Qbg3Ol85OM2NpLPP312fN3ugLxHW3/sQxOXGfyJ+kZTxYU6WbV dse+IDKecxM5L7gJg82DbJ29AYuywCDrEe0FPWVMNNhOArVY8unyXpzJ0bbTrdeq6s KVPX/mq4n6Z56fZUqs01zK4joih4tfUDRj74o2HtTbJTPqxuEJ7YdccvrouDjxEgV9 GiNT+KrGuU7i/S+sDK7mJY2utptb7CDgbPQ3Prpvq7573JxzQjA6CQGqVP3Bqtho+y PpyAC+E5ul9Tb1ySj2Z71r6/7La1JtiXzmCXgqeVeZy5RZyTwnvKzTcxAjZ6CuXGw8 gHvKQEVv/bT+g== Received: from reddoxx.intern.sigma-chemnitz.de (reddoxx.sigma.local [192.168.16.32]) by mail-mta-2.intern.sigma-chemnitz.de (8.16.1/8.16.1) with ESMTP id 24JA68eT791537 for from enrico.scholz@sigma-chemnitz.de; Thu, 19 May 2022 12:06:08 +0200 Received: from mail-msa-2.intern.sigma-chemnitz.de ( [192.168.12.72]) by reddoxx.intern.sigma-chemnitz.de (Reddoxx engine) with SMTP id 186941B4F97; Thu, 19 May 2022 12:06:06 +0200 Received: from ensc-pc.intern.sigma-chemnitz.de (ensc-pc.intern.sigma-chemnitz.de [192.168.3.24]) by mail-msa-2.intern.sigma-chemnitz.de (8.16.1/8.16.1) with ESMTPS id 24JA66h1596900 (version=TLSv1.3 cipher=TLS_AES_256_GCM_SHA384 bits=256 verify=NO); Thu, 19 May 2022 12:06:06 +0200 Received: from ensc by ensc-pc.intern.sigma-chemnitz.de with local (Exim 4.95) (envelope-from ) id 1nrd2k-004Imb-Lj; Thu, 19 May 2022 12:06:06 +0200 From: Enrico Scholz To: openembedded-core@lists.openembedded.org Cc: Enrico Scholz Subject: [PATCH 5/6] lib:npm_registry: initial checkin Date: Thu, 19 May 2022 12:05:55 +0200 Message-Id: X-Mailer: git-send-email 2.36.1 In-Reply-To: References: MIME-Version: 1.0 Sender: Enrico Scholz List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Thu, 19 May 2022 10:06:28 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/165874 Helper module to: - generate meta information from package.json content. This data has a format as provided by https://registry.npmjs.org - put this meta information and the corresponding tarball in the nodejs cache. This uses an external, nodejs version specific helper script (oe-npm-cache) shipped in oe-meta To avoid further nodejs version dependencies, future versions of this module might omit the caching completely and serve meta information and tarball by an http server. Signed-off-by: Enrico Scholz --- meta/lib/oe/npm_registry.py | 169 ++++++++++++++++++++++++++++++++++++ 1 file changed, 169 insertions(+) create mode 100644 meta/lib/oe/npm_registry.py diff --git a/meta/lib/oe/npm_registry.py b/meta/lib/oe/npm_registry.py new file mode 100644 index 0000000000..96c0affb45 --- /dev/null +++ b/meta/lib/oe/npm_registry.py @@ -0,0 +1,169 @@ +import bb +import json +import subprocess + +_ALWAYS_SAFE = frozenset('ABCDEFGHIJKLMNOPQRSTUVWXYZ' + 'abcdefghijklmnopqrstuvwxyz' + '0123456789' + '_.-~') + +MISSING_OK = object() + +REGISTRY = "https://registry.npmjs.org" + +# we can not use urllib.parse here because npm expects lowercase +# hex-chars but urllib generates uppercase ones +def uri_quote(s, safe = '/'): + res = "" + safe_set = set(safe) + for c in s: + if c in _ALWAYS_SAFE or c in safe_set: + res += c + else: + res += '%%%02x' % ord(c) + return res + +class PackageJson: + def __init__(self, spec): + self.__spec = spec + + @property + def name(self): + return self.__spec['name'] + + @property + def version(self): + return self.__spec['version'] + + @property + def empty_manifest(self): + return { + 'name': self.name, + 'description': self.__spec.get('description', ''), + 'versions': {}, + } + + def base_filename(self): + return uri_quote(self.name, safe = '@') + + def as_manifest_entry(self, tarball_uri): + res = {} + + ## NOTE: 'npm install' requires more than basic meta information; + ## e.g. it takes 'bin' from this manifest entry but not the actual + ## 'package.json' + for (idx,dflt) in [('name', None), + ('description', ""), + ('version', None), + ('bin', MISSING_OK), + ('man', MISSING_OK), + ('scripts', MISSING_OK), + ('directories', MISSING_OK), + ('dependencies', MISSING_OK), + ('devDependencies', MISSING_OK), + ('optionalDependencies', MISSING_OK), + ('license', "unknown")]: + if idx in self.__spec: + res[idx] = self.__spec[idx] + elif dflt == MISSING_OK: + pass + elif dflt != None: + res[idx] = dflt + else: + raise Exception("%s-%s: missing key %s" % (self.name, + self.version, + idx)) + + res['dist'] = { + 'tarball': tarball_uri, + } + + return res + +class ManifestImpl: + def __init__(self, base_fname, spec): + self.__base = base_fname + self.__spec = spec + + def load(self): + try: + with open(self.filename, "r") as f: + res = json.load(f) + except IOError: + res = self.__spec.empty_manifest + + return res + + def save(self, meta): + with open(self.filename, "w") as f: + json.dump(meta, f, indent = 2) + + @property + def filename(self): + return self.__base + ".meta" + +class Manifest: + def __init__(self, base_fname, spec): + self.__base = base_fname + self.__spec = spec + self.__lockf = None + self.__impl = None + + def __enter__(self): + self.__lockf = bb.utils.lockfile(self.__base + ".lock") + self.__impl = ManifestImpl(self.__base, self.__spec) + return self.__impl + + def __exit__(self, exc_type, exc_val, exc_tb): + bb.utils.unlockfile(self.__lockf) + +class NpmCache: + def __init__(self, cache): + self.__cache = cache + + @property + def path(self): + return self.__cache + + def run(self, type, key, fname): + subprocess.run(['oe-npm-cache', self.__cache, type, key, fname], + check = True) + +class NpmRegistry: + def __init__(self, path, cache): + self.__path = path + self.__cache = NpmCache(cache + '/_cacache') + bb.utils.mkdirhier(self.__path) + bb.utils.mkdirhier(self.__cache.path) + + @staticmethod + ## This function is critical and must match nodejs expectations + def _meta_uri(spec): + return REGISTRY + '/' + uri_quote(spec.name, safe = '@') + + @staticmethod + ## Exact return value does not matter; just make it look like a + ## usual registry url + def _tarball_uri(spec): + return '%s/%s/-/%s-%s.tgz' % (REGISTRY, + uri_quote(spec.name, safe = '@'), + uri_quote(spec.name, safe = '@/'), + spec.version) + + def add_pkg(self, tarball, pkg_json): + pkg_json = PackageJson(pkg_json) + base = os.path.join(self.__path, pkg_json.base_filename()) + + with Manifest(base, pkg_json) as manifest: + meta = manifest.load() + tarball_uri = self._tarball_uri(pkg_json) + + meta['versions'][pkg_json.version] = pkg_json.as_manifest_entry(tarball_uri) + + manifest.save(meta) + + ## Cache entries are a little bit dependent on the nodejs + ## version; version specific cache implementation must + ## mitigate differences + self.__cache.run('meta', self._meta_uri(pkg_json), manifest.filename); + self.__cache.run('tgz', tarball_uri, tarball); From patchwork Thu May 19 10:05:56 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Enrico Scholz X-Patchwork-Id: 8262 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 4F3B3C433F5 for ; Thu, 19 May 2022 10:06:28 +0000 (UTC) Received: from smtpout.cvg.de (smtpout.cvg.de [87.128.211.67]) by mx.groups.io with SMTP id smtpd.web08.5769.1652954775987690835 for ; Thu, 19 May 2022 03:06:19 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@sigma-chemnitz.de header.s=v2022040800 header.b=iF8FYysq; spf=pass (domain: sigma-chemnitz.de, ip: 87.128.211.67, mailfrom: enrico.scholz@sigma-chemnitz.de) Received: from mail-mta-2.intern.sigma-chemnitz.de (mail-mta-2.intern.sigma-chemnitz.de [192.168.12.70]) by mail-out-2.intern.sigma-chemnitz.de (8.16.1/8.16.1) with ESMTPS id 24JA6AmT621379 (version=TLSv1.3 cipher=TLS_AES_256_GCM_SHA384 bits=256 verify=OK) for ; Thu, 19 May 2022 12:06:11 +0200 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=sigma-chemnitz.de; s=v2022040800; t=1652954771; bh=oabpkaatAsg+o6yoxc3Os1uQmWXvlIwpVHqzUBOqdv0=; l=2574; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=iF8FYysqmBW5vYtzcN/xnsczu2dII5fY9HMc6yaS+7Y1swQNdNnyZpBFd99dXwiTc xI7x3XDPF7IBOBmz3lMFAYLrfTFYCEpCtOKNDry608MhKiRVdEnauWowlRhLOBbbeX 26KOF0gpUFtQ7vjyr7e793GLSF2nEAYdrKvBRI2p070W5AScmQrnq1GUgHft/4B1Dc A7FOUi6+cJToxT+tN2reIoV9XiMmTqhOkBXqSVj0VC5/cYcjwm9pWprh0oQYxWRlND kqTNViPRiQSOsF+YAxOVZTSnevuWI+0MbS8QX6p3IKjJNaSuztl1FjutbFd5Cqkqox CffCBtKxJGeGA== Received: from reddoxx.intern.sigma-chemnitz.de (reddoxx.sigma.local [192.168.16.32]) by mail-mta-2.intern.sigma-chemnitz.de (8.16.1/8.16.1) with ESMTP id 24JA68rm791535 for from enrico.scholz@sigma-chemnitz.de; Thu, 19 May 2022 12:06:08 +0200 Received: from mail-msa-2.intern.sigma-chemnitz.de ( [192.168.12.72]) by reddoxx.intern.sigma-chemnitz.de (Reddoxx engine) with SMTP id 559B71EC74D; Thu, 19 May 2022 12:06:06 +0200 Received: from ensc-pc.intern.sigma-chemnitz.de (ensc-pc.intern.sigma-chemnitz.de [192.168.3.24]) by mail-msa-2.intern.sigma-chemnitz.de (8.16.1/8.16.1) with ESMTPS id 24JA66ui596901 (version=TLSv1.3 cipher=TLS_AES_256_GCM_SHA384 bits=256 verify=NO); Thu, 19 May 2022 12:06:06 +0200 Received: from ensc by ensc-pc.intern.sigma-chemnitz.de with local (Exim 4.95) (envelope-from ) id 1nrd2k-004Ime-Mx; Thu, 19 May 2022 12:06:06 +0200 From: Enrico Scholz To: openembedded-core@lists.openembedded.org Cc: Enrico Scholz Subject: [PATCH 6/6] npm: use npm_registry to cache package Date: Thu, 19 May 2022 12:05:56 +0200 Message-Id: <26cd22d02a7c2544fbcb8646ca611d88ccb5286b.1652954597.git.enrico.scholz@sigma-chemnitz.de> X-Mailer: git-send-email 2.36.1 In-Reply-To: References: MIME-Version: 1.0 Sender: Enrico Scholz List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Thu, 19 May 2022 10:06:28 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/165876 With nodejs 16, the simple 'npm cache add' approach does not work anymore because its fetcher implementation downloads also meta information from the registry. We have to generate these information and add them to the cache. There is no direct support in 'npm' for task so we have to implement it manually. This implementation consists of a openembedded python module (in oe-core) and a nodejs version specific helper (in oe-meta). Signed-off-by: Enrico Scholz --- meta/classes/npm.bbclass | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/meta/classes/npm.bbclass b/meta/classes/npm.bbclass index 1e41072116..f3c3e2f5b7 100644 --- a/meta/classes/npm.bbclass +++ b/meta/classes/npm.bbclass @@ -19,7 +19,7 @@ inherit python3native -DEPENDS:prepend = "nodejs-native " +DEPENDS:prepend = "nodejs-native nodejs-oe-cache-native " RDEPENDS:${PN}:append:class-target = " nodejs" EXTRA_OENPM = "" @@ -46,6 +46,7 @@ NPM_ARCH ?= "${@npm_target_arch_map(d.getVar("TARGET_ARCH"))}" NPM_PACKAGE = "${WORKDIR}/npm-package" NPM_CACHE = "${WORKDIR}/npm-cache" NPM_BUILD = "${WORKDIR}/npm-build" +NPM_REGISTRY = "${WORKDIR}/npm-registry" def npm_global_configs(d): """Get the npm global configuration""" @@ -111,16 +112,18 @@ python npm_do_configure() { from bb.fetch2.npm import npm_unpack from bb.fetch2.npmsw import foreach_dependencies from bb.progress import OutOfProgressHandler + from oe.npm_registry import NpmRegistry bb.utils.remove(d.getVar("NPM_CACHE"), recurse=True) bb.utils.remove(d.getVar("NPM_PACKAGE"), recurse=True) env = NpmEnvironment(d, configs=npm_global_configs(d)) + registry = NpmRegistry(d.getVar('NPM_REGISTRY'), d.getVar('NPM_CACHE')) - def _npm_cache_add(tarball): - """Run 'npm cache add' for a specified tarball""" - cmd = "npm cache add %s" % shlex.quote(tarball) - env.run(cmd) + def _npm_cache_add(tarball, pkg): + """Add tarball to local registry and register it in the + cache""" + registry.add_pkg(tarball, pkg) def _npm_integrity(tarball): """Return the npm integrity of a specified tarball""" @@ -184,7 +187,7 @@ python npm_do_configure() { # Add the dependency to the npm cache destdir = os.path.join(d.getVar("S"), destsuffix) (tarball, pkg) = npm_pack(env, destdir, tmpdir) - _npm_cache_add(tarball) + _npm_cache_add(tarball, pkg) # Add its signature to the cached shrinkwrap dep = _npmsw_dependency_dict(cached_shrinkwrap, deptree) dep["version"] = pkg['version']