[meta-oe][kirkstone][add-npm-commit-from-main-to-kirkstone] nodejs-oe-cache-native: initial checkin


Paulo Vitor Becker Ayrosa Monteiro de Andrade
 
Edited

This is a commit from the master branch which is already merged.
In order to get yocto's build working on kirkstone branch, this commit is necessary on meta-openembedded as well.
I ran into npm issues when doing some work here and stumbled upon Enrico's solution to this, and it worked perfectly. If this was already merged to kirkstone branch, I believe more people would benefit.

From: Enrico Scholz <enrico.scholz@...>

This implements an 'npm cache add' like functionality but allows to
specify the key of the data and sets metadata which are required to
find the data.

It is used to cache information as done during 'npm install'.

Keyformat and metadata are nodejs version specific.

Signed-off-by: Enrico Scholz <enrico.scholz@...>
Signed-off-by: Khem Raj <raj.khem@...>
---
.../nodejs/nodejs-oe-cache-16.14/oe-npm-cache | 77 +++++++++++++++++++
.../nodejs/nodejs-oe-cache-native_16.14.bb | 21 +++++
2 files changed, 98 insertions(+)
create mode 100755 meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-16.14/oe-npm-cache
create mode 100644 meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-native_16.14.bb

diff --git a/meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-16.14/oe-npm-cache b/meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-16.14/oe-npm-cache
new file mode 100755
index 000000000..f59620764
--- /dev/null
+++ b/meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-16.14/oe-npm-cache
@@ -0,0 +1,77 @@
+#!/usr/bin/env node
+
+/// Usage: oe-npm-cache <cache-dir> <type> <key> <file-name>
+/// <type> ... meta - metainformation about package
+/// tgz - tarball
+
+const process = require("node:process");
+
+module.paths.unshift("@@libdir@@/node_modules/npm/node_modules");
+
+const cacache = require('cacache')
+const fs = require('fs')
+
+// argv[0] is 'node', argv[1] is this script
+const cache_dir = process.argv[2]
+const type = process.argv[3]
+const key = process.argv[4]
+const file = process.argv[5]
+
+const data = fs.readFileSync(file)
+
+// metadata content is highly nodejs dependent; when cache entries are not
+// found, place debug statements in 'make-fetch-happen/lib/cache/policy.js'
+// (CachePolicy::satisfies())
+const xlate = {
+ 'meta': {
+ 'key_prefix': 'make-fetch-happen:request-cache:',
+ 'metadata': function() {
+ return {
+ time: Date.now(),
+ url: key,
+ reqHeaders: {
+ 'accept': 'application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8, */*',
+ },
+ resHeaders: {
+ "content-type": "application/json",
+ "status": 200,
+ },
+ options: {
+ compress: true,
+ }
+ };
+ },
+ },
+
+ 'tgz': {
+ 'key_prefix': 'make-fetch-happen:request-cache:',
+ 'metadata': function() {
+ return {
+ time: Date.now(),
+ url: key,
+ reqHeaders: {
+ 'accept': '*/*',
+ },
+ resHeaders: {
+ "content-type": "application/octet-stream",
+ "status": 200,
+ },
+ options: {
+ compress: true,
+ },
+ };
+ },
+ },
+};
+
+const info = xlate[type];
+let opts = {}
+
+if (info.metadata) {
+ opts['metadata'] = info.metadata();
+}
+
+cacache.put(cache_dir, info.key_prefix + key, data, opts)
+ .then(integrity => {
+ console.log(`Saved content of ${key} (${file}).`);
+})
diff --git a/meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-native_16.14.bb b/meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-native_16.14.bb
new file mode 100644
index 000000000..a61dd5018
--- /dev/null
+++ b/meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-native_16.14.bb
@@ -0,0 +1,21 @@
+DESCRIPTION = "OE helper for manipulating npm cache"
+LICENSE = "Apache-2.0"
+LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/Apache-2.0;md5=89aea4e17d99a7cacdbeed46a0096b10"
+
+SRC_URI = "\
+ file://oe-npm-cache \
+"
+
+inherit native
+
+B = "${WORKDIR}/build"
+
+do_configure() {
+ sed -e 's!@@libdir@@!${libdir}!g' < '${WORKDIR}/oe-npm-cache' > '${B}/oe-npm-cache'
+}
+
+do_install() {
+ install -D -p -m 0755 ${B}/oe-npm-cache ${D}${bindir}/oe-npm-cache
+}
+
+RDEPENDS:${PN} = "nodejs-native"
--
2.17.1


Martin Jansa
 

I've already sent this backport request yesterday.


On Wed, Sep 14, 2022 at 4:26 PM Paulo Vitor Becker Ayrosa Monteiro de Andrade <paulo@...> wrote:
[Edited Message Follows]

This is a commit from the master branch which is already merged.
In order to get yocto's build working on kirkstone branch, this commit is necessary on meta-openembedded as well.
I ran into npm issues when doing some work here and stumbled upon Enrico's solution to this, and it worked perfectly. If this was already merged to kirkstone branch, I believe more people would benefit.

From: Enrico Scholz <enrico.scholz@...>

This implements an 'npm cache add' like functionality but allows to
specify the key of the data and sets metadata which are required to
find the data.

It is used to cache information as done during 'npm install'.

Keyformat and metadata are nodejs version specific.

Signed-off-by: Enrico Scholz <enrico.scholz@...>
Signed-off-by: Khem Raj <raj.khem@...>
---
 .../nodejs/nodejs-oe-cache-16.14/oe-npm-cache | 77 +++++++++++++++++++
 .../nodejs/nodejs-oe-cache-native_16.14.bb    | 21 +++++
 2 files changed, 98 insertions(+)
 create mode 100755 meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-16.14/oe-npm-cache
 create mode 100644 meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-native_16.14.bb

diff --git a/meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-16.14/oe-npm-cache b/meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-16.14/oe-npm-cache
new file mode 100755
index 000000000..f59620764
--- /dev/null
+++ b/meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-16.14/oe-npm-cache
@@ -0,0 +1,77 @@
+#!/usr/bin/env node
+
+/// Usage: oe-npm-cache <cache-dir> <type> <key> <file-name>
+///    <type> ... meta - metainformation about package
+///               tgz  - tarball
+
+const process = require("node:process");
+
+module.paths.unshift("@@libdir@@/node_modules/npm/node_modules");
+
+const cacache = require('cacache')
+const fs = require('fs')
+
+// argv[0] is 'node', argv[1] is this script
+const cache_dir = process.argv[2]
+const type      = process.argv[3]
+const key       = process.argv[4]
+const file      = process.argv[5]
+
+const data = fs.readFileSync(file)
+
+// metadata content is highly nodejs dependent; when cache entries are not
+// found, place debug statements in 'make-fetch-happen/lib/cache/policy.js'
+// (CachePolicy::satisfies())
+const xlate = {
+    'meta': {
+       'key_prefix': 'make-fetch-happen:request-cache:',
+       'metadata': function() {
+           return {
+               time: Date.now(),
+               url:  key,
+               reqHeaders: {
+                   'accept': 'application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8, */*',
+               },
+               resHeaders: {
+                   "content-type": "application/json",
+                   "status": 200,
+               },
+               options: {
+                   compress: true,
+               }
+           };
+       },
+    },
+
+    'tgz': {
+       'key_prefix': 'make-fetch-happen:request-cache:',
+       'metadata': function() {
+           return {
+               time: Date.now(),
+               url:  key,
+               reqHeaders: {
+                   'accept': '*/*',
+               },
+               resHeaders: {
+                   "content-type": "application/octet-stream",
+                   "status": 200,
+               },
+               options: {
+                   compress: true,
+               },
+           };
+       },
+    },
+};
+
+const info = xlate[type];
+let opts = {}
+
+if (info.metadata) {
+    opts['metadata'] = info.metadata();
+}
+
+cacache.put(cache_dir, info.key_prefix + key, data, opts)
+    .then(integrity => {
+       console.log(`Saved content of ${key} (${file}).`);
+})
diff --git a/meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-native_16.14.bb b/meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-native_16.14.bb
new file mode 100644
index 000000000..a61dd5018
--- /dev/null
+++ b/meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-native_16.14.bb
@@ -0,0 +1,21 @@
+DESCRIPTION = "OE helper for manipulating npm cache"
+LICENSE = "Apache-2.0"
+LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/Apache-2.0;md5=89aea4e17d99a7cacdbeed46a0096b10"
+
+SRC_URI = "\
+    file://oe-npm-cache \
+"
+
+inherit native
+
+B = "${WORKDIR}/build"
+
+do_configure() {
+    sed -e 's!@@libdir@@!${libdir}!g' < '${WORKDIR}/oe-npm-cache' > '${B}/oe-npm-cache'
+}
+
+do_install() {
+    install -D -p -m 0755 ${B}/oe-npm-cache ${D}${bindir}/oe-npm-cache
+}
+
+RDEPENDS:${PN} = "nodejs-native"
--
2.17.1





Paulo Vitor Becker Ayrosa Monteiro de Andrade
 

Awesome!

I tried sending it earlier, but had some problems. Thanks a bunch, then!


On Wed, Sep 14, 2022 at 10:27 AM Martin Jansa <martin.jansa@...> wrote:
I've already sent this backport request yesterday.

On Wed, Sep 14, 2022 at 4:26 PM Paulo Vitor Becker Ayrosa Monteiro de Andrade <paulo@...> wrote:
[Edited Message Follows]

This is a commit from the master branch which is already merged.
In order to get yocto's build working on kirkstone branch, this commit is necessary on meta-openembedded as well.
I ran into npm issues when doing some work here and stumbled upon Enrico's solution to this, and it worked perfectly. If this was already merged to kirkstone branch, I believe more people would benefit.

From: Enrico Scholz <enrico.scholz@...>

This implements an 'npm cache add' like functionality but allows to
specify the key of the data and sets metadata which are required to
find the data.

It is used to cache information as done during 'npm install'.

Keyformat and metadata are nodejs version specific.

Signed-off-by: Enrico Scholz <enrico.scholz@...>
Signed-off-by: Khem Raj <raj.khem@...>
---
 .../nodejs/nodejs-oe-cache-16.14/oe-npm-cache | 77 +++++++++++++++++++
 .../nodejs/nodejs-oe-cache-native_16.14.bb    | 21 +++++
 2 files changed, 98 insertions(+)
 create mode 100755 meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-16.14/oe-npm-cache
 create mode 100644 meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-native_16.14.bb

diff --git a/meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-16.14/oe-npm-cache b/meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-16.14/oe-npm-cache
new file mode 100755
index 000000000..f59620764
--- /dev/null
+++ b/meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-16.14/oe-npm-cache
@@ -0,0 +1,77 @@
+#!/usr/bin/env node
+
+/// Usage: oe-npm-cache <cache-dir> <type> <key> <file-name>
+///    <type> ... meta - metainformation about package
+///               tgz  - tarball
+
+const process = require("node:process");
+
+module.paths.unshift("@@libdir@@/node_modules/npm/node_modules");
+
+const cacache = require('cacache')
+const fs = require('fs')
+
+// argv[0] is 'node', argv[1] is this script
+const cache_dir = process.argv[2]
+const type      = process.argv[3]
+const key       = process.argv[4]
+const file      = process.argv[5]
+
+const data = fs.readFileSync(file)
+
+// metadata content is highly nodejs dependent; when cache entries are not
+// found, place debug statements in 'make-fetch-happen/lib/cache/policy.js'
+// (CachePolicy::satisfies())
+const xlate = {
+    'meta': {
+       'key_prefix': 'make-fetch-happen:request-cache:',
+       'metadata': function() {
+           return {
+               time: Date.now(),
+               url:  key,
+               reqHeaders: {
+                   'accept': 'application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8, */*',
+               },
+               resHeaders: {
+                   "content-type": "application/json",
+                   "status": 200,
+               },
+               options: {
+                   compress: true,
+               }
+           };
+       },
+    },
+
+    'tgz': {
+       'key_prefix': 'make-fetch-happen:request-cache:',
+       'metadata': function() {
+           return {
+               time: Date.now(),
+               url:  key,
+               reqHeaders: {
+                   'accept': '*/*',
+               },
+               resHeaders: {
+                   "content-type": "application/octet-stream",
+                   "status": 200,
+               },
+               options: {
+                   compress: true,
+               },
+           };
+       },
+    },
+};
+
+const info = xlate[type];
+let opts = {}
+
+if (info.metadata) {
+    opts['metadata'] = info.metadata();
+}
+
+cacache.put(cache_dir, info.key_prefix + key, data, opts)
+    .then(integrity => {
+       console.log(`Saved content of ${key} (${file}).`);
+})
diff --git a/meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-native_16.14.bb b/meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-native_16.14.bb
new file mode 100644
index 000000000..a61dd5018
--- /dev/null
+++ b/meta-oe/recipes-devtools/nodejs/nodejs-oe-cache-native_16.14.bb
@@ -0,0 +1,21 @@
+DESCRIPTION = "OE helper for manipulating npm cache"
+LICENSE = "Apache-2.0"
+LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/Apache-2.0;md5=89aea4e17d99a7cacdbeed46a0096b10"
+
+SRC_URI = "\
+    file://oe-npm-cache \
+"
+
+inherit native
+
+B = "${WORKDIR}/build"
+
+do_configure() {
+    sed -e 's!@@libdir@@!${libdir}!g' < '${WORKDIR}/oe-npm-cache' > '${B}/oe-npm-cache'
+}
+
+do_install() {
+    install -D -p -m 0755 ${B}/oe-npm-cache ${D}${bindir}/oe-npm-cache
+}
+
+RDEPENDS:${PN} = "nodejs-native"
--
2.17.1