Adds support for running native binaries without JS intermediaries (#…

…5508)

**What's the problem this PR addresses?**

Yarn currently cannot run native binaries without going through a JS
jumper script. Other package managers don't have this restriction, and
it makes the `yarn run` overhead worse on some scripts for little
reasons - or doesn't work at all when packages don't use jumper scripts.

**How did you fix it?**

Two mechanisms are used:

- First we check for the binary extension
- Then we check its magic number

If one of the two match a predetermined list, the binary is spawned
without going through Node. This ensures that this logic is called only
when the binary is truly a native binary, and will not affect the
behaviour of other existing scripts.

**Checklist**
<!--- Don't worry if you miss something, chores are automatically
tested. -->
<!--- This checklist exists to help you remember doing the chores when
you submit a PR. -->
<!--- Put an `x` in all the boxes that apply. -->
- [x] I have read the [Contributing
Guide](https://yarnpkg.com/advanced/contributing).

<!-- See
https://yarnpkg.com/advanced/contributing#preparing-your-pr-to-be-released
for more details. -->
<!-- Check with `yarn version check` and fix with `yarn version check
-i` -->
- [x] I have set the packages that need to be released for my changes to
be effective.

<!-- The "Testing chores" workflow validates that your PR follows our
guidelines. -->
<!-- If it doesn't pass, click on it to see details as to what your PR
might be missing. -->
- [x] I will check that all automated PR checks pass before the PR gets
reviewed.

.yarn/versions/92ff1c9e.yml

@@ -0,0 +1,34 @@
+releases:
+  "@yarnpkg/cli": patch
+  "@yarnpkg/core": patch
+  "@yarnpkg/plugin-pnp": patch
+
+declined:
+  - "@yarnpkg/plugin-compat"
+  - "@yarnpkg/plugin-constraints"
+  - "@yarnpkg/plugin-dlx"
+  - "@yarnpkg/plugin-essentials"
+  - "@yarnpkg/plugin-exec"
+  - "@yarnpkg/plugin-file"
+  - "@yarnpkg/plugin-git"
+  - "@yarnpkg/plugin-github"
+  - "@yarnpkg/plugin-http"
+  - "@yarnpkg/plugin-init"
+  - "@yarnpkg/plugin-interactive-tools"
+  - "@yarnpkg/plugin-link"
+  - "@yarnpkg/plugin-nm"
+  - "@yarnpkg/plugin-npm"
+  - "@yarnpkg/plugin-npm-cli"
+  - "@yarnpkg/plugin-pack"
+  - "@yarnpkg/plugin-patch"
+  - "@yarnpkg/plugin-pnpm"
+  - "@yarnpkg/plugin-stage"
+  - "@yarnpkg/plugin-typescript"
+  - "@yarnpkg/plugin-version"
+  - "@yarnpkg/plugin-workspace-tools"
+  - "@yarnpkg/builder"
+  - "@yarnpkg/doctor"
+  - "@yarnpkg/extensions"
+  - "@yarnpkg/nm"
+  - "@yarnpkg/pnpify"
+  - "@yarnpkg/sdks"

packages/acceptance-tests/pkg-tests-specs/sources/script.test.js → packages/acceptance-tests/pkg-tests-specs/sources/script.test.ts

@@ -1,8 +1,12 @@
-const {npath, ppath, xfs, Filename} = require(`@yarnpkg/fslib`);
-const {isAbsolute, resolve} = require(`path`);
+import {npath, ppath, xfs, Filename, PortablePath} from '@yarnpkg/fslib';
+import {execFile}                                  from 'child_process';
+import {isAbsolute, resolve}                       from 'path';
+import {fs}                                        from 'pkg-tests-core';
+import util                                        from 'util';
 
+const execP = util.promisify(execFile);
 const {
-  fs: {createTemporaryFolder, makeFakeBinary, walk, readFile, writeJson, writeFile},
+  fs: {createTemporaryFolder, walk, readFile},
 } = require(`pkg-tests-core`);
 
 const globalName = makeTemporaryEnv.getPackageManagerName();
@@ -21,7 +25,7 @@ describe(`Scripts tests`, () => {
     makeTemporaryEnv({scripts: {myScript: `node --version`}}, async ({path, run, source}) => {
       await run(`install`);
 
-      await makeFakeBinary(`${path}/bin/node`, {exitCode: 0});
+      await fs.makeFakeBinary(ppath.join(path, `/bin/node` as PortablePath), {exitCode: 0});
 
       await expect(run(`run`, `myScript`)).resolves.not.toMatchObject({stdout: `Fake binary`});
     }),
@@ -32,7 +36,7 @@ describe(`Scripts tests`, () => {
     makeTemporaryEnv({scripts: {myScript: `${globalName} --version`}}, async ({path, run, source}) => {
       await run(`install`);
 
-      await makeFakeBinary(`${path}/bin/${globalName}`, {exitCode: 0});
+      await fs.makeFakeBinary(ppath.join(path, `/bin/${globalName}` as PortablePath), {exitCode: 0});
 
       await expect(run(`run`, `myScript`)).resolves.not.toMatchObject({stdout: `Fake binary`});
     }),
@@ -69,9 +73,9 @@ describe(`Scripts tests`, () => {
       private: true,
       workspaces: [`packages/*`],
     }, async ({path, run, source}) => {
-      await xfs.mkdirpPromise(`${path}/packages/test`);
+      await xfs.mkdirpPromise(ppath.join(path, `/packages/test` as PortablePath));
 
-      await xfs.writeJsonPromise(`${path}/packages/test/package.json`, {
+      await xfs.writeJsonPromise(ppath.join(path, `/packages/test/package.json` as PortablePath), {
         scripts: {
           [`test:script`]: `echo "$INIT_CWD"`,
         },
@@ -84,9 +88,9 @@ describe(`Scripts tests`, () => {
       });
 
       await expect(run(`run`, `test:script`, {
-        cwd: `${path}/packages`,
+        cwd: ppath.join(path, `/packages` as PortablePath),
       })).resolves.toMatchObject({
-        stdout: `${npath.fromPortablePath(`${path}/packages`)}\n`,
+        stdout: `${npath.fromPortablePath(ppath.join(path, `/packages` as PortablePath))}\n`,
       });
     }),
   );
@@ -97,9 +101,9 @@ describe(`Scripts tests`, () => {
       private: true,
       workspaces: [`packages/*`],
     }, async ({path, run, source}) => {
-      await xfs.mkdirpPromise(`${path}/packages/test`);
+      await xfs.mkdirpPromise(ppath.join(path, `/packages/test` as PortablePath));
 
-      await xfs.writeJsonPromise(`${path}/packages/test/package.json`, {
+      await xfs.writeJsonPromise(ppath.join(path, `/packages/test/package.json` as PortablePath), {
         scripts: {
           [`test:script`]: `echo "$PROJECT_CWD"`,
         },
@@ -112,7 +116,7 @@ describe(`Scripts tests`, () => {
       });
 
       await expect(run(`run`, `test:script`, {
-        cwd: `${path}/packages`,
+        cwd: ppath.join(path, `/packages` as PortablePath),
       })).resolves.toMatchObject({
         stdout: `${npath.fromPortablePath(path)}\n`,
       });
@@ -125,7 +129,8 @@ describe(`Scripts tests`, () => {
       private: true,
       workspaces: [`packages/*`],
     }, async ({path, run, source}) => {
-      await writeJson(`${path}/packages/test 1/package.json`, {
+      await xfs.mkdirpPromise(ppath.join(path, `/packages/test 1` as PortablePath));
+      await xfs.writeJsonPromise(ppath.join(path, `/packages/test 1/package.json` as PortablePath), {
         scripts: {
           [`ws:foo2`]: `yarn run ws:foo`,
           [`ws:foo`]: `node -e 'console.log(1)'`,
@@ -174,22 +179,22 @@ describe(`Scripts tests`, () => {
     }, async ({path, run, source}) => {
       await run(`install`);
 
-      await writeFile(`${path}/test.js`, `
-      const {existsSync} = require('fs');
-      const {join} = require('path');
+      await xfs.writeFilePromise(ppath.join(path, `/test.js` as PortablePath), `
+        const {existsSync} = require('fs');
+        const {join} = require('path');
 
-      const files = ['has-bin-entries'];
-      if (process.platform === 'win32')
-        files.push('has-bin-entries.cmd');
+        const files = ['has-bin-entries'];
+        if (process.platform === 'win32')
+          files.push('has-bin-entries.cmd');
 
-      for (const file of files) {
-        if (!existsSync(join(process.env.BERRY_BIN_FOLDER, file))) {
-          console.error('Expected ' + file + ' to exist');
-          process.exit(1);
+        for (const file of files) {
+          if (!existsSync(join(process.env.BERRY_BIN_FOLDER, file))) {
+            console.error('Expected ' + file + ' to exist');
+            process.exit(1);
+          }
         }
-      }
 
-      console.log('ok');
+        console.log('ok');
       `);
 
       await expect(run(`test`)).resolves.toMatchObject({
@@ -209,7 +214,7 @@ describe(`Scripts tests`, () => {
         },
       },
       async ({path, run, source}) => {
-        await xfs.writeFilePromise(`${path}/å.js`, `console.log('ok')`);
+        await xfs.writeFilePromise(ppath.join(path, `/å.js` as PortablePath), `console.log('ok')`);
         await run(`install`);
 
         await expect(run(`test`)).resolves.toMatchObject({
@@ -230,11 +235,11 @@ describe(`Scripts tests`, () => {
       async ({path, run, source}) => {
         await run(`install`);
 
-        await xfs.mkdirpPromise(`${path}/foo/bar`);
+        await xfs.mkdirpPromise(ppath.join(path, `/foo/bar` as PortablePath));
 
         await expect(
           run(`run`, `has-bin-entries`, `success`, {
-            cwd: `${path}/foo/bar`,
+            cwd: ppath.join(path, `/foo/bar` as PortablePath),
           }),
         ).resolves.toMatchObject({
           stdout: `success\n`,
@@ -364,15 +369,15 @@ describe(`Scripts tests`, () => {
             [`install`]: `echo 'module.exports.push("root");' >> log.js`,
           },
         }, async ({path, run, source}) => {
-          await xfs.mkdirPromise(ppath.join(path, `child`));
-          await xfs.writeJsonPromise(ppath.join(path, `child/package.json`), {
+          await xfs.mkdirPromise(ppath.join(path, `child` as PortablePath));
+          await xfs.writeJsonPromise(ppath.join(path, `child/package.json` as PortablePath), {
             name: `child`,
             scripts: {
               [`install`]: `echo 'module.exports.push("child");' >> ../log.js`,
             },
           });
 
-          await xfs.writeFilePromise(ppath.join(path, `log.js`), `module.exports = [];\n`);
+          await xfs.writeFilePromise(ppath.join(path, `log.js` as PortablePath), `module.exports = [];\n`);
           await run(`install`);
 
           await expect(source(`require("./log")`)).resolves.toEqual([
@@ -396,16 +401,16 @@ describe(`Scripts tests`, () => {
             [`install`]: `echo 'module.exports.push("root");' >> log.js`,
           },
         }, async ({path, run, source}) => {
-          await xfs.mkdirPromise(ppath.join(path, `child`), {recursive: true});
-          await xfs.writeJsonPromise(ppath.join(path, `child/package.json`), {
+          await xfs.mkdirPromise(ppath.join(path, `child` as PortablePath), {recursive: true});
+          await xfs.writeJsonPromise(ppath.join(path, `child/package.json` as PortablePath), {
             name: `child`,
             scripts: {
               postinstall: `echo 'module.exports.push("child");' >> ../log.js`,
             },
           });
 
           await run(`install`);
-          await xfs.writeFilePromise(ppath.join(path, `log.js`), `module.exports = [];\n`);
+          await xfs.writeFilePromise(ppath.join(path, `log.js` as PortablePath), `module.exports = [];\n`);
 
           await run(`./child`, `add`, `no-deps@1.0.0`);
 
@@ -430,19 +435,19 @@ describe(`Scripts tests`, () => {
             [`install`]: `echo 'module.exports.push("root");' >> log.js`,
           },
         }, async ({path, run, source}) => {
-          await xfs.mkdirPromise(ppath.join(path, `packages/first`), {recursive: true});
-          await xfs.mkdirPromise(ppath.join(path, `packages/second`), {recursive: true});
+          await xfs.mkdirPromise(ppath.join(path, `packages/first` as PortablePath), {recursive: true});
+          await xfs.mkdirPromise(ppath.join(path, `packages/second` as PortablePath), {recursive: true});
 
-          await xfs.writeJsonPromise(ppath.join(path, `packages/first/package.json`), {
+          await xfs.writeJsonPromise(ppath.join(path, `packages/first/package.json` as PortablePath), {
             name: `first`,
           });
 
-          await xfs.writeJsonPromise(ppath.join(path, `packages/second/package.json`), {
+          await xfs.writeJsonPromise(ppath.join(path, `packages/second/package.json` as PortablePath), {
             name: `bar`,
           });
 
           await run(`install`);
-          await xfs.writeFilePromise(ppath.join(path, `log.js`), `module.exports = [];`);
+          await xfs.writeFilePromise(ppath.join(path, `log.js` as PortablePath), `module.exports = [];`);
 
           await run(`packages/second`, `add`, `no-deps@1.0.0`);
 
@@ -603,13 +608,13 @@ describe(`Scripts tests`, () => {
           },
           config,
           async ({path, run, source}) => {
-            await xfs.mkdirPromise(`${path}/soft-link`);
-            await xfs.writeJsonPromise(`${path}/soft-link/package.json`, {
+            await xfs.mkdirPromise(ppath.join(path, `/soft-link` as PortablePath));
+            await xfs.writeJsonPromise(ppath.join(path, `/soft-link/package.json` as PortablePath), {
               name: `soft-link`,
               version: `1.0.0`,
               bin: `./bin`,
             });
-            await xfs.writeFilePromise(`${path}/soft-link/bin.js`, `console.log(42);\n`);
+            await xfs.writeFilePromise(ppath.join(path, `/soft-link/bin.js` as PortablePath), `console.log(42);\n`);
 
             await run(`install`);
 
@@ -621,6 +626,53 @@ describe(`Scripts tests`, () => {
           },
         ),
       );
+
+      test(
+        `it should run native binaries`,
+        makeTemporaryEnv({}, async ({path, run, source}) => {
+          const gitProcess = await execP(`git`, [`--exec-path`]);
+
+          const gitExt = process.platform === `win32` ? `.exe` : ``;
+          const gitPath = ppath.join(npath.toPortablePath(gitProcess.stdout.trim()), `git${gitExt}` as PortablePath);
+
+          await xfs.copyFilePromise(gitPath, ppath.join(path, `foo${gitExt}` as PortablePath));
+
+          await xfs.writeJsonPromise(ppath.join(path, Filename.manifest), {
+            bin: {
+              foo: `./foo${gitExt}`,
+            },
+          });
+
+          await run(`install`);
+
+          await run(`run`, `foo`, `--version`);
+        }),
+      );
+
+      test(
+        `it should add native binaries to the PATH`,
+        makeTemporaryEnv({}, async ({path, run, source}) => {
+          const gitProcess = await execP(`git`, [`--exec-path`]);
+
+          const gitExt = process.platform === `win32` ? `.exe` : ``;
+          const gitPath = ppath.join(npath.toPortablePath(gitProcess.stdout.trim()), `git${gitExt}` as PortablePath);
+
+          await xfs.copyFilePromise(gitPath, ppath.join(path, `foo${gitExt}` as PortablePath));
+
+          await xfs.writeJsonPromise(ppath.join(path, Filename.manifest), {
+            bin: {
+              foo: `./foo${gitExt}`,
+            },
+            scripts: {
+              bar: `foo`,
+            },
+          });
+
+          await run(`install`);
+
+          console.log(await run(`run`, `bar`, `--version`));
+        }),
+      );
     });
   }
 });

packages/plugin-pnp/sources/jsInstallUtils.ts

@@ -60,6 +60,8 @@ export function extractBuildScripts(pkg: Package, requirements: ExtractBuildScri
 const FORCED_EXTRACT_FILETYPES = new Set([
   // Windows can't execute exe files inside zip archives
   `.exe`,
+  // May be used for some binaries on Linux; https://askubuntu.com/a/174356
+  `.bin`,
   // The c/c++ compiler can't read files from zip archives
   `.h`, `.hh`, `.hpp`, `.c`, `.cc`, `.cpp`,
   // The java runtime can't read files from zip archives