fix: tracking really large repos in chunks, lower limit for windows

* fix: chunk really large git.add, better error output

* test: nut for large project scale

* refactor: smaller file limit for windows

* refactor: come on windows, you can do it

* refactor: use ternary correctly

Author: mshanemc

package.json

@@ -47,7 +47,7 @@
     "@salesforce/kit": "^1.5.17",
     "@salesforce/source-deploy-retrieve": "^5.9.4",
     "graceful-fs": "^4.2.9",
-    "isomorphic-git": "1.16.0",
+    "isomorphic-git": "1.17.0",
     "ts-retry-promise": "^0.6.0"
   },
   "devDependencies": {

src/shared/functions.ts

@@ -38,3 +38,7 @@ export const pathIsInFolder = (filePath: string, folder: string): boolean => {
 const nonEmptyStringFilter = (value: string): boolean => {
   return isString(value) && value.length > 0;
 };
+
+// adapted for TS from https://github.com/30-seconds/30-seconds-of-code/blob/master/snippets/chunk.md
+export const chunkArray = <T>(arr: T[], size: number): T[][] =>
+  Array.from({ length: Math.ceil(arr.length / size) }, (v, i) => arr.slice(i * size, i * size + size));

src/shared/localShadowRepo.ts

@@ -8,14 +8,11 @@
 import * as path from 'path';
 import * as os from 'os';
 import * as fs from 'graceful-fs';
-import { NamedPackageDir, Logger } from '@salesforce/core';
+import { NamedPackageDir, Logger, SfdxError } from '@salesforce/core';
 import * as git from 'isomorphic-git';
-import { pathIsInFolder } from './functions';
+import { chunkArray, pathIsInFolder } from './functions';
 
-const gitIgnoreFileName = '.gitignore';
-/**
- * returns the full path to where we store the shadow repo
- */
+/** returns the full path to where we store the shadow repo */
 const getGitDir = (orgId: string, projectPath: string): string => {
   return path.join(projectPath, '.sfdx', 'orgs', orgId, 'localSourceTracking');
 };
@@ -53,11 +50,17 @@ export class ShadowRepo {
   private packageDirs!: NamedPackageDir[];
   private status!: StatusRow[];
   private logger!: Logger;
+  private isWindows: boolean;
+
+  /** do not try to add more than this many files at a time through isogit.  You'll hit EMFILE: too many open files even with graceful-fs */
+  private maxFileAdd: number;
 
   private constructor(options: ShadowRepoOptions) {
     this.gitDir = getGitDir(options.orgId, options.projectPath);
     this.projectPath = options.projectPath;
     this.packageDirs = options.packageDirs;
+    this.isWindows = os.type() === 'Windows_NT';
+    this.maxFileAdd = this.isWindows ? 8000 : 15000;
   }
 
   // think of singleton behavior but unique to the projectPath
@@ -113,14 +116,12 @@ export class ShadowRepo {
    */
   public async getStatus(noCache = false): Promise<StatusRow[]> {
     if (!this.status || noCache) {
-      // only ask about OS once but use twice
-      const isWindows = os.type() === 'Windows_NT';
       // iso-git uses relative, posix paths
       // but packageDirs has already resolved / normalized them
       // so we need to make them project-relative again and convert if windows
       const filepaths = this.packageDirs
         .map((dir) => path.relative(this.projectPath, dir.fullPath))
-        .map((p) => (isWindows ? p.split(path.sep).join(path.posix.sep) : p));
+        .map((p) => (this.isWindows ? p.split(path.sep).join(path.posix.sep) : p));
 
       // status hasn't been initalized yet
       this.status = await git.statusMatrix({
@@ -135,12 +136,12 @@ export class ShadowRepo {
           // no lwc tests
           !f.includes('__tests__') &&
           // no gitignore files
-          !f.endsWith(gitIgnoreFileName) &&
+          !f.endsWith('.gitignore') &&
           // isogit uses `startsWith` for filepaths so it's possible to get a false positive
           filepaths.some((pkgDir) => pathIsInFolder(f, pkgDir)),
       });
       // isomorphic-git stores things in unix-style tree.  Convert to windows-style if necessary
-      if (isWindows) {
+      if (this.isWindows) {
         this.status = this.status.map((row) => [path.normalize(row[FILE]), row[HEAD], row[WORKDIR], row[3]]);
       }
     }
@@ -229,13 +230,26 @@ export class ShadowRepo {
     }
 
     if (deployedFiles.length) {
-      await git.add({
-        fs,
-        dir: this.projectPath,
-        gitdir: this.gitDir,
-        filepath: [...new Set(deployedFiles)],
-        force: true,
-      });
+      const chunks = chunkArray([...new Set(deployedFiles)], this.maxFileAdd);
+      for (const chunk of chunks) {
+        try {
+          await git.add({
+            fs,
+            dir: this.projectPath,
+            gitdir: this.gitDir,
+            filepath: chunk,
+            force: true,
+          });
+        } catch (e) {
+          if (e instanceof git.Errors.MultipleGitError) {
+            this.logger.error('multiple errors on git.add', e.errors.slice(0, 5));
+            const error = new SfdxError(e.message, e.name, [], 1);
+            error.setData(e.errors);
+            throw error;
+          }
+          throw e;
+        }
+      }
     }
 
     for (const filepath of [...new Set(deletedFiles)]) {

test/nuts/local/tracking-scale.nut.ts

@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2020, salesforce.com, inc.
+ * All rights reserved.
+ * Licensed under the BSD 3-Clause license.
+ * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+ */
+import * as path from 'path';
+import { TestSession } from '@salesforce/cli-plugins-testkit';
+import { expect } from 'chai';
+import { fs } from '@salesforce/core';
+import { ShadowRepo } from '../../../src/shared/localShadowRepo';
+
+const dirCount = 200;
+const classesPerDir = 500;
+const classCount = dirCount * classesPerDir;
+
+describe(`verify tracking handles an add of ${classCount.toLocaleString()} classes (${(
+  classCount * 2
+).toLocaleString()} files across ${dirCount.toLocaleString()} folders)`, () => {
+  let session: TestSession;
+  let repo: ShadowRepo;
+  let filesToSync: string[];
+
+  before(async () => {
+    session = await TestSession.create({
+      project: {
+        name: 'large-repo',
+      },
+      authStrategy: 'NONE',
+    });
+    // create some number of files
+    const classdir = path.join(session.project.dir, 'force-app', 'main', 'default', 'classes');
+    for (let d = 0; d < dirCount; d++) {
+      const dirName = path.join(classdir, `dir${d}`);
+      await fs.promises.mkdir(dirName);
+      for (let c = 0; c < classesPerDir; c++) {
+        const className = `x${d}x${c}`;
+        await Promise.all([
+          fs.promises.writeFile(
+            path.join(dirName, `${className}.cls`),
+            `public with sharing class ${className} {public ${className}() {}}`
+          ),
+          fs.promises.writeFile(
+            path.join(dirName, `${className}.cls-meta.xml`),
+            '<?xml version="1.0" encoding="UTF-8"?><ApexClass xmlns="http://soap.sforce.com/2006/04/metadata"><apiVersion>54.0</apiVersion><status>Active</status></ApexClass>'
+          ),
+        ]);
+      }
+    }
+  });
+
+  after(async () => {
+    await session?.clean();
+  });
+
+  it('initialize the local tracking', async () => {
+    repo = await ShadowRepo.getInstance({
+      orgId: 'fakeOrgId',
+      projectPath: session.project.dir,
+      packageDirs: [{ path: 'force-app', name: 'force-app', fullPath: path.join(session.project.dir, 'force-app') }],
+    });
+  });
+
+  it(`should see ${(classCount * 2).toLocaleString()} files (git status)`, async () => {
+    filesToSync = await repo.getChangedFilenames();
+    expect(filesToSync)
+      .to.be.an('array')
+      // windows ends up with 2 extra files!?
+      .with.length.greaterThanOrEqual(classCount * 2);
+  });
+
+  it('should sync (commit) them locally without error', async () => {
+    await repo.commitChanges({ deployedFiles: filesToSync, needsUpdatedStatus: false });
+  });
+});

yarn.lock

@@ -3387,10 +3387,10 @@ isexe@^2.0.0:
   resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10"
   integrity sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=
 
-isomorphic-git@1.16.0:
-  version "1.16.0"
-  resolved "https://registry.yarnpkg.com/isomorphic-git/-/isomorphic-git-1.16.0.tgz#f4de2fd0e1c78bbbe03230fbefa88a84463e736a"
-  integrity sha512-pyYcMRp0125hmxsagSaAN63WgHd4x+4sR3eJ71+xdIN0aOij0q5ASPIN7jiJissUHx9/FE4dY73pzDehk+Xomw==
+isomorphic-git@1.17.0:
+  version "1.17.0"
+  resolved "https://registry.yarnpkg.com/isomorphic-git/-/isomorphic-git-1.17.0.tgz#8bd423ddb8ebfb924799be0ac75fb5bede5cfad7"
+  integrity sha512-8ToEVqYLeTE1Ys3UQ21yAxQf0rW7GYRvsENhvXNDONAHgNks1fsgUJH3mVzgbsGf4LpW3kuJI6e/e3VIeaTW3w==
   dependencies:
     async-lock "^1.1.0"
     clean-git-ref "^2.0.1"