feat: push supporting bundle types

Author: mshanemc

.gitignore

@@ -39,3 +39,5 @@ node_modules
 # os specific files
 .DS_Store
 .idea
+
+testProj

README.md

@@ -8,12 +8,16 @@ You should use the class named sourceTracking.
 
 ## TODO
 
+consolidate the conflict logic into SourceTracking. Should return a ChangeResult[] of the conflicts so commands can display.
+
 after pushing ebikes (testProj) the remote changes are showing in source tracking (they should have been polled for and retrieved!)
 
 can migrate maxRevision.json to its new home
 
 integration testing
 
+why does push take so long?
+
 Push can have partial successes and needs a proper status code ex:
 
 ```json

src/commands/source/pull.ts

@@ -12,7 +12,6 @@ import { SfdxProject, Org } from '@salesforce/core';
 import { ComponentSet, ComponentStatus } from '@salesforce/source-deploy-retrieve';
 import { writeConflictTable } from '../../writeConflictTable';
 import { SourceTracking, ChangeResult } from '../../sourceTracking';
-
 export default class SourcePull extends SfdxCommand {
   public static description = 'get local changes';
   protected static readonly flagsConfig: FlagsConfig = {
@@ -67,14 +66,25 @@ export default class SourcePull extends SfdxCommand {
       const changesToDeleteWithFilePaths = tracking.populateFilePaths(changesToDelete);
       // delete the files
       const filenames = changesToDeleteWithFilePaths
+        // TODO: test that this works for undefined, string and string[]
         .map((change) => change.filenames as string[])
         .flat()
         .filter(Boolean);
       await Promise.all(filenames.map((filename) => unlink(filename)));
       await Promise.all([
         tracking.updateLocalTracking({ deletedFiles: filenames }),
         tracking.updateRemoteTracking(
-          changesToDeleteWithFilePaths.map((change) => ({ type: change.type as string, name: change.name as string }))
+          changesToDeleteWithFilePaths
+            .map((change) =>
+              change && change.filenames
+                ? change.filenames.map((filename) => ({
+                    type: change.type as string,
+                    fullName: change.name as string,
+                    filePath: filename,
+                  }))
+                : []
+            )
+            .flat()
         ),
       ]);
     }
@@ -113,9 +123,8 @@ export default class SourcePull extends SfdxCommand {
       tracking.updateLocalTracking({
         files: successes.map((fileResponse) => fileResponse.filePath as string).filter(Boolean),
       }),
-      // calling with no metadata types gets the latest sourceMembers from the org
       tracking.updateRemoteTracking(
-        successes.map((fileResponse) => ({ name: fileResponse.fullName, type: fileResponse.type }))
+        successes.map((success) => ({ filePath: success.filePath, type: success.type, fullName: success.fullName }))
       ),
     ]);
     return retrieveResult.getFileResponses();

src/commands/source/push.ts

@@ -8,9 +8,8 @@
 import { FlagsConfig, flags, SfdxCommand } from '@salesforce/command';
 import { SfdxProject, Org } from '@salesforce/core';
 import { ComponentSet, FileResponse, ComponentStatus } from '@salesforce/source-deploy-retrieve';
-import { MetadataKeyPair, SourceTracking, stringGuard } from '../../sourceTracking';
+import { SourceTracking, stringGuard } from '../../sourceTracking';
 import { writeConflictTable } from '../../writeConflictTable';
-
 export default class SourcePush extends SfdxCommand {
   public static description = 'get local changes';
   protected static readonly flagsConfig: FlagsConfig = {
@@ -66,7 +65,7 @@ export default class SourcePush extends SfdxCommand {
 
     if (deletes.length > 0) {
       this.ux.warn(
-        `Delete not yet implemented in SDR.  Would have deleted ${deletes.length > 0 ? deletes.join(',') : 'nothing'}`
+        `Delete not yet implemented.  Would have deleted ${deletes.length > 0 ? deletes.join(',') : 'nothing'}`
       );
     }
 
@@ -82,21 +81,9 @@ export default class SourcePush extends SfdxCommand {
     if (!this.flags.json) {
       this.ux.logJson(result.response);
     }
-    // and update the remote tracking
-    const successComponentKeys = (
-      Array.isArray(result.response.details.componentSuccesses)
-        ? result.response.details.componentSuccesses
-        : [result.response.details.componentSuccesses]
-    )
-      .map((success) =>
-        success?.fullName && success?.componentType
-          ? { name: success?.fullName, type: success?.componentType }
-          : undefined
-      )
-      .filter(Boolean) as MetadataKeyPair[]; // we don't want package.xml
 
     // this includes polling for sourceMembers
-    await tracking.updateRemoteTracking(successComponentKeys);
+    await tracking.updateRemoteTracking(successes);
     return result.getFileResponses();
   }
 }

src/shared/remoteSourceTrackingService.ts

@@ -13,6 +13,8 @@ import { ConfigFile, Logger, Org, SfdxError, Messages } from '@salesforce/core';
 import { Dictionary, Optional } from '@salesforce/ts-types';
 import { Duration, env, toNumber } from '@salesforce/kit';
 import { retryDecorator } from 'ts-retry-promise';
+import { RemoteSyncInput } from '../shared/types';
+import { getMetadataKeyFromFileResponse } from './metadataKeys';
 
 export type MemberRevision = {
   serverRevisionCounter: number;
@@ -53,6 +55,7 @@ export namespace RemoteSourceTrackingService {
 export const getMetadataKey = (metadataType: string, metadataName: string): string => {
   return `${metadataType}__${metadataName}`;
 };
+
 /**
  * This service handles source tracking of metadata between a local project and an org.
  * Source tracking state is persisted to .sfdx/orgs/<orgId>/maxRevision.json.
@@ -178,19 +181,21 @@ export class RemoteSourceTrackingService extends ConfigFile<RemoteSourceTracking
    * pass in a set of metadata keys (type__name like 'ApexClass__MyClass').\
    * it sets their last retrieved revision to the current revision counter from the server.
    */
-  public async syncNamedElementsByKey(metadataKeys: string[]): Promise<void> {
-    if (metadataKeys.length === 0) {
+  public async syncSpecifiedElements(elements: RemoteSyncInput[]): Promise<void> {
+    if (elements.length === 0) {
       return;
     }
-    const quiet = metadataKeys.length > 100;
+    const quiet = elements.length > 100;
     if (quiet) {
-      this.logger.debug(`Syncing ${metadataKeys.length} Revisions by key`);
+      this.logger.debug(`Syncing ${elements.length} Revisions by key`);
     }
 
-    // makes sure we have updated SourceMember data for the org; uses cache
-    await this.retrieveUpdates();
     const revisions = this.getSourceMembers();
-    metadataKeys.map((metadataKey) => {
+
+    // this can be super-repetitive on a large ExperienceBundle where there is an element for each file but only one Revision for the entire bundle
+    // any item in an aura/LWC bundle needs to represent the top (bundle) level and the file itself
+    // so we de-dupe via a set
+    [...new Set(elements.map((element) => getMetadataKeyFromFileResponse(element)).flat())].map((metadataKey) => {
       const revision = revisions[metadataKey];
       if (revision && revision.lastRetrievedFromServer !== revision.serverRevisionCounter) {
         if (!quiet) {
@@ -201,11 +206,13 @@ export class RemoteSourceTrackingService extends ConfigFile<RemoteSourceTracking
         revision.lastRetrievedFromServer = revision.serverRevisionCounter;
         this.setMemberRevision(metadataKey, revision);
       } else {
-        this.logger.warn(`could not find remote tracking entry for ${metadataKey}`);
+        this.logger.warn(`found no matching revision for ${metadataKey}`);
       }
     });
+
     await this.write();
   }
+
   /**
    * Returns the `ChangeElement` currently being tracked given a metadata key,
    * or `undefined` if not found.
@@ -381,9 +388,9 @@ export class RemoteSourceTrackingService extends ConfigFile<RemoteSourceTracking
   // to sync the retrieved SourceMembers; meaning it will update the lastRetrievedFromServer
   // field to the SourceMember's RevisionCounter, and update the serverMaxRevisionCounter
   // to the highest RevisionCounter.
-  public async retrieveUpdates(sync = false): Promise<RemoteChangeElement[]> {
+  public async retrieveUpdates({ sync = false, cache = true } = {}): Promise<RemoteChangeElement[]> {
     // Always track new SourceMember data, or update tracking when we sync.
-    const queriedSourceMembers = await this.querySourceMembersFrom();
+    const queriedSourceMembers = await this.querySourceMembersFrom({ useCache: cache });
     if (queriedSourceMembers.length || sync) {
       await this.trackSourceMembers(queriedSourceMembers, sync);
     }

src/shared/types.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright (c) 2020, salesforce.com, inc.
+ * All rights reserved.
+ * Licensed under the BSD 3-Clause license.
+ * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+ */
+
+import { FileResponse } from '@salesforce/source-deploy-retrieve';
+
+export type RemoteSyncInput = Pick<FileResponse, 'fullName' | 'filePath' | 'type'>;

src/sourceTracking.ts

@@ -10,6 +10,7 @@ import { ComponentSet, MetadataResolver, SourceComponent } from '@salesforce/sou
 
 import { RemoteSourceTrackingService, RemoteChangeElement, getMetadataKey } from './shared/remoteSourceTrackingService';
 import { ShadowRepo } from './shared/localShadowRepo';
+import { RemoteSyncInput } from './shared/types';
 
 export const getKeyFromObject = (element: RemoteChangeElement | ChangeResult): string => {
   if (element.type && element.name) {
@@ -21,11 +22,6 @@ export const getKeyFromObject = (element: RemoteChangeElement | ChangeResult): s
 // external users of SDR might need to convert a fileResponse to a key
 export const getKeyFromStrings = getMetadataKey;
 
-export interface MetadataKeyPair {
-  type: string;
-  name: string;
-}
-
 export interface ChangeOptions {
   origin?: 'local' | 'remote';
   state: 'add' | 'delete' | 'changed' | 'unchanged' | 'moved';
@@ -44,6 +40,11 @@ export type ChangeResult = Partial<RemoteChangeElement> & {
   filenames?: string[];
 };
 
+export interface ConflictError {
+  message: string;
+  name: 'conflict';
+  conflicts: ChangeResult[];
+}
 export class SourceTracking {
   private orgId: string;
   private projectPath: string;
@@ -63,6 +64,14 @@ export class SourceTracking {
     this.logger = Logger.childFromRoot('SourceTracking');
   }
 
+  public async deployLocalChanges({ overwrite = false, ignoreWarnings = false, wait = 33 }): Promise<void> {
+    // TODO: this is basically the logic for a push
+  }
+
+  public async retrieveRemoteChanges(): Promise<void> {
+    // TODO: this is basically the logic for a pull
+  }
+
   /**
    * Get metadata changes made locally and in the org.
    *
@@ -125,11 +134,12 @@ export class SourceTracking {
   /**
    * Mark remote source tracking files that we have received to the latest version
    */
-  public async updateRemoteTracking(metadataKeys: MetadataKeyPair[]): Promise<void> {
+  public async updateRemoteTracking(fileResponses: RemoteSyncInput[]): Promise<void> {
     await this.ensureRemoteTracking();
-    await this.remoteSourceTrackingService.syncNamedElementsByKey(
-      metadataKeys.map((input) => getKeyFromStrings(input.type, input.name))
-    );
+    // TODO: poll for source tracking to be complete
+    // to make sure we have the updates before syncing the ones from metadataKeys
+    await this.remoteSourceTrackingService.retrieveUpdates({ cache: false });
+    await this.remoteSourceTrackingService.syncSpecifiedElements(fileResponses);
   }
 
   /**

tsconfig.json

@@ -3,5 +3,5 @@
   "compilerOptions": {
     "outDir": "./lib"
   },
-  "include": ["src/**/*.ts"]
+  "include": ["src/**/*.ts", "test/unit/metadataKeys.test.ts"]
 }