diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/Secrets.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/Secrets.scala index 3c36e2233c..ee854f10bd 100644 --- a/core/src/test/scala/com/microsoft/azure/synapse/ml/Secrets.scala +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/Secrets.scala @@ -63,7 +63,7 @@ object Secrets { lazy val PowerbiURL: String = getSecret("powerbi-url") lazy val AdbToken: String = getSecret("adb-token") lazy val SynapseStorageKey: String = getSecret("synapse-storage-key") - lazy val SynapseSpnKey: String = getSecret("synapse-spn-key") + lazy val MADTestStorageKey: String = getSecret("madtest-storage-key") lazy val ArtifactStore: String = getSecret("synapse-artifact-store") diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/SynapseTests.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/SynapseTests.scala index 5e4a5cfd5d..c89eb5a770 100644 --- a/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/SynapseTests.scala +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/SynapseTests.scala @@ -71,7 +71,7 @@ class SynapseTests extends TestBase { println(s"Creating $expectedPoolCount Spark Pools...") // val sparkPools: Seq[String] = createSparkPools(expectedPoolCount) - val sparkPools: Seq[String] = Seq.fill(expectedPoolCount)("synapseml34pool") + val sparkPools: Seq[String] = Seq.fill(expectedPoolCount)("sml34pool3") val livyBatches: Array[LivyBatch] = selectedPythonFiles.zip(sparkPools).map { case (file, poolName) => diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/SynapseUtilities.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/SynapseUtilities.scala index c86b615206..6128f1cced 100644 --- a/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/SynapseUtilities.scala +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/SynapseUtilities.scala @@ -118,9 +118,9 @@ object SynapseUtilities { import SynapseJsonProtocol._ - lazy val SynapseToken: String = getAccessToken(ClientId, Secrets.SynapseSpnKey, + lazy val SynapseToken: String = getAccessToken(ClientId, Secrets.ServiceConnectionSecret, "https://dev.azuresynapse.net/") - lazy val ArmToken: String = getAccessToken(ClientId, Secrets.SynapseSpnKey, + lazy val ArmToken: String = getAccessToken(ClientId, Secrets.ServiceConnectionSecret, "https://management.azure.com/") val LineSeparator: String = sys.props("line.separator").toLowerCase // Platform agnostic (\r\n:windows, \n:linux) @@ -129,7 +129,7 @@ object SynapseUtilities { val StorageAccount: String = "mmlsparkbuildsynapse" val StorageContainer: String = "synapse" val TenantId: String = "72f988bf-86f1-41af-91ab-2d7cd011db47" - val ClientId: String = "85dde348-dd2b-43e5-9f5a-22262af45332" + val ClientId: String = Secrets.ServicePrincipalClientId val PoolNodeSize: String = "Small" val PoolLocation: String = "eastus2" val WorkspaceName: String = "mmlsparkbuild" diff --git a/tools/docker/clean_acr.py b/tools/docker/clean_acr.py deleted file mode 100644 index 2ef4a2866a..0000000000 --- a/tools/docker/clean_acr.py +++ /dev/null @@ -1,76 +0,0 @@ -import os -import json -from azure.storage.blob import BlobClient -import sys -import subprocess -from tqdm import tqdm - -acr = "mmlsparkmcr" -container = "acrbackup" -rg = "marhamil-mmlspark" -pipeline = "mmlsparkacrexport3" - -conn_string = sys.argv[1] - -repos = json.loads(os.popen("az acr repository list -n {}".format(acr)).read()) -for repo in repos: - tags = json.loads( - os.popen( - "az acr repository show-tags -n {} --repository {} --orderby time_desc".format( - acr, - repo, - ), - ).read(), - ) - - for tag in tqdm(tags): - target_blob = repo + "/" + tag + ".tar" - image = repo + ":" + tag - - backup_exists = BlobClient.from_connection_string( - conn_string, - container_name=container, - blob_name=target_blob, - ).exists() - if not backup_exists: - subprocess.run( - [ - "sudo", - "az", - "acr", - "pipeline-run", - "create", - "--resource-group", - rg, - "--registry", - acr, - "--pipeline", - pipeline, - "--name", - str(abs(hash(target_blob))), - "--pipeline-type", - "export", - "--storage-blob", - target_blob, - "-a", - image, - ], - ) - print("Transferred {}".format(target_blob)) - else: - print("Skipped existing {}".format(image)) - - backup_exists = BlobClient.from_connection_string( - conn_string, - container_name=container, - blob_name=target_blob, - ).exists() - if backup_exists: - print("Deleting {}".format(image)) - result = os.system( - "az acr repository delete --name {} --image {} --yes".format( - acr, - image, - ), - ) - assert result == 0