chore: move build to new subscription (#888)

pipeline.yaml

@@ -37,7 +37,7 @@ jobs:
     - task: AzureCLI@1
       displayName: 'Style Check'
       inputs:
-        azureSubscription: 'MMLSpark Connection'
+        azureSubscription: 'MMLSpark Connection 2'
         scriptLocation: inlineScript
         inlineScript: 'sbt scalastyle test:scalastyle it:scalastyle'
 
@@ -54,7 +54,7 @@ jobs:
       displayName: Create Anaconda environment
     - task: AzureKeyVault@1
       inputs:
-        azureSubscription: 'MMLSpark Connection'
+        azureSubscription: 'MMLSpark Connection 2'
         keyVaultName: mmlspark-keys
     - bash: |
         source activate mmlspark
@@ -90,7 +90,7 @@ jobs:
       displayName: Create Anaconda environment
     - task: AzureKeyVault@1
       inputs:
-        azureSubscription: 'MMLSpark Connection'
+        azureSubscription: 'MMLSpark Connection 2'
         keyVaultName: mmlspark-keys
     - bash: |
         source activate mmlspark
@@ -107,7 +107,7 @@ jobs:
     - task: AzureCLI@1
       displayName: 'E2E'
       inputs:
-        azureSubscription:  'MMLSpark Connection'
+        azureSubscription:  'MMLSpark Connection 2'
         scriptLocation: inlineScript
         inlineScript: 'sbt it:test'
       condition: and(succeeded(), eq(variables.runTests, 'True'))
@@ -132,7 +132,7 @@ jobs:
     - task: Docker@2
       displayName: Demo Image Build
       inputs:
-        containerRegistry: 'mmlsparkmcr-connection-new'
+        containerRegistry: 'mmlsparkmcr-connection-2'
         repository: 'public/mmlspark/build-demo'
         command: 'build'
         buildContext: "."
@@ -142,14 +142,14 @@ jobs:
     - task: Docker@2
       displayName: Demo Image Push
       inputs:
-        containerRegistry: 'mmlsparkmcr-connection-new'
+        containerRegistry: 'mmlsparkmcr-connection-2'
         repository: 'public/mmlspark/build-demo'
         command: 'push'
         tags: $(version)
     - task: Docker@2
       displayName: Minimal Image Build + Push
       inputs:
-        containerRegistry: 'mmlsparkmcr-connection-new'
+        containerRegistry: 'mmlsparkmcr-connection-2'
         repository: 'public/mmlspark/build-minimal'
         command: 'build'
         buildContext: "."
@@ -159,15 +159,15 @@ jobs:
     - task: Docker@2
       displayName: Minimal Image Build + Push
       inputs:
-        containerRegistry: 'mmlsparkmcr-connection-new'
+        containerRegistry: 'mmlsparkmcr-connection-2'
         repository: 'public/mmlspark/build-minimal'
         command: 'push'
         tags: $(version)
     - task: Docker@2
       condition: startsWith(variables['gittag'], 'v')
       displayName: Release Image Build
       inputs:
-        containerRegistry: 'mmlsparkmcr-connection-new'
+        containerRegistry: 'mmlsparkmcr-connection-2'
         repository: 'public/mmlspark/release'
         command: 'build'
         buildContext: "."
@@ -180,7 +180,7 @@ jobs:
       condition: startsWith(variables['gittag'], 'v')
       displayName: Release Image Push
       inputs:
-        containerRegistry: 'mmlsparkmcr-connection-new'
+        containerRegistry: 'mmlsparkmcr-connection-2'
         repository: 'public/mmlspark/release'
         command: 'push'
         tags: |
@@ -236,13 +236,13 @@ jobs:
     - task: AzureCLI@1
       displayName: 'Generate Codecov report'
       inputs:
-        azureSubscription: 'MMLSpark Connection'
+        azureSubscription: 'MMLSpark Connection 2'
         scriptLocation: inlineScript
         inlineScript: 'sbt coverageReport'
       condition: succeededOrFailed()
     - task: AzureKeyVault@1
       inputs:
-        azureSubscription: 'MMLSpark Connection'
+        azureSubscription: 'MMLSpark Connection 2'
         keyVaultName: mmlspark-keys
       condition: succeededOrFailed()
     - bash: |
@@ -317,7 +317,7 @@ jobs:
     - task: AzureCLI@1
       displayName: 'Setup repo'
       inputs:
-        azureSubscription: 'MMLSpark Connection'
+        azureSubscription: 'MMLSpark Connection 2'
         scriptLocation: inlineScript
         inlineScript: |
           (timeout 30s pip install requests) || (echo "retrying" && timeout 30s pip install requests)
@@ -326,7 +326,7 @@ jobs:
       displayName: 'Unit Test'
       timeoutInMinutes: 60
       inputs:
-        azureSubscription: 'MMLSpark Connection'
+        azureSubscription: 'MMLSpark Connection 2'
         scriptLocation: inlineScript
         inlineScript: |
           (${FFMPEG:-false} && sudo add-apt-repository ppa:jonathonf/ffmpeg-4 -y && \
@@ -346,13 +346,13 @@ jobs:
     - task: AzureCLI@1
       displayName: 'Generate Codecov report'
       inputs:
-        azureSubscription: 'MMLSpark Connection'
+        azureSubscription: 'MMLSpark Connection 2'
         scriptLocation: inlineScript
         inlineScript: 'sbt coverageReport'
       condition: succeededOrFailed()
     - task: AzureKeyVault@1
       inputs:
-        azureSubscription: 'MMLSpark Connection'
+        azureSubscription: 'MMLSpark Connection 2'
         keyVaultName: mmlspark-keys
       condition: succeededOrFailed()
     - bash: |

src/it/scala/com/microsoft/ml/nbtest/DatabricksUtilities.scala

@@ -26,7 +26,7 @@ import scala.concurrent.{ExecutionContext, Future, blocking}
 object DatabricksUtilities extends HasHttpClient {
 
   // ADB Info
-  val Region = "eastus2"
+  val Region = "eastus"
   val PoolName = "mmlspark-build"
   val AdbRuntime = "5.5.x-scala2.11"
   val NumWorkers = 5

src/main/scala/com/microsoft/ml/spark/cognitive/SpeechToTextSDK.scala

@@ -232,6 +232,7 @@ class SpeechToTextSDK(override val uid: String) extends Transformer
       val queue = new LinkedBlockingQueue[Option[String]]()
 
       def recognizedHandler(s: Any, e: SpeechRecognitionEventArgs): Unit = {
+        println(e)
         if (e.getResult.getReason eq ResultReason.RecognizedSpeech) {
           queue.put(Some(e.getResult.getProperties.getProperty(PropertyId.SpeechServiceResponse_JsonResult)))
         }

src/test/scala/com/microsoft/ml/spark/Secrets.scala

@@ -3,18 +3,13 @@
 
 package com.microsoft.ml.spark
 
-import spray.json.DefaultJsonProtocol._
-import spray.json._
-
-import scala.sys.process._
-
 import sys.process._
 import spray.json._
 import DefaultJsonProtocol._
 
 object Secrets {
   private val KvName = "mmlspark-keys"
-  private val SubscriptionID = "ce1dee05-8cf6-4ad6-990a-9c80868800ba"
+  private val SubscriptionID = "f9b96b36-1f5e-4021-8959-51527e26e6d3"
 
   protected def exec(command: String): String = {
     val os = sys.props("os.name").toLowerCase
@@ -31,15 +26,12 @@ object Secrets {
     secretJson.parseJson.asJsObject().fields("value").convertTo[String]
   }
 
-  lazy val TextApiKey: String = getSecret("text-api-key")
+  lazy val CognitiveApiKey: String = getSecret("cognitive-api-key")
+  lazy val CustomSpeechApiKey: String = getSecret("custom-speech-api-key")
   lazy val AnomalyApiKey: String = getSecret("anomaly-api-key")
   lazy val AzureSearchKey: String = getSecret("azure-search-key")
   lazy val BingImageSearchKey: String = getSecret("bing-image-search-key")
-  lazy val FaceApiKey: String = getSecret("face-api-key")
   lazy val PowerbiURL: String = getSecret("powerbi-url")
-  lazy val SpeechApiKey: String = getSecret("speech-api-key")
-  lazy val CustomSpeechApiKey: String = getSecret("custom-speech-api-key")
-  lazy val VisionApiKey: String = getSecret("vision-api-key")
   lazy val AdbToken: String = getSecret("adb-token")
 
 }

src/test/scala/com/microsoft/ml/spark/cognitive/split1/ComputerVisionSuite.scala

@@ -14,11 +14,11 @@ import org.apache.spark.sql.{DataFrame, Dataset, Row}
 import org.scalactic.Equality
 import org.scalatest.Assertion
 
-trait VisionKey {
-  lazy val visionKey = sys.env.getOrElse("VISION_API_KEY", Secrets.VisionApiKey)
+trait CognitiveKey {
+  lazy val cognitiveKey = sys.env.getOrElse("COGNITIVE_API_KEY", Secrets.CognitiveApiKey)
 }
 
-class OCRSuite extends TransformerFuzzing[OCR] with VisionKey with Flaky {
+class OCRSuite extends TransformerFuzzing[OCR] with CognitiveKey with Flaky {
 
   import session.implicits._
 
@@ -29,7 +29,7 @@ class OCRSuite extends TransformerFuzzing[OCR] with VisionKey with Flaky {
   ).toDF("url")
 
   lazy val ocr = new OCR()
-    .setSubscriptionKey(visionKey)
+    .setSubscriptionKey(cognitiveKey)
     .setLocation("eastus")
     .setDefaultLanguage("en")
     .setImageUrlCol("url")
@@ -42,7 +42,7 @@ class OCRSuite extends TransformerFuzzing[OCR] with VisionKey with Flaky {
     .select("imageBytes")
 
   lazy val bytesOCR = new OCR()
-    .setSubscriptionKey(visionKey)
+    .setSubscriptionKey(cognitiveKey)
     .setLocation("eastus")
     .setDefaultLanguage("en")
     .setImageBytesCol("imageBytes")
@@ -52,7 +52,7 @@ class OCRSuite extends TransformerFuzzing[OCR] with VisionKey with Flaky {
   test("Getters") {
     assert(ocr.getDetectOrientation)
     assert(ocr.getImageUrlCol === "url")
-    assert(ocr.getSubscriptionKey == visionKey)
+    assert(ocr.getSubscriptionKey == cognitiveKey)
     assert(bytesOCR.getImageBytesCol === "imageBytes")
   }
 
@@ -81,7 +81,7 @@ class OCRSuite extends TransformerFuzzing[OCR] with VisionKey with Flaky {
   override def reader: MLReadable[_] = OCR
 }
 
-class AnalyzeImageSuite extends TransformerFuzzing[AnalyzeImage] with VisionKey with Flaky {
+class AnalyzeImageSuite extends TransformerFuzzing[AnalyzeImage] with CognitiveKey with Flaky {
 
   import session.implicits._
 
@@ -92,7 +92,7 @@ class AnalyzeImageSuite extends TransformerFuzzing[AnalyzeImage] with VisionKey
   ).toDF("url", "language")
 
   def baseAI: AnalyzeImage = new AnalyzeImage()
-    .setSubscriptionKey(visionKey)
+    .setSubscriptionKey(cognitiveKey)
     .setLocation("eastus")
     .setOutputCol("features")
 
@@ -190,7 +190,7 @@ class AnalyzeImageSuite extends TransformerFuzzing[AnalyzeImage] with VisionKey
 
 }
 
-class RecognizeTextSuite extends TransformerFuzzing[RecognizeText] with VisionKey with Flaky {
+class RecognizeTextSuite extends TransformerFuzzing[RecognizeText] with CognitiveKey with Flaky {
 
   import com.microsoft.ml.spark.FluentAPI._
   import session.implicits._
@@ -202,7 +202,7 @@ class RecognizeTextSuite extends TransformerFuzzing[RecognizeText] with VisionKe
   ).toDF("url")
 
   lazy val rt: RecognizeText = new RecognizeText()
-    .setSubscriptionKey(visionKey)
+    .setSubscriptionKey(cognitiveKey)
     .setLocation("eastus")
     .setImageUrlCol("url")
     .setMode("Printed")
@@ -215,7 +215,7 @@ class RecognizeTextSuite extends TransformerFuzzing[RecognizeText] with VisionKe
     .select("imageBytes")
 
   lazy val bytesRT: RecognizeText = new RecognizeText()
-    .setSubscriptionKey(visionKey)
+    .setSubscriptionKey(cognitiveKey)
     .setLocation("eastus")
     .setImageBytesCol("imageBytes")
     .setMode("Printed")
@@ -247,7 +247,7 @@ class RecognizeTextSuite extends TransformerFuzzing[RecognizeText] with VisionKe
 }
 
 class RecognizeDomainSpecificContentSuite extends TransformerFuzzing[RecognizeDomainSpecificContent]
-  with VisionKey with Flaky {
+  with CognitiveKey with Flaky {
 
   import session.implicits._
 
@@ -256,7 +256,7 @@ class RecognizeDomainSpecificContentSuite extends TransformerFuzzing[RecognizeDo
   ).toDF("url")
 
   lazy val celeb: RecognizeDomainSpecificContent = new RecognizeDomainSpecificContent()
-    .setSubscriptionKey(visionKey)
+    .setSubscriptionKey(cognitiveKey)
     .setModel("celebrities")
     .setLocation("eastus")
     .setImageUrlCol("url")
@@ -268,7 +268,7 @@ class RecognizeDomainSpecificContentSuite extends TransformerFuzzing[RecognizeDo
     .select("imageBytes")
 
   lazy val bytesCeleb: RecognizeDomainSpecificContent = new RecognizeDomainSpecificContent()
-    .setSubscriptionKey(visionKey)
+    .setSubscriptionKey(cognitiveKey)
     .setModel("celebrities")
     .setLocation("eastus")
     .setImageBytesCol("imageBytes")
@@ -303,7 +303,7 @@ class RecognizeDomainSpecificContentSuite extends TransformerFuzzing[RecognizeDo
 }
 
 class GenerateThumbnailsSuite extends TransformerFuzzing[GenerateThumbnails]
-  with VisionKey with Flaky {
+  with CognitiveKey with Flaky {
 
   import session.implicits._
 
@@ -312,7 +312,7 @@ class GenerateThumbnailsSuite extends TransformerFuzzing[GenerateThumbnails]
   ).toDF("url")
 
   lazy val t: GenerateThumbnails = new GenerateThumbnails()
-    .setSubscriptionKey(visionKey)
+    .setSubscriptionKey(cognitiveKey)
     .setLocation("eastus")
     .setHeight(50).setWidth(50).setSmartCropping(true)
     .setImageUrlCol("url")
@@ -324,7 +324,7 @@ class GenerateThumbnailsSuite extends TransformerFuzzing[GenerateThumbnails]
     .select("imageBytes")
 
   lazy val bytesGT: GenerateThumbnails = new GenerateThumbnails()
-    .setSubscriptionKey(visionKey)
+    .setSubscriptionKey(cognitiveKey)
     .setLocation("eastus")
     .setHeight(50).setWidth(50).setSmartCropping(true)
     .setImageBytesCol("imageBytes")
@@ -346,7 +346,7 @@ class GenerateThumbnailsSuite extends TransformerFuzzing[GenerateThumbnails]
   override def reader: MLReadable[_] = GenerateThumbnails
 }
 
-class TagImageSuite extends TransformerFuzzing[TagImage] with VisionKey with Flaky {
+class TagImageSuite extends TransformerFuzzing[TagImage] with CognitiveKey with Flaky {
 
   import session.implicits._
 
@@ -355,7 +355,7 @@ class TagImageSuite extends TransformerFuzzing[TagImage] with VisionKey with Fla
   ).toDF("url")
 
   lazy val t: TagImage = new TagImage()
-    .setSubscriptionKey(visionKey)
+    .setSubscriptionKey(cognitiveKey)
     .setLocation("eastus")
     .setImageUrlCol("url")
     .setOutputCol("tags")
@@ -366,7 +366,7 @@ class TagImageSuite extends TransformerFuzzing[TagImage] with VisionKey with Fla
     .select("imageBytes")
 
   lazy val bytesTI: TagImage = new TagImage()
-    .setSubscriptionKey(visionKey)
+    .setSubscriptionKey(cognitiveKey)
     .setLocation("eastus")
     .setImageBytesCol("imageBytes")
     .setOutputCol("tags")
@@ -402,7 +402,7 @@ class TagImageSuite extends TransformerFuzzing[TagImage] with VisionKey with Fla
 }
 
 class DescribeImageSuite extends TransformerFuzzing[DescribeImage]
-  with VisionKey with Flaky {
+  with CognitiveKey with Flaky {
 
   import session.implicits._
 
@@ -411,7 +411,7 @@ class DescribeImageSuite extends TransformerFuzzing[DescribeImage]
   ).toDF("url")
 
   lazy val t: DescribeImage = new DescribeImage()
-    .setSubscriptionKey(visionKey)
+    .setSubscriptionKey(cognitiveKey)
     .setLocation("eastus")
     .setMaxCandidates(3)
     .setImageUrlCol("url")
@@ -423,7 +423,7 @@ class DescribeImageSuite extends TransformerFuzzing[DescribeImage]
     .select("imageBytes")
 
   lazy val bytesDI: DescribeImage = new DescribeImage()
-    .setSubscriptionKey(visionKey)
+    .setSubscriptionKey(cognitiveKey)
     .setLocation("eastus")
     .setMaxCandidates(3)
     .setImageBytesCol("imageBytes")