Migration to V10 Pipeline (#47)

* Migration to V10 Pipeline

* bumpity bump

* changed container names

* removing subscription address

* helm chart bump

* provision file update

* npm audit fix

* storage test

* actually include file

* MiD on table storage

Jenkinsfile

@@ -1,3 +1,3 @@
-@Library('defra-library@v-9') _
+@Library('defra-library@v-10') _
 
 buildNodeJs()

app/config/message.js

@@ -16,7 +16,8 @@ const schema = Joi.object({
   },
   dataQueue: {
     address: Joi.string()
-  }
+  },
+  managedIdentityClientId: Joi.string().optional()
 })
 
 const config = {
@@ -34,7 +35,8 @@ const config = {
   },
   dataQueue: {
     address: process.env.DATARESPONSE_QUEUE_ADDRESS
-  }
+  },
+  managedIdentityClientId: process.env.AZURE_CLIENT_ID
 }
 
 const result = schema.validate(config, {

app/config/storage.js

@@ -9,7 +9,8 @@ const schema = Joi.object({
   warningTable: Joi.string().default('warnings'),
   batchTable: Joi.string().default('batches'),
   container: Joi.string().default('reports'),
-  createEntities: Joi.bool().default(false)
+  createEntities: Joi.bool().default(false),
+  managedIdentityClientId: Joi.string().optional()
 })
 
 const config = {
@@ -21,7 +22,8 @@ const config = {
   warningTable: process.env.AZURE_STORAGE_WARNING_TABLE,
   batchTable: process.env.AZURE_STORAGE_BATCH_TABLE,
   container: process.env.AZURE_STORAGE_CONTAINER,
-  createEntities: process.env.AZURE_STORAGE_CREATE_ENTITIES
+  createEntities: process.env.AZURE_STORAGE_CREATE_ENTITIES,
+  managedIdentityClientId: process.env.AZURE_CLIENT_ID
 }
 
 const result = schema.validate(config, {

app/data/events/order-grouped-events.js

@@ -1,4 +1,3 @@
-
 const { getEventOrder } = require('./get-event-order')
 
 const orderGroupedEvents = (events) => {

app/data/events/scheme-id/order-grouped-events-by-scheme.js

@@ -1,4 +1,3 @@
-
 const orderGroupedEventsByScheme = (events) => {
   return events.sort((a, b) => a.schemeId - b.schemeId)
 }

app/storage.js

@@ -1,4 +1,3 @@
-
 const { DefaultAzureCredential } = require('@azure/identity')
 const { TableClient, odata } = require('@azure/data-tables')
 const { BlobServiceClient } = require('@azure/storage-blob')
@@ -22,11 +21,11 @@ const initialise = async () => {
     blobServiceClient = BlobServiceClient.fromConnectionString(storageConfig.connectionString)
   } else {
     console.log('Using DefaultAzureCredential for Table Client')
-    paymentClient = new TableClient(`https://${storageConfig.account}.table.core.windows.net`, storageConfig.paymentTable, new DefaultAzureCredential())
-    holdClient = new TableClient(`https://${storageConfig.account}.table.core.windows.net`, storageConfig.holdTable, new DefaultAzureCredential())
-    warningClient = new TableClient(`https://${storageConfig.account}.table.core.windows.net`, storageConfig.warningTable, new DefaultAzureCredential())
-    batchClient = new TableClient(`https://${storageConfig.account}.table.core.windows.net`, storageConfig.batchTable, new DefaultAzureCredential())
-    blobServiceClient = new BlobServiceClient(`https://${storageConfig.storageAccount}.blob.core.windows.net`, new DefaultAzureCredential())
+    paymentClient = new TableClient(`https://${storageConfig.account}.table.core.windows.net`, storageConfig.paymentTable, new DefaultAzureCredential({ managedIdentityClientId: storageConfig.managedIdentityClientId }))
+    holdClient = new TableClient(`https://${storageConfig.account}.table.core.windows.net`, storageConfig.holdTable, new DefaultAzureCredential({ managedIdentityClientId: storageConfig.managedIdentityClientId }))
+    warningClient = new TableClient(`https://${storageConfig.account}.table.core.windows.net`, storageConfig.warningTable, new DefaultAzureCredential({ managedIdentityClientId: storageConfig.managedIdentityClientId }))
+    batchClient = new TableClient(`https://${storageConfig.account}.table.core.windows.net`, storageConfig.batchTable, new DefaultAzureCredential({ managedIdentityClientId: storageConfig.managedIdentityClientId }))
+    blobServiceClient = new BlobServiceClient(`https://${storageConfig.storageAccount}.blob.core.windows.net`, new DefaultAzureCredential({ managedIdentityClientId: storageConfig.managedIdentityClientId }))
   }
   container = blobServiceClient.getContainerClient(storageConfig.container)
   if (storageConfig.createEntities) {

appconfig/common.yaml

@@ -0,0 +1,2 @@
+container.dataResponseQueueAddress: queue:ffc-pay-data-request-response
+container.dataTopicAddress: queue:ffc-pay-data-request

helm/ffc-pay-data-hub/Chart.yaml

@@ -4,6 +4,6 @@ name: ffc-pay-data-hub
 # this version is automatically updated in the build pipeline
 version: 1.0.0
 dependencies:
-- name: ffc-helm-library
-  version: 4.0.0
-  repository: https://raw.githubusercontent.com/defra/ffc-helm-repository/master/
+  - name: ffc-helm-library
+    version: 4.7.4
+    repository: https://raw.githubusercontent.com/defra/ffc-helm-repository/master/

helm/ffc-pay-data-hub/templates/service-account.yaml

@@ -0,0 +1,3 @@
+{{- include "ffc-helm-library.service-account" (list . "ffc-pay-data-hub.service-account") -}}
+{{- define "ffc-pay-data-hub.service-account" -}}
+{{- end -}}

helm/ffc-pay-data-hub/values.yaml

@@ -41,7 +41,7 @@ container:
   azureStorageUseConnectionString: false
   azureStorageAccountName: ffcpay
 
-aadPodIdentity: true
+workloadIdentity: true
 
 azureIdentity:
   clientID: not-a-real-clientID

jest.config.js

@@ -13,7 +13,8 @@ module.exports = {
     '<rootDir>/node_modules/',
     '<rootDir>/test-output/',
     '<rootDir>/test/',
-    '<rootDir>/jest.config.js'
+    '<rootDir>/jest.config.js',
+    '<rootDir>/app/config/'
   ],
   modulePathIgnorePatterns: [
     'node_modules'

package.json

@@ -1,6 +1,6 @@
 {
   "name": "ffc-pay-data-hub",
-  "version": "1.4.17",
+  "version": "1.4.18",
   "description": "Data hub for FFC Payments",
   "homepage": "https://github.com/DEFRA/ffc-pay-data-hub",
   "main": "app/index.js",
@@ -18,16 +18,16 @@
     "John Watson john.watson1@defra.gov.uk",
     "Simon Dunn simon.dunn1@defra.gov.uk",
     "Sam Plackett samuel.plackett@eviden.com",
-    "Leigh Godson leigh.godson.external@eviden.com"
+    "Leigh Godson leigh.godson@eviden.com"
   ],
   "license": "OGL-UK-3.0",
   "dependencies": {
     "@azure/data-tables": "13.1.2",
-    "@azure/identity": "4.3.0",
+    "@azure/identity": "4.4.1",
     "@azure/storage-blob": "12.13.0",
     "@hapi/hoek": "11.0.2",
     "applicationinsights": "2.9.6",
-    "ffc-messaging": "2.9.1",
+    "ffc-messaging": "2.10.1",
     "joi": "17.7.1",
     "log-timestamp": "0.3.0",
     "moment": "2.29.4",
@@ -57,4 +57,4 @@
       "**/test-output/**"
     ]
   }
-}
+}

provision.azure.yaml

@@ -1,6 +1,11 @@
 resources:
+  identity: pay-data-hub
   topics:
-    - name: data
+    - name: ffc-pay-data-request
+      role: receiver
+      subscriptions:
+        -name: ffc-pay-data-hub
   queues:
-    - name: dataResponse
-      enableSessions: true
+    - name: ffc-pay-data-request-response
+      role: sender
+      session: true

sonar-project.properties

@@ -1,3 +1,3 @@
-sonar.javascript.exclusions=**/jest.config.js,**/__mocks__/**,**/node_modules/**,**/test/**,**/test-output/**
+sonar.javascript.exclusions=**/jest.config.js,**/__mocks__/**,**/node_modules/**,**/test/**,**/test-output/**,**/app/config/**
 sonar.javascript.lcov.reportPaths=test-output/lcov.info
 sonar.exclusions=/test/**,**/*.test.js,*snyk_report.html,*snyk_report.css

test/unit/storage.test.js

@@ -0,0 +1,141 @@
+const { PAYMENT_EVENT, HOLD_EVENT, WARNING_EVENT, BATCH_EVENT } = require('../../app/constants/event-types')
+
+describe('Storage initialization and functionality', () => {
+  let consoleLogSpy
+  let storageConfig
+  let BlobServiceClient
+  let DefaultAzureCredential
+  let TableClient
+  let storage
+
+  beforeAll(() => {
+    jest.doMock('@azure/storage-blob', () => {
+      const getBlockBlobClientMock = jest.fn().mockReturnValue({
+        upload: jest.fn().mockResolvedValue()
+      })
+      const getContainerClientMock = jest.fn().mockReturnValue({
+        createIfNotExists: jest.fn().mockResolvedValue(),
+        getBlockBlobClient: getBlockBlobClientMock
+      })
+      const fromConnectionStringMock = jest.fn().mockReturnValue({
+        getContainerClient: getContainerClientMock
+      })
+      const BlobServiceClientMock = jest.fn().mockImplementation(() => ({
+        getContainerClient: getContainerClientMock
+      }))
+      BlobServiceClientMock.fromConnectionString = fromConnectionStringMock
+      return { BlobServiceClient: BlobServiceClientMock }
+    })
+
+    jest.doMock('@azure/identity', () => ({
+      DefaultAzureCredential: jest.fn().mockImplementation((options) => ({
+        type: 'DefaultAzureCredential',
+        options
+      }))
+    }))
+
+    jest.doMock('@azure/data-tables', () => {
+      const createTableMock = jest.fn().mockResolvedValue()
+      const TableClientMock = jest.fn().mockImplementation(() => ({
+        createTable: createTableMock
+      }))
+      TableClientMock.fromConnectionString = jest.fn().mockReturnValue({
+        createTable: createTableMock
+      })
+      return { TableClient: TableClientMock, odata: {} }
+    })
+  })
+
+  beforeEach(() => {
+    jest.resetModules()
+    jest.clearAllMocks()
+
+    storageConfig = require('../../app/config/storage')
+    consoleLogSpy = jest.spyOn(console, 'log').mockImplementation(() => { })
+    ; ({ BlobServiceClient } = require('@azure/storage-blob'))
+    ; ({ DefaultAzureCredential } = require('@azure/identity'))
+    ; ({ TableClient } = require('@azure/data-tables'))
+    storage = require('../../app/storage')
+  })
+
+  afterEach(() => {
+    consoleLogSpy.mockRestore()
+    jest.clearAllMocks()
+  })
+
+  test('should use connection string when storageConfig.useConnectionString is true', async () => {
+    storageConfig.useConnectionString = true
+    storageConfig.connectionString = 'fake-connection-string'
+
+    await storage.initialise()
+
+    expect(consoleLogSpy).toHaveBeenCalledWith('Using connection string for Table Client')
+    expect(BlobServiceClient.fromConnectionString).toHaveBeenCalledWith(storageConfig.connectionString)
+    expect(TableClient.fromConnectionString).toHaveBeenCalledTimes(4)
+  })
+
+  test('should use DefaultAzureCredential when storageConfig.useConnectionString is false', async () => {
+    storageConfig.useConnectionString = false
+    storageConfig.account = 'fakeaccount'
+    storageConfig.storageAccount = 'fakestorageaccount'
+    storageConfig.managedIdentityClientId = 'fake-managed-id'
+
+    await storage.initialise()
+
+    const expectedBlobUri = `https://${storageConfig.storageAccount}.blob.core.windows.net`
+    const expectedTableUri = `https://${storageConfig.account}.table.core.windows.net`
+
+    expect(consoleLogSpy).toHaveBeenCalledWith('Using DefaultAzureCredential for Table Client')
+    expect(DefaultAzureCredential).toHaveBeenCalledWith({ managedIdentityClientId: storageConfig.managedIdentityClientId })
+    expect(BlobServiceClient).toHaveBeenCalledWith(expectedBlobUri, expect.any(Object))
+    expect(TableClient).toHaveBeenCalledTimes(4)
+    expect(TableClient).toHaveBeenCalledWith(expectedTableUri, expect.any(String), expect.any(Object))
+  })
+
+  test('should create tables and container when createEntities is true', async () => {
+    storageConfig.createEntities = true
+    storageConfig.useConnectionString = true
+    storageConfig.connectionString = 'fake-connection-string'
+
+    await storage.initialise()
+
+    expect(consoleLogSpy).toHaveBeenCalledWith('Making sure tables exist')
+    expect(consoleLogSpy).toHaveBeenCalledWith('Making sure blob containers exist')
+    expect(consoleLogSpy).toHaveBeenCalledWith('Storage ready')
+    expect(TableClient.fromConnectionString().createTable).toHaveBeenCalledTimes(4)
+    expect(BlobServiceClient.fromConnectionString().getContainerClient().createIfNotExists).toHaveBeenCalled()
+  })
+
+  test('should return correct client based on event type', async () => {
+    await storage.initialise()
+
+    const paymentClient = storage.getClient(PAYMENT_EVENT)
+    const holdClient = storage.getClient(HOLD_EVENT)
+    const warningClient = storage.getClient(WARNING_EVENT)
+    const batchClient = storage.getClient(BATCH_EVENT)
+
+    expect(paymentClient).toBeDefined()
+    expect(holdClient).toBeDefined()
+    expect(warningClient).toBeDefined()
+    expect(batchClient).toBeDefined()
+  })
+
+  test('should throw error for unknown event type', async () => {
+    await storage.initialise()
+
+    expect(() => storage.getClient('UNKNOWN_EVENT')).toThrow('Unknown event type: UNKNOWN_EVENT')
+  })
+
+  test('should write file to blob storage', async () => {
+    await storage.initialise()
+
+    const filename = 'test.txt'
+    const content = 'Hello, World!'
+
+    await storage.writeFile(filename, content)
+
+    const containerClient = BlobServiceClient.fromConnectionString().getContainerClient()
+    expect(containerClient.getBlockBlobClient).toHaveBeenCalledWith(filename)
+    expect(containerClient.getBlockBlobClient().upload).toHaveBeenCalledWith(content, content.length)
+  })
+})