Add the actual result to compare, fix evidence and a few fixes. (inluxc#12)

Co-authored-by: Fúlvio Carvalhido <fulvio@diller.no>

Author: inluxc

README.md

@@ -80,13 +80,33 @@ test('[J79] basic test', async ({ page }) => {
 Then run your tests with `npx playwright test` command and you'll see the result in console:
 
 ```sh
-✅ Test cycle JARV-C2901 has been created
+-------------------------------------
+ 
+✅ Test status: PROJ-104
+✅ Tests ran: 4
+✅ Tests passed: 1
+✅ Tests failed: 3
+✅ Tests todo: 0
+ 
+-------------------------------------
+ 
+✅ Test cycle PROJ-104 has been created
 👇 Check out the test result
-🔗 https://jira.your-company-domain.com/browser/JARV-C2901
+🔗 https://diller.atlassian.net/browse/PROJ-104
+ 
+-------------------------------------
 ```
 
 And you'll see the result in the Xray:
 
+![alt text](./assets/xray-result.png)
+
+
+## Notes
+
+- To have the steps imported you have to create then in the test issue itself.
+The steps will be imported by order of execution and inserted into the test.
+
 ## License
 
 playwright-xray is [MIT licensed](./LICENSE).

assets/xray-result.png

@@ -1,13 +1,16 @@
-import type { XrayTestResult, XrayTest, XrayTestSteps } from '../types/cloud.types';
+import type { XrayTestResult, XrayTestSteps, XrayTestEvidence, XrayTest } from '../types/cloud.types';
 import type { XrayOptions } from '../types/xray.types';
 import type { Reporter, TestCase, TestResult } from '@playwright/test/reporter';
+import * as fs from 'fs';
+import * as path from 'path';
 
 import { XrayService } from './xray.service';
 
 class XrayReporter implements Reporter {
   private xrayService!: XrayService;
   private testResults!: XrayTestResult;
   private testCaseKeyPattern = /\[(.*?)\]/;
+  private receivedRegEx: RegExp = /Received string: "(.*?)"(?=\n)/;  
   private options: XrayOptions;
   private totalDuration: number;
   private readonly defaultRunName = `[${new Date().toUTCString()}] - Automated run`;
@@ -16,8 +19,6 @@ class XrayReporter implements Reporter {
     this.options = options;
     this.xrayService = new XrayService(this.options);
     this.totalDuration = 0;
-
-    //const finishTime = new Date(this.xrayService.startTime.getTime() + (result.duration * 1000));
     const testResults: XrayTestResult = {
       info: {
         summary: this.defaultRunName,
@@ -31,15 +32,13 @@ class XrayReporter implements Reporter {
     this.testResults = testResults;
   }
 
-  async onBegin() {}
-
   async onTestEnd(testCase: TestCase, result: TestResult) {
     const testCaseId = testCase.title.match(this.testCaseKeyPattern);
     const testCode: string = testCaseId != null ? testCaseId[1]! : '';
     if (testCode != '') {
       // @ts-ignore
       const browserName = testCase._pool.registrations.get('browserName').fn;
-      const finishTime = new Date(result.startTime.getTime() + result.duration * 1000);
+      const finishTime = new Date(result.startTime.getTime() + result.duration);
       this.totalDuration = this.totalDuration + result.duration;
 
       let xrayTestData: XrayTest = {
@@ -49,28 +48,51 @@ class XrayReporter implements Reporter {
         finish: finishTime.toISOString(),
         steps: [] as XrayTestSteps[],
       };
-
-      // Generated step and error messages
+      
       await Promise.all(
         result.steps.map(async (step) => {
           if (step.category != 'hook') {
+            // Add Step to request
+            const errorMessage = step.error?.stack?.toString()?.replace(/[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]/g, '');
+            const received = this.receivedRegEx.exec(errorMessage!);
+            let dataReceived = ""
+            if(received?.[1] !== undefined) {
+              dataReceived = received?.[1];
+            }
+
             const xrayTestStep: XrayTestSteps = {
-              status: typeof step.error == 'object' ? 'FAILED' : 'SUCCESS',
-              comment: step.title,
-              actualResult: typeof step.error == 'object' ? step.error.message?.toString()! : '',
+              status: typeof step.error == 'object' ? 'FAILED' : 'PASSED',
+              comment: typeof step.error == 'object' ? errorMessage : '',
+              actualResult: dataReceived,
             };
             xrayTestData.steps!.push(xrayTestStep);
           }
         }),
       );
+
+      // Get evidences from test results (video, images, text)
+      const evidences: XrayTestEvidence[] = [];
+      if (result.attachments.length > 0) {
+        result.attachments.map(async (attach) => {
+          const filename = path.basename(attach.path!);
+          const attachData = fs.readFileSync(attach.path!, { encoding: 'base64' });
+          const evid: XrayTestEvidence = {
+            data: attachData,
+            filename: filename,
+            contentType: attach.contentType
+          }
+          evidences.push(evid);
+        })
+      }
+
+      xrayTestData.evidence = evidences;
       this.testResults.tests!.push(xrayTestData);
     }
   }
 
   async onEnd() {
     // Update test Duration
-    this.testResults.info.finishDate = new Date(new Date(this.testResults.info.startDate).getTime() + this.totalDuration).toISOString();
-
+    this.testResults?.info?.finishDate != new Date(new Date(this.testResults?.info?.startDate!).getTime() + this.totalDuration).toISOString();
     if (typeof this.testResults != 'undefined' && typeof this.testResults.tests != 'undefined' && this.testResults.tests.length > 0) {
       await this.xrayService.createRun(this.testResults);
     } else {

assets/zephyr-result.png

@@ -1,8 +1,8 @@
 import { XrayOptions } from '../types/xray.types';
-import { XrayTestResult } from '../types/cloud.types';
+import { Execution } from '../types/cloud.types';
 import axios, { Axios, AxiosError } from 'axios';
 import { inspect } from 'util';
-import { bold, green } from 'picocolors';
+import { blue, bold, green, red, yellow } from 'picocolors';
 
 function isAxiosError(error: any): error is AxiosError {
   return error.isAxiosError === true;
@@ -105,20 +105,48 @@ export class XrayService {
     if (!options.testPlan) throw new Error('"testPlan" option are missed. Please provide them in the config');
   }
 
-  async createRun(results: XrayTestResult) {
+  async createRun(results: Execution) {
     const URL = `${this.requestUrl}/import/execution`;
+    const total = results.tests.length;
+    let passed = 0;
+    let failed = 0;
+    let todo = 0;
+
+
+    results.tests!.forEach((test: { status: any; }) => {
+      switch (test.status) {
+        case 'PASSED':
+          passed = passed + 1;
+          break;
+        case 'FAILED':
+          failed = failed + 1;
+          break;
+      }
+    });
 
     try {
       const response = await this.axios.post(URL, JSON.stringify(results));
       if (response.status !== 200) throw new Error(`${response.status} - Failed to create test cycle`);
-
       const {
-        data: { key, id },
+        data: { key },
       } = response;
 
-      console.log(`${bold(green(`✅ Test cycle ${key} has been created`))}`);
-      console.log(`${bold(green('👇 Check out the test result'))}`);
-      console.log(`${bold(green(`🔗 ${this.jira}/browse/${id}`))}`);
+      // Results
+      console.log(`${bold(blue(`-------------------------------------`))}`);
+      console.log(`${bold(blue(` `))}`);
+      console.log(`${bold(blue(`✅ Test status: ${key}`))}`);
+      console.log(`${bold(blue(`✅ Tests ran: ${total}`))}`);
+      console.log(`${bold(green(`✅ Tests passed: ${passed}`))}`);
+      console.log(`${bold(red(`✅ Tests failed: ${failed}`))}`);
+      console.log(`${bold(blue(` `))}`);
+      console.log(`${bold(blue(`-------------------------------------`))}`);
+      console.log(`${bold(blue(` `))}`);
+      console.log(`${bold(blue(`✅ Test cycle ${key} has been created`))}`);
+      console.log(`${bold(blue('👇 Check out the test result'))}`);
+      console.log(`${bold(blue(`🔗 ${this.jira}/browse/${key}`))}`);
+      console.log(`${bold(blue(` `))}`);
+      console.log(`${bold(blue(`-------------------------------------`))}`);
+
     } catch (error) {
       if (isAxiosError(error)) {
         console.error(`Config: ${inspect(error.config)}`);

src/index.ts

@@ -40,5 +40,4 @@ export interface XrayTestEvidence {
   data: string,
   filename: string,
   contentType: string
-};
-
+};