Frontend - Add support for artifacts stored in S3 (#1278)

* Add s3 and local support for artifact viewer

* Remove local file support and use ViewerCRD instead

* Fix condition failed tests

* Keep using minio for metadata manifests

frontend/server/server.ts

@@ -39,6 +39,12 @@ const minioClient = new MinioClient({
   useSSL: false,
 } as any);
 
+const s3Client = new MinioClient({
+  endPoint: 's3.amazonaws.com',
+  accessKey: process.env.AWS_ACCESS_KEY_ID,
+  secretKey: process.env.AWS_SECRET_ACCESS_KEY,
+} as any);
+
 const app = express() as Application;
 
 app.use(function (req, _, next) {
@@ -171,6 +177,28 @@ const artifactsHandler = async (req, res) => {
         }
       });
       break;
+    case 's3':
+      s3Client.getObject(bucket, key, (err, stream) => {
+        if (err) {
+          res.status(500).send(`Failed to get object in bucket ${bucket} at path ${key}: ${err}`);
+          return;
+        }
+
+        try {
+          let contents = '';
+          stream.on('data', (chunk) => {
+            contents = chunk.toString();
+          });
+
+          stream.on('end', () => {
+            res.send(contents);
+          });
+        } catch (err) {
+          res.status(500).send(`Failed to get object in bucket ${bucket} at path ${key}: ${err}`);
+        }
+      });
+      break;
+
     default:
       res.status(500).send('Unknown storage source: ' + source);
       return;

frontend/src/lib/WorkflowParser.test.ts

@@ -686,6 +686,30 @@ describe('WorkflowParser', () => {
         source: StorageService.MINIO,
       });
     });
+
+    it('handles S3 bucket without key', () => {
+      expect(WorkflowParser.parseStoragePath('s3://testbucket/')).toEqual({
+        bucket: 'testbucket',
+        key: '',
+        source: StorageService.S3,
+      });
+    });
+
+    it('handles S3 bucket and key', () => {
+      expect(WorkflowParser.parseStoragePath('s3://testbucket/testkey')).toEqual({
+        bucket: 'testbucket',
+        key: 'testkey',
+        source: StorageService.S3,
+      });
+    });
+
+    it('handles S3 bucket and multi-part key', () => {
+      expect(WorkflowParser.parseStoragePath('s3://testbucket/test/key/path')).toEqual({
+        bucket: 'testbucket',
+        key: 'test/key/path',
+        source: StorageService.S3,
+      });
+    });
   });
 
   describe('getOutboundNodes', () => {

frontend/src/lib/WorkflowParser.ts

@@ -26,6 +26,7 @@ import { NodePhase, statusToBgColor, hasFinished } from './StatusUtils';
 export enum StorageService {
   GCS = 'gcs',
   MINIO = 'minio',
+  S3 = 's3'
 }
 
 export interface StoragePath {
@@ -254,6 +255,13 @@ export default class WorkflowParser {
         key: pathParts.slice(1).join('/'),
         source: StorageService.MINIO,
       };
+    } else if (strPath.startsWith('s3://')) {
+      const pathParts = strPath.substr('s3://'.length).split('/');
+      return {
+        bucket: pathParts[0],
+        key: pathParts.slice(1).join('/'),
+        source: StorageService.S3,
+      };
     } else {
       throw new Error('Unsupported storage path: ' + strPath);
     }