From 756476a922005dc0aef7b8233559a0778dcf07eb Mon Sep 17 00:00:00 2001 From: kwasi Date: Tue, 28 Jan 2020 14:59:39 -0800 Subject: [PATCH] [MLMD] Backport MLMD Utils and APIs added to Pipelines since Jan 8 (#23) * Backport changes to ArtifactCustomProperties and ExecutionCustomProperties * Backport Util and API changes needed to merge LineageView into KFP --- build/lib/index.js | 1 + build/lib/index.js.map | 2 +- build/lib/mlmd/Api.js | 5 ++++ build/lib/mlmd/Api.js.map | 2 +- build/lib/mlmd/LineageApi.js | 41 ++++++++++++++++++++++++++ build/lib/mlmd/LineageApi.js.map | 2 +- build/lib/mlmd/Utils.js | 9 ++++-- build/lib/mlmd/Utils.js.map | 2 +- src/index.ts | 2 +- src/mlmd/Api.ts | 5 ++++ src/mlmd/LineageApi.ts | 49 ++++++++++++++++++++++++++++++-- src/mlmd/Utils.tsx | 15 ++++++---- 12 files changed, 121 insertions(+), 14 deletions(-) diff --git a/build/lib/index.js b/build/lib/index.js index 63f59ae..36081a7 100644 --- a/build/lib/index.js +++ b/build/lib/index.js @@ -34,6 +34,7 @@ exports.ExecutionProperties = Api_1.ExecutionProperties; exports.ExecutionCustomProperties = Api_1.ExecutionCustomProperties; var LineageApi_1 = require("./mlmd/LineageApi"); exports.getArtifactTypes = LineageApi_1.getArtifactTypes; +exports.getArtifactCreationTime = LineageApi_1.getArtifactCreationTime; var LineageView_1 = require("./mlmd/LineageView"); exports.LineageView = LineageView_1.LineageView; var Utils_1 = require("./mlmd/Utils"); diff --git a/build/lib/index.js.map b/build/lib/index.js.map index 8924db8..035f409 100644 --- a/build/lib/index.js.map +++ b/build/lib/index.js.map @@ -1 +1 @@ -{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":";;AASA,0FAO8D;AAN5D,uCAAA,QAAQ,CAAA;AACR,2CAAA,YAAY,CAAA;AACZ,oCAAA,KAAK,CAAA;AACL,wCAAA,SAAS,CAAA;AACT,4CAAA,aAAa,CAAA;AACb,oCAAA,KAAK,CAAA;AAEP,4HAE+E;AAD7E,iFAAA,iCAAiC,CAAA;AAEnC,0GAiBqE;AAhBnE,8DAAA,uBAAuB,CAAA;AACvB,+DAAA,wBAAwB,CAAA;AACxB,8DAAA,uBAAuB,CAAA;AACvB,+DAAA,wBAAwB,CAAA;AACxB,0DAAA,mBAAmB,CAAA;AACnB,2DAAA,oBAAoB,CAAA;AACpB,oEAAA,6BAA6B,CAAA;AAC7B,qEAAA,8BAA8B,CAAA;AAC9B,qEAAA,8BAA8B,CAAA;AAC9B,sEAAA,+BAA+B,CAAA;AAC/B,+DAAA,wBAAwB,CAAA;AACxB,gEAAA,yBAAyB,CAAA;AACzB,+DAAA,wBAAwB,CAAA;AACxB,gEAAA,yBAAyB,CAAA;AACzB,2DAAA,oBAAoB,CAAA;AACpB,4DAAA,qBAAqB,CAAA;AAEvB,kCAMmB;AALjB,oBAAA,GAAG,CAAA;AACH,yCAAA,wBAAwB,CAAA;AACxB,mCAAA,kBAAkB,CAAA;AAClB,oCAAA,mBAAmB,CAAA;AACnB,0CAAA,yBAAyB,CAAA;AAE3B,gDAAkD;AAA1C,wCAAA,gBAAgB,CAAA;AACxB,kDAA8C;AAAtC,oCAAA,WAAW,CAAA;AACnB,sCAA+E;AAAvE,mCAAA,gBAAgB,CAAA;AAAE,sCAAA,mBAAmB,CAAA;AAAE,8BAAA,WAAW,CAAA;AAC1D,wCAAkE;AAA1D,yBAAA,MAAM,CAAA;AAAE,mCAAA,gBAAgB,CAAA;AAAE,4BAAA,SAAS,CAAA"} \ No newline at end of file +{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":";;AASA,0FAO8D;AAN5D,uCAAA,QAAQ,CAAA;AACR,2CAAA,YAAY,CAAA;AACZ,oCAAA,KAAK,CAAA;AACL,wCAAA,SAAS,CAAA;AACT,4CAAA,aAAa,CAAA;AACb,oCAAA,KAAK,CAAA;AAEP,4HAE+E;AAD7E,iFAAA,iCAAiC,CAAA;AAEnC,0GAiBqE;AAhBnE,8DAAA,uBAAuB,CAAA;AACvB,+DAAA,wBAAwB,CAAA;AACxB,8DAAA,uBAAuB,CAAA;AACvB,+DAAA,wBAAwB,CAAA;AACxB,0DAAA,mBAAmB,CAAA;AACnB,2DAAA,oBAAoB,CAAA;AACpB,oEAAA,6BAA6B,CAAA;AAC7B,qEAAA,8BAA8B,CAAA;AAC9B,qEAAA,8BAA8B,CAAA;AAC9B,sEAAA,+BAA+B,CAAA;AAC/B,+DAAA,wBAAwB,CAAA;AACxB,gEAAA,yBAAyB,CAAA;AACzB,+DAAA,wBAAwB,CAAA;AACxB,gEAAA,yBAAyB,CAAA;AACzB,2DAAA,oBAAoB,CAAA;AACpB,4DAAA,qBAAqB,CAAA;AAEvB,kCAMmB;AALjB,oBAAA,GAAG,CAAA;AACH,yCAAA,wBAAwB,CAAA;AACxB,mCAAA,kBAAkB,CAAA;AAClB,oCAAA,mBAAmB,CAAA;AACnB,0CAAA,yBAAyB,CAAA;AAE3B,gDAA2E;AAAnE,wCAAA,gBAAgB,CAAA;AAAE,+CAAA,uBAAuB,CAAA;AACjD,kDAA8C;AAAtC,oCAAA,WAAW,CAAA;AACnB,sCAA+E;AAAvE,mCAAA,gBAAgB,CAAA;AAAE,sCAAA,mBAAmB,CAAA;AAAE,8BAAA,WAAW,CAAA;AAC1D,wCAAkE;AAA1D,yBAAA,MAAM,CAAA;AAAE,mCAAA,gBAAgB,CAAA;AAAE,4BAAA,SAAS,CAAA"} \ No newline at end of file diff --git a/build/lib/mlmd/Api.js b/build/lib/mlmd/Api.js index 533a041..9525aca 100644 --- a/build/lib/mlmd/Api.js +++ b/build/lib/mlmd/Api.js @@ -16,6 +16,9 @@ var ArtifactCustomProperties; (function (ArtifactCustomProperties) { ArtifactCustomProperties["WORKSPACE"] = "__kf_workspace__"; ArtifactCustomProperties["RUN"] = "__kf_run__"; + ArtifactCustomProperties["NAME"] = "name"; + ArtifactCustomProperties["PIPELINE_NAME"] = "pipeline_name"; + ArtifactCustomProperties["RUN_ID"] = "run_id"; })(ArtifactCustomProperties = exports.ArtifactCustomProperties || (exports.ArtifactCustomProperties = {})); /** Known Execution properties */ var ExecutionProperties; @@ -29,6 +32,8 @@ var ExecutionProperties; var ExecutionCustomProperties; (function (ExecutionCustomProperties) { ExecutionCustomProperties["WORKSPACE"] = "__kf_workspace__"; + ExecutionCustomProperties["RUN_ID"] = "run_id"; + ExecutionCustomProperties["TASK_ID"] = "task_id"; })(ExecutionCustomProperties = exports.ExecutionCustomProperties || (exports.ExecutionCustomProperties = {})); /** * Class to wrap backend APIs. diff --git a/build/lib/mlmd/Api.js.map b/build/lib/mlmd/Api.js.map index e67514a..9a8d0c2 100644 --- a/build/lib/mlmd/Api.js.map +++ b/build/lib/mlmd/Api.js.map @@ -1 +1 @@ -{"version":3,"file":"Api.js","sourceRoot":"","sources":["../../../src/mlmd/Api.ts"],"names":[],"mappings":";;AAAA,wBAAqD;AAErD,gCAAgC;AAChC,IAAY,kBAOX;AAPD,WAAY,kBAAkB;IAC5B,+CAAyB,CAAA;IACzB,iDAA2B,CAAA;IAC3B,iDAA2B,CAAA;IAC3B,mCAAa,CAAA;IACb,qDAA+B,CAAA;IAC/B,yCAAmB,CAAA;AACrB,CAAC,EAPW,kBAAkB,GAAlB,0BAAkB,KAAlB,0BAAkB,QAO7B;AAED,uCAAuC;AACvC,IAAY,wBAGX;AAHD,WAAY,wBAAwB;IAClC,0DAA8B,CAAA;IAC9B,8CAAkB,CAAA;AACpB,CAAC,EAHW,wBAAwB,GAAxB,gCAAwB,KAAxB,gCAAwB,QAGnC;AAED,iCAAiC;AACjC,IAAY,mBAKX;AALD,WAAY,mBAAmB;IAC7B,oCAAa,CAAA;IACb,oDAA6B,CAAA;IAC7B,sDAA+B,CAAA;IAC/B,sCAAe,CAAA;AACjB,CAAC,EALW,mBAAmB,GAAnB,2BAAmB,KAAnB,2BAAmB,QAK9B;AAED,wCAAwC;AACxC,IAAY,yBAEX;AAFD,WAAY,yBAAyB;IACnC,2DAA8B,CAAA;AAChC,CAAC,EAFW,yBAAyB,GAAzB,iCAAyB,KAAzB,iCAAyB,QAEpC;AAWD;;GAEG;AACH;IAAA;QAGU,iCAA4B,GAAG,IAAI,qCAAiC,CAAC,EAAE,CAAC,CAAC;IAenF,CAAC;IAbC;;OAEG;IACW,eAAW,GAAzB;QACE,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE;YACjB,GAAG,CAAC,QAAQ,GAAG,IAAI,GAAG,EAAE,CAAC;SAC1B;QACD,OAAO,GAAG,CAAC,QAAQ,CAAC;IACtB,CAAC;IAED,sBAAI,qCAAoB;aAAxB;YACE,OAAO,IAAI,CAAC,4BAA4B,CAAC;QAC3C,CAAC;;;OAAA;IACH,UAAC;AAAD,CAAC,AAlBD,IAkBC;AAlBY,kBAAG"} \ No newline at end of file +{"version":3,"file":"Api.js","sourceRoot":"","sources":["../../../src/mlmd/Api.ts"],"names":[],"mappings":";;AAAA,wBAAqD;AAErD,gCAAgC;AAChC,IAAY,kBAOX;AAPD,WAAY,kBAAkB;IAC5B,+CAAyB,CAAA;IACzB,iDAA2B,CAAA;IAC3B,iDAA2B,CAAA;IAC3B,mCAAa,CAAA;IACb,qDAA+B,CAAA;IAC/B,yCAAmB,CAAA;AACrB,CAAC,EAPW,kBAAkB,GAAlB,0BAAkB,KAAlB,0BAAkB,QAO7B;AAED,uCAAuC;AACvC,IAAY,wBAMX;AAND,WAAY,wBAAwB;IAClC,0DAA8B,CAAA;IAC9B,8CAAkB,CAAA;IAClB,yCAAa,CAAA;IACb,2DAA+B,CAAA;IAC/B,6CAAiB,CAAA;AACnB,CAAC,EANW,wBAAwB,GAAxB,gCAAwB,KAAxB,gCAAwB,QAMnC;AAED,iCAAiC;AACjC,IAAY,mBAKX;AALD,WAAY,mBAAmB;IAC7B,oCAAa,CAAA;IACb,oDAA6B,CAAA;IAC7B,sDAA+B,CAAA;IAC/B,sCAAe,CAAA;AACjB,CAAC,EALW,mBAAmB,GAAnB,2BAAmB,KAAnB,2BAAmB,QAK9B;AAED,wCAAwC;AACxC,IAAY,yBAIX;AAJD,WAAY,yBAAyB;IACnC,2DAA8B,CAAA;IAC9B,8CAAiB,CAAA;IACjB,gDAAmB,CAAA;AACrB,CAAC,EAJW,yBAAyB,GAAzB,iCAAyB,KAAzB,iCAAyB,QAIpC;AAWD;;GAEG;AACH;IAAA;QAGU,iCAA4B,GAAG,IAAI,qCAAiC,CAAC,EAAE,CAAC,CAAC;IAenF,CAAC;IAbC;;OAEG;IACW,eAAW,GAAzB;QACE,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE;YACjB,GAAG,CAAC,QAAQ,GAAG,IAAI,GAAG,EAAE,CAAC;SAC1B;QACD,OAAO,GAAG,CAAC,QAAQ,CAAC;IACtB,CAAC;IAED,sBAAI,qCAAoB;aAAxB;YACE,OAAO,IAAI,CAAC,4BAA4B,CAAC;QAC3C,CAAC;;;OAAA;IACH,UAAC;AAAD,CAAC,AAlBD,IAkBC;AAlBY,kBAAG"} \ No newline at end of file diff --git a/build/lib/mlmd/LineageApi.js b/build/lib/mlmd/LineageApi.js index 418371e..fb93646 100644 --- a/build/lib/mlmd/LineageApi.js +++ b/build/lib/mlmd/LineageApi.js @@ -38,6 +38,7 @@ var __generator = (this && this.__generator) || function (thisArg, body) { Object.defineProperty(exports, "__esModule", { value: true }); var __1 = require(".."); var __2 = require(".."); +var EventType = __1.Event.Type; function getArtifactTypes(metadataStoreService, errorCallback) { return __awaiter(this, void 0, void 0, function () { var response, artifactTypesMap; @@ -86,4 +87,44 @@ function getExecutionTypes(metadataStoreService, errorCallback) { }); } exports.getExecutionTypes = getExecutionTypes; +function getArtifactCreationTime(artifactId, metadataStoreService, errorCallback) { + return __awaiter(this, void 0, void 0, function () { + var eventsRequest, response, data, lastOutputEvent; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: + if (!artifactId) { + throw new Error('artifactId is empty'); + } + eventsRequest = new __1.GetEventsByArtifactIDsRequest(); + eventsRequest.setArtifactIdsList([artifactId]); + return [4 /*yield*/, metadataStoreService.getEventsByArtifactIDs(eventsRequest)]; + case 1: + response = _a.sent(); + if (!response) { + if (errorCallback) { + errorCallback("Unable to retrieve Events for artifactId: " + artifactId); + } + return [2 /*return*/, '']; + } + data = response.getEventsList().map(function (event) { return ({ + time: event.getMillisecondsSinceEpoch(), + type: event.getType() || EventType.UNKNOWN, + }); }); + lastOutputEvent = data + .reverse() + .find(function (event) { return event.type === EventType.DECLARED_OUTPUT || event.type === EventType.OUTPUT; }); + if (lastOutputEvent && lastOutputEvent.time) { + return [2 /*return*/, __1.formatDateString(new Date(lastOutputEvent.time))]; + } + else { + // No valid time found, just return empty + return [2 /*return*/, '']; + } + return [2 /*return*/]; + } + }); + }); +} +exports.getArtifactCreationTime = getArtifactCreationTime; //# sourceMappingURL=LineageApi.js.map \ No newline at end of file diff --git a/build/lib/mlmd/LineageApi.js.map b/build/lib/mlmd/LineageApi.js.map index 5d0ad63..cd70c2e 100644 --- a/build/lib/mlmd/LineageApi.js.map +++ b/build/lib/mlmd/LineageApi.js.map @@ -1 +1 @@ -{"version":3,"file":"LineageApi.js","sourceRoot":"","sources":["../../../src/mlmd/LineageApi.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,wBAAyE;AACzE,wBAA8E;AAK9E,SAAsB,gBAAgB,CACpC,oBAAuD,EACvD,aAAyC;;;;;wBAGvC,qBAAM,oBAAoB,CAAC,gBAAgB,CAAC,IAAI,2BAAuB,EAAE,CAAC,EAAA;;oBADtE,QAAQ,GACZ,SAA0E;oBAE5E,IAAI,CAAC,QAAQ,EAAE;wBACb,IAAI,aAAa,EAAE;4BACjB,aAAa,CAAC,gEAAgE,CAAC,CAAC;yBACjF;wBACD,sBAAO,IAAI,GAAG,EAAE,EAAC;qBAClB;oBAEK,gBAAgB,GAAG,IAAI,GAAG,EAAwB,CAAC;oBAEzD,CAAC,QAAS,CAAC,oBAAoB,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,UAAC,YAA0B;wBAC1E,gBAAgB,CAAC,GAAG,CAAC,YAAY,CAAC,KAAK,EAAG,EAAE,YAAY,CAAC,CAAC;oBAC5D,CAAC,CAAC,CAAC;oBAEH,sBAAO,gBAAgB,EAAC;;;;CACzB;AArBD,4CAqBC;AAED,SAAsB,iBAAiB,CACrC,oBAAuD,EACvD,aAAyC;;;;;wBAGvC,qBAAM,oBAAoB,CAAC,iBAAiB,CAAC,IAAI,4BAAwB,EAAE,CAAC,EAAA;;oBADxE,QAAQ,GACZ,SAA4E;oBAE9E,IAAI,CAAC,QAAQ,EAAE;wBACb,IAAI,aAAa,EAAE;4BACjB,aAAa,CAAC,iEAAiE,CAAC,CAAC;yBAClF;wBACD,sBAAO,IAAI,GAAG,EAAE,EAAC;qBAClB;oBAEK,iBAAiB,GAAG,IAAI,GAAG,EAAyB,CAAC;oBAE3D,CAAC,QAAS,CAAC,qBAAqB,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,UAAC,aAA4B;wBAC7E,iBAAiB,CAAC,GAAG,CAAC,aAAa,CAAC,KAAK,EAAG,EAAE,aAAa,CAAC,CAAC;oBAC/D,CAAC,CAAC,CAAC;oBAEH,sBAAO,iBAAiB,EAAC;;;;CAC1B;AArBD,8CAqBC"} \ No newline at end of file +{"version":3,"file":"LineageApi.js","sourceRoot":"","sources":["../../../src/mlmd/LineageApi.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,wBAMY;AACZ,wBAA8E;AAE9E,IAAM,SAAS,GAAG,SAAK,CAAC,IAAI,CAAC;AAK7B,SAAsB,gBAAgB,CACpC,oBAAuD,EACvD,aAAyC;;;;;wBAGvC,qBAAM,oBAAoB,CAAC,gBAAgB,CAAC,IAAI,2BAAuB,EAAE,CAAC,EAAA;;oBADtE,QAAQ,GACZ,SAA0E;oBAE5E,IAAI,CAAC,QAAQ,EAAE;wBACb,IAAI,aAAa,EAAE;4BACjB,aAAa,CAAC,gEAAgE,CAAC,CAAC;yBACjF;wBACD,sBAAO,IAAI,GAAG,EAAE,EAAC;qBAClB;oBAEK,gBAAgB,GAAG,IAAI,GAAG,EAAwB,CAAC;oBAEzD,CAAC,QAAS,CAAC,oBAAoB,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,UAAC,YAA0B;wBAC1E,gBAAgB,CAAC,GAAG,CAAC,YAAY,CAAC,KAAK,EAAG,EAAE,YAAY,CAAC,CAAC;oBAC5D,CAAC,CAAC,CAAC;oBAEH,sBAAO,gBAAgB,EAAC;;;;CACzB;AArBD,4CAqBC;AAED,SAAsB,iBAAiB,CACrC,oBAAuD,EACvD,aAAyC;;;;;wBAGvC,qBAAM,oBAAoB,CAAC,iBAAiB,CAAC,IAAI,4BAAwB,EAAE,CAAC,EAAA;;oBADxE,QAAQ,GACZ,SAA4E;oBAE9E,IAAI,CAAC,QAAQ,EAAE;wBACb,IAAI,aAAa,EAAE;4BACjB,aAAa,CAAC,iEAAiE,CAAC,CAAC;yBAClF;wBACD,sBAAO,IAAI,GAAG,EAAE,EAAC;qBAClB;oBAEK,iBAAiB,GAAG,IAAI,GAAG,EAAyB,CAAC;oBAE3D,CAAC,QAAS,CAAC,qBAAqB,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,UAAC,aAA4B;wBAC7E,iBAAiB,CAAC,GAAG,CAAC,aAAa,CAAC,KAAK,EAAG,EAAE,aAAa,CAAC,CAAC;oBAC/D,CAAC,CAAC,CAAC;oBAEH,sBAAO,iBAAiB,EAAC;;;;CAC1B;AArBD,8CAqBC;AAED,SAAsB,uBAAuB,CAC3C,UAAkB,EAClB,oBAAuD,EACvD,aAAyC;;;;;;oBAEzC,IAAI,CAAC,UAAU,EAAE;wBACf,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;qBACxC;oBAEK,aAAa,GAAG,IAAI,iCAA6B,EAAE,CAAC;oBAC1D,aAAa,CAAC,kBAAkB,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;oBAE7C,qBAAM,oBAAoB,CAAC,sBAAsB,CAAC,aAAa,CAAC,EAAA;;oBAD5D,QAAQ,GACZ,SAAgE;oBAElE,IAAI,CAAC,QAAQ,EAAE;wBACb,IAAI,aAAa,EAAE;4BACjB,aAAa,CAAC,+CAA6C,UAAY,CAAC,CAAC;yBAC1E;wBACD,sBAAO,EAAE,EAAC;qBACX;oBAEK,IAAI,GAAG,QAAQ,CAAC,aAAa,EAAE,CAAC,GAAG,CAAC,UAAA,KAAK,IAAI,OAAA,CAAC;wBAClD,IAAI,EAAE,KAAK,CAAC,yBAAyB,EAAE;wBACvC,IAAI,EAAE,KAAK,CAAC,OAAO,EAAE,IAAI,SAAS,CAAC,OAAO;qBAC3C,CAAC,EAHiD,CAGjD,CAAC,CAAC;oBAEE,eAAe,GAAG,IAAI;yBACzB,OAAO,EAAE;yBACT,IAAI,CAAC,UAAA,KAAK,IAAI,OAAA,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,eAAe,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,MAAM,EAA3E,CAA2E,CAAC,CAAC;oBAC9F,IAAI,eAAe,IAAI,eAAe,CAAC,IAAI,EAAE;wBAC3C,sBAAO,oBAAgB,CAAC,IAAI,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,EAAC;qBACzD;yBAAM;wBACL,yCAAyC;wBACzC,sBAAO,EAAE,EAAC;qBACX;;;;;CACF;AAnCD,0DAmCC"} \ No newline at end of file diff --git a/build/lib/mlmd/Utils.js b/build/lib/mlmd/Utils.js index 4c9a79d..b29e44a 100644 --- a/build/lib/mlmd/Utils.js +++ b/build/lib/mlmd/Utils.js @@ -17,6 +17,7 @@ Object.defineProperty(exports, "__esModule", { value: true }); var Api_1 = require("./Api"); var __1 = require(".."); +var UNNAMED_RESOURCE_DISPLAY_NAME = '(unnamed)'; function getResourceProperty(resource, propertyName, fromCustomProperties) { if (fromCustomProperties === void 0) { fromCustomProperties = false; } var props = fromCustomProperties @@ -27,10 +28,14 @@ function getResourceProperty(resource, propertyName, fromCustomProperties) { } exports.getResourceProperty = getResourceProperty; function getArtifactName(artifact) { - return String(getResourceProperty(artifact, Api_1.ArtifactProperties.NAME)); + var artifactName = getResourceProperty(artifact, Api_1.ArtifactProperties.NAME) || + getResourceProperty(artifact, Api_1.ArtifactCustomProperties.NAME, true); + return artifactName ? artifactName.toString() : UNNAMED_RESOURCE_DISPLAY_NAME; } function getExecutionName(execution) { - return String(getResourceProperty(execution, Api_1.ExecutionProperties.NAME)); + var executionName = getResourceProperty(execution, Api_1.ExecutionProperties.COMPONENT_ID) || + getResourceProperty(execution, Api_1.ExecutionCustomProperties.TASK_ID, true); + return executionName ? executionName.toString() : UNNAMED_RESOURCE_DISPLAY_NAME; } function getResourceName(resource) { if (resource instanceof __1.Artifact) { diff --git a/build/lib/mlmd/Utils.js.map b/build/lib/mlmd/Utils.js.map index ef8a844..bc5bf33 100644 --- a/build/lib/mlmd/Utils.js.map +++ b/build/lib/mlmd/Utils.js.map @@ -1 +1 @@ -{"version":3,"file":"Utils.js","sourceRoot":"","sources":["../../../src/mlmd/Utils.tsx"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;AAEH,6BAKe;AAGf,wBAA8C;AAE9C,SAAgB,mBAAmB,CAAC,QAA8B,EAC9D,YAAoB,EAAE,oBAA4B;IAA5B,qCAAA,EAAA,4BAA4B;IACpD,IAAM,KAAK,GAAG,oBAAoB;QAC9B,CAAC,CAAC,QAAQ,CAAC,sBAAsB,EAAE;QACnC,CAAC,CAAC,QAAQ,CAAC,gBAAgB,EAAE,CAAC;IAElC,OAAO,CAAC,KAAK,IAAI,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,gBAAgB,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC;WAC/E,IAAI,CAAC;AACd,CAAC;AARD,kDAQC;AAED,SAAS,eAAe,CAAC,QAAkB;IACzC,OAAO,MAAM,CAAC,mBAAmB,CAAC,QAAQ,EAAE,wBAAkB,CAAC,IAAI,CAAC,CAAC,CAAA;AACvE,CAAC;AAED,SAAS,gBAAgB,CAAC,SAAoB;IAC5C,OAAO,MAAM,CAAC,mBAAmB,CAAC,SAAS,EAAE,yBAAmB,CAAC,IAAI,CAAC,CAAC,CAAA;AACzE,CAAC;AAED,SAAgB,eAAe,CAAC,QAAyB;IACvD,IAAI,QAAQ,YAAY,YAAQ,EAAE;QAChC,OAAO,eAAe,CAAC,QAAQ,CAAC,CAAC;KAClC;IACD,OAAO,gBAAgB,CAAC,QAAQ,CAAC,CAAC;AACpC,CAAC;AALD,0CAKC;AAED,SAAgB,sBAAsB,CAAC,QAAyB;IAC9D,IAAI,WAAW,CAAC;IAChB,IAAI,QAAQ,YAAY,YAAQ,EAAE;QAChC,WAAW,GAAG,mBAAmB,CAAC,QAAQ,EAAE,wBAAkB,CAAC,aAAa,CAAC;eACxE,mBAAmB,CAAC,QAAQ,EAAE,8BAAwB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;KAC9E;SAAM;QACL,WAAW,GAAG,mBAAmB,CAAC,QAAQ,EAAE,yBAAmB,CAAC,aAAa,CAAC;eACzE,mBAAmB,CAAC,QAAQ,EAAE,+BAAyB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;KAC/E;IACD,OAAO,MAAM,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;AACnC,CAAC;AAVD,wDAUC;AAED,SAAgB,WAAW,CAAC,MAAc,EAAE,aAA8B;IACxE,OAAO,aAAa,IAAI,aAAa,CAAC,GAAG,CAAC,MAAO,CAAC,CAAC,CAAC;QAClD,aAAa,CAAC,GAAG,CAAC,MAAO,CAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;AAC3D,CAAC;AAHD,kCAGC;AAED,SAAgB,gBAAgB,CAAC,KAAa;IAC5C,IAAI,CAAC,KAAK,EAAE;QACV,OAAO,EAAE,CAAC;KACX;IAED,QAAQ,KAAK,CAAC,YAAY,EAAE,EAAE;QAC5B,KAAK,SAAK,CAAC,SAAS,CAAC,YAAY;YAC/B,OAAO,KAAK,CAAC,cAAc,EAAE,CAAC;QAChC,KAAK,SAAK,CAAC,SAAS,CAAC,SAAS;YAC5B,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC;QAC7B,KAAK,SAAK,CAAC,SAAS,CAAC,YAAY;YAC/B,OAAO,KAAK,CAAC,cAAc,EAAE,CAAC;QAChC,KAAK,SAAK,CAAC,SAAS,CAAC,aAAa;YAChC,OAAO,EAAE,CAAC;KACb;AACH,CAAC;AAfD,4CAeC"} \ No newline at end of file +{"version":3,"file":"Utils.js","sourceRoot":"","sources":["../../../src/mlmd/Utils.tsx"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;AAEH,6BAKe;AAEf,wBAA8C;AAE9C,IAAM,6BAA6B,GAAG,WAAW,CAAC;AAElD,SAAgB,mBAAmB,CAAC,QAA8B,EAC9D,YAAoB,EAAE,oBAA4B;IAA5B,qCAAA,EAAA,4BAA4B;IACpD,IAAM,KAAK,GAAG,oBAAoB;QAC9B,CAAC,CAAC,QAAQ,CAAC,sBAAsB,EAAE;QACnC,CAAC,CAAC,QAAQ,CAAC,gBAAgB,EAAE,CAAC;IAElC,OAAO,CAAC,KAAK,IAAI,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,gBAAgB,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC;WAC/E,IAAI,CAAC;AACd,CAAC;AARD,kDAQC;AAED,SAAS,eAAe,CAAC,QAAkB;IACzC,IAAM,YAAY,GAAG,mBAAmB,CAAC,QAAQ,EAAE,wBAAkB,CAAC,IAAI,CAAC;QACzE,mBAAmB,CAAC,QAAQ,EAAE,8BAAwB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACrE,OAAO,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,6BAA6B,CAAC;AAChF,CAAC;AAED,SAAS,gBAAgB,CAAC,SAAoB;IAC5C,IAAM,aAAa,GAAG,mBAAmB,CAAC,SAAS,EAAE,yBAAmB,CAAC,YAAY,CAAC;QACpF,mBAAmB,CAAC,SAAS,EAAE,+BAAyB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAC1E,OAAO,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,6BAA6B,CAAC;AAClF,CAAC;AAED,SAAgB,eAAe,CAAC,QAA8B;IAC5D,IAAI,QAAQ,YAAY,YAAQ,EAAE;QAChC,OAAO,eAAe,CAAC,QAAQ,CAAC,CAAC;KAClC;IACD,OAAO,gBAAgB,CAAC,QAAQ,CAAC,CAAC;AACpC,CAAC;AALD,0CAKC;AAED,SAAgB,sBAAsB,CAAC,QAA8B;IACnE,IAAI,WAAW,CAAC;IAChB,IAAI,QAAQ,YAAY,YAAQ,EAAE;QAChC,WAAW,GAAG,mBAAmB,CAAC,QAAQ,EAAE,wBAAkB,CAAC,aAAa,CAAC;eACxE,mBAAmB,CAAC,QAAQ,EAAE,8BAAwB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;KAC9E;SAAM;QACL,WAAW,GAAG,mBAAmB,CAAC,QAAQ,EAAE,yBAAmB,CAAC,aAAa,CAAC;eACzE,mBAAmB,CAAC,QAAQ,EAAE,+BAAyB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;KAC/E;IACD,OAAO,MAAM,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;AACnC,CAAC;AAVD,wDAUC;AAED,SAAgB,WAAW,CAAC,MAAc,EAAE,aAA8B;IACxE,OAAO,aAAa,IAAI,aAAa,CAAC,GAAG,CAAC,MAAO,CAAC,CAAC,CAAC;QAClD,aAAa,CAAC,GAAG,CAAC,MAAO,CAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;AAC3D,CAAC;AAHD,kCAGC;AAED,SAAgB,gBAAgB,CAAC,KAAa;IAC5C,IAAI,CAAC,KAAK,EAAE;QACV,OAAO,EAAE,CAAC;KACX;IAED,QAAQ,KAAK,CAAC,YAAY,EAAE,EAAE;QAC5B,KAAK,SAAK,CAAC,SAAS,CAAC,YAAY;YAC/B,OAAO,KAAK,CAAC,cAAc,EAAE,CAAC;QAChC,KAAK,SAAK,CAAC,SAAS,CAAC,SAAS;YAC5B,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC;QAC7B,KAAK,SAAK,CAAC,SAAS,CAAC,YAAY;YAC/B,OAAO,KAAK,CAAC,cAAc,EAAE,CAAC;QAChC,KAAK,SAAK,CAAC,SAAS,CAAC,aAAa;YAChC,OAAO,EAAE,CAAC;KACb;AACH,CAAC;AAfD,4CAeC"} \ No newline at end of file diff --git a/src/index.ts b/src/index.ts index 5396235..0c8f756 100644 --- a/src/index.ts +++ b/src/index.ts @@ -43,7 +43,7 @@ export { ExecutionProperties, ExecutionCustomProperties, } from './mlmd/Api' -export {getArtifactTypes} from './mlmd/LineageApi' +export {getArtifactTypes, getArtifactCreationTime} from './mlmd/LineageApi' export {LineageView} from './mlmd/LineageView' export {getMetadataValue, getResourceProperty, getTypeName} from './mlmd/Utils' export {logger, formatDateString, titleCase} from './common/Utils' diff --git a/src/mlmd/Api.ts b/src/mlmd/Api.ts index 1e42a4d..a8f8ac7 100644 --- a/src/mlmd/Api.ts +++ b/src/mlmd/Api.ts @@ -14,6 +14,9 @@ export enum ArtifactProperties { export enum ArtifactCustomProperties { WORKSPACE = '__kf_workspace__', RUN = '__kf_run__', + NAME = 'name', + PIPELINE_NAME = 'pipeline_name', // TODO: Remove when switching to contexts + RUN_ID = 'run_id', // TODO: Remove when switching to contexts } /** Known Execution properties */ @@ -27,6 +30,8 @@ export enum ExecutionProperties { /** Known Execution custom properties */ export enum ExecutionCustomProperties { WORKSPACE = '__kf_workspace__', + RUN_ID = 'run_id', // TODO: Remove when switching to contexts + TASK_ID = 'task_id', } /** Format for a List request */ diff --git a/src/mlmd/LineageApi.ts b/src/mlmd/LineageApi.ts index ebdf852..a4c4851 100644 --- a/src/mlmd/LineageApi.ts +++ b/src/mlmd/LineageApi.ts @@ -1,6 +1,14 @@ -import {ArtifactType, ExecutionType, GetExecutionTypesRequest} from ".."; +import { + ArtifactType, + Event, + ExecutionType, formatDateString, + GetEventsByArtifactIDsRequest, + GetExecutionTypesRequest, +} from ".."; import {GetArtifactTypesRequest, MetadataStoreServicePromiseClient} from ".."; +const EventType = Event.Type; + export type ArtifactTypeMap = Map; export type ExecutionTypeMap = Map; @@ -48,4 +56,41 @@ export async function getExecutionTypes( }); return executionTypesMap; -} \ No newline at end of file +} + +export async function getArtifactCreationTime( + artifactId: number, + metadataStoreService: MetadataStoreServicePromiseClient, + errorCallback?: (message: string) => void +): Promise { + if (!artifactId) { + throw new Error('artifactId is empty'); + } + + const eventsRequest = new GetEventsByArtifactIDsRequest(); + eventsRequest.setArtifactIdsList([artifactId]); + const response = + await metadataStoreService.getEventsByArtifactIDs(eventsRequest); + + if (!response) { + if (errorCallback) { + errorCallback(`Unable to retrieve Events for artifactId: ${artifactId}`); + } + return ''; + } + + const data = response.getEventsList().map(event => ({ + time: event.getMillisecondsSinceEpoch(), + type: event.getType() || EventType.UNKNOWN, + })); + // The last output event is the event that produced current artifact. + const lastOutputEvent = data + .reverse() + .find(event => event.type === EventType.DECLARED_OUTPUT || event.type === EventType.OUTPUT); + if (lastOutputEvent && lastOutputEvent.time) { + return formatDateString(new Date(lastOutputEvent.time)); + } else { + // No valid time found, just return empty + return ''; + } +} diff --git a/src/mlmd/Utils.tsx b/src/mlmd/Utils.tsx index 355660a..2093718 100644 --- a/src/mlmd/Utils.tsx +++ b/src/mlmd/Utils.tsx @@ -20,10 +20,11 @@ import { ExecutionCustomProperties, ExecutionProperties, } from './Api'; -import {LineageResource} from "./LineageTypes"; import {ArtifactTypeMap} from "./LineageApi"; import {Artifact, Execution, Value} from '..'; +const UNNAMED_RESOURCE_DISPLAY_NAME = '(unnamed)'; + export function getResourceProperty(resource: Artifact | Execution, propertyName: string, fromCustomProperties = false): string | number | null { const props = fromCustomProperties @@ -35,21 +36,25 @@ export function getResourceProperty(resource: Artifact | Execution, } function getArtifactName(artifact: Artifact): string { - return String(getResourceProperty(artifact, ArtifactProperties.NAME)) + const artifactName = getResourceProperty(artifact, ArtifactProperties.NAME) || + getResourceProperty(artifact, ArtifactCustomProperties.NAME, true); + return artifactName ? artifactName.toString() : UNNAMED_RESOURCE_DISPLAY_NAME; } function getExecutionName(execution: Execution): string { - return String(getResourceProperty(execution, ExecutionProperties.NAME)) + const executionName = getResourceProperty(execution, ExecutionProperties.COMPONENT_ID) || + getResourceProperty(execution, ExecutionCustomProperties.TASK_ID, true); + return executionName ? executionName.toString() : UNNAMED_RESOURCE_DISPLAY_NAME; } -export function getResourceName(resource: LineageResource): string { +export function getResourceName(resource: Artifact | Execution): string { if (resource instanceof Artifact) { return getArtifactName(resource); } return getExecutionName(resource); } -export function getResourceDescription(resource: LineageResource): string { +export function getResourceDescription(resource: Artifact | Execution): string { let description; if (resource instanceof Artifact) { description = getResourceProperty(resource, ArtifactProperties.PIPELINE_NAME)