From 46be24f9adba2b7cd690b2b833b477a0b0e3cc31 Mon Sep 17 00:00:00 2001 From: Ruslans Uralovs Date: Fri, 7 Dec 2018 15:02:25 +0000 Subject: [PATCH] KYLO-3073 Profile page with a lot of entries is slow --- .../rest/controller/FeedRestController.java | 28 ++++++- .../hive/service/HiveService.java | 7 ++ .../define-feed-ng2/define-feed.module.ts | 2 + .../history/profile-history.component.html | 73 +++++-------------- .../history/profile-history.component.ts | 61 ++++++++++------ .../js/feed-mgr/services/RestUrlConstants.ts | 4 +- 6 files changed, 94 insertions(+), 81 deletions(-) diff --git a/services/feed-manager-service/feed-manager-controller/src/main/java/com/thinkbiganalytics/feedmgr/rest/controller/FeedRestController.java b/services/feed-manager-service/feed-manager-controller/src/main/java/com/thinkbiganalytics/feedmgr/rest/controller/FeedRestController.java index 25485bf04c..e9f3a9df80 100644 --- a/services/feed-manager-service/feed-manager-controller/src/main/java/com/thinkbiganalytics/feedmgr/rest/controller/FeedRestController.java +++ b/services/feed-manager-service/feed-manager-controller/src/main/java/com/thinkbiganalytics/feedmgr/rest/controller/FeedRestController.java @@ -87,6 +87,7 @@ import org.slf4j.LoggerFactory; import org.springframework.dao.DataAccessException; import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Sort; import org.springframework.stereotype.Component; @@ -103,6 +104,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Comparator; import java.util.EnumSet; import java.util.List; import java.util.Map; @@ -995,10 +997,27 @@ public Response mergeTemplate(@PathParam("feedId") String feedId, FeedMetadata f @ApiResponse(code = 200, message = "Returns the profile summaries.", response = Map.class, responseContainer = "List"), @ApiResponse(code = 500, message = "The profiles are unavailable.", response = RestResponseStatus.class) }) - public Response profileSummary(@PathParam("feedId") String feedId) { + public Response profileSummary(@PathParam("feedId") String feedId, + @QueryParam("page") Integer page, + @QueryParam("pageSize") Integer pageSize) { + if (page == null || page <= 0) { + page = 1; + } + if (pageSize == null || pageSize <= 0) { + pageSize = 10; + } + + FeedMetadata feedMetadata = getMetadataService().getFeedById(feedId); - final String profileTable = HiveUtils.quoteIdentifier(feedMetadata.getProfileTableName()); - String query = "SELECT * from " + profileTable + " where columnname = '(ALL)'"; + String profileTableName = feedMetadata.getProfileTableName(); + QueryResult tablePartitions = hiveService.getTablePartitions(profileTableName); + List> partitions = tablePartitions.getRows(); + Stream sortedPartitions = partitions.stream().map(row -> Long.parseLong(row.get("partition").toString().substring("processing_dttm=".length()))).sorted(Comparator.reverseOrder()); + long totalPartitions = partitions.size(); + List partitionsPage = sortedPartitions.skip((page - 1) * pageSize).limit(pageSize).map(partition -> "'" + Long.toString(partition) + "'").collect(Collectors.toList()); + + final String profileTable = HiveUtils.quoteIdentifier(profileTableName); + String query = "SELECT * from " + profileTable + " where columnname = '(ALL)' and processing_dttm in (" + StringUtils.join(partitionsPage, ',') + ")"; List> rows = new ArrayList<>(); try { @@ -1032,7 +1051,8 @@ public Response profileSummary(@PathParam("feedId") String feedId) { } } - return Response.ok(rows).build(); + PageImpl> response = new PageImpl<>(rows, null, totalPartitions); + return Response.ok(response).build(); } @GET diff --git a/services/thrift-proxy-service/thrift-proxy-core/src/main/java/com/thinkbiganalytics/hive/service/HiveService.java b/services/thrift-proxy-service/thrift-proxy-core/src/main/java/com/thinkbiganalytics/hive/service/HiveService.java index 0808b016ad..845789ee0b 100644 --- a/services/thrift-proxy-service/thrift-proxy-core/src/main/java/com/thinkbiganalytics/hive/service/HiveService.java +++ b/services/thrift-proxy-service/thrift-proxy-core/src/main/java/com/thinkbiganalytics/hive/service/HiveService.java @@ -272,4 +272,11 @@ public void refreshHiveAccessCacheForImpersonatedUser() { this.perUserAccessCache.refresh(currentUser); } + public QueryResult getTablePartitions(String tableName) { + if (!isTableAccessibleByImpersonatedUser(tableName)) { + throw new RuntimeException(String.format("Table '%s' not found", tableName)); + } + + return new QueryRunner(jdbcTemplate).query("show partitions " + HiveUtils.quoteIdentifier(tableName)); + } } diff --git a/ui/ui-app/src/main/resources/static/js/feed-mgr/feeds/define-feed-ng2/define-feed.module.ts b/ui/ui-app/src/main/resources/static/js/feed-mgr/feeds/define-feed-ng2/define-feed.module.ts index 96f019c7fa..cc44b4b478 100644 --- a/ui/ui-app/src/main/resources/static/js/feed-mgr/feeds/define-feed-ng2/define-feed.module.ts +++ b/ui/ui-app/src/main/resources/static/js/feed-mgr/feeds/define-feed-ng2/define-feed.module.ts @@ -126,6 +126,7 @@ import {TruncatePipe} from './summary/profile/container/stats/truncate-pipe'; import {FeedStatsModule} from '../../../ops-mgr/feeds/feed-stats-ng2/feed-stats.module'; import {MetadataIndexingComponent} from "../../shared/metadata-indexing/metadata-indexing.component"; import {JitCompilerFactory} from '@angular/platform-browser-dynamic'; +import {CovalentPagingModule} from '@covalent/core/paging'; export function createCompilerFn(c: CompilerFactory) { return c.createCompiler([{useJit: true}]); @@ -217,6 +218,7 @@ export function createCompilerFn(c: CompilerFactory) { CovalentCommonModule, CovalentChipsModule, CovalentDataTableModule, + CovalentPagingModule, CovalentDialogsModule, CovalentLayoutModule, CovalentLoadingModule, diff --git a/ui/ui-app/src/main/resources/static/js/feed-mgr/feeds/define-feed-ng2/summary/profile/history/profile-history.component.html b/ui/ui-app/src/main/resources/static/js/feed-mgr/feeds/define-feed-ng2/summary/profile/history/profile-history.component.html index 4b12cd0e4a..d7ccab2723 100644 --- a/ui/ui-app/src/main/resources/static/js/feed-mgr/feeds/define-feed-ng2/summary/profile/history/profile-history.component.html +++ b/ui/ui-app/src/main/resources/static/js/feed-mgr/feeds/define-feed-ng2/summary/profile/history/profile-history.component.html @@ -10,59 +10,26 @@
- - - - - - - -
- -
-
- {{row.TOTAL_COUNT}} -
-
views.profile-history.Rows
-
- -
- - {{row.VALID_COUNT}} - -
views.profile-history.Valid
-
- -
-
- {{row.INVALID_COUNT}} -
-
views.profile-history.Invalid
-
- -
-
- {{row.DATE_TIME | timeAgo:reference}} -
-
views.profile-history.ProcessingDate
-
- -
-
- {{row.MIN_TIMESTAMP}} to {{row.MAX_TIMESTAMP}} -
-
views.profile-history.Dates Processed
-
- -
- - - - - - + + +
+

No results to display.

+
+ + Rows per page: + + + {{size}} + + + {{pagingBar.range}} of {{pagingBar.total}} +
diff --git a/ui/ui-app/src/main/resources/static/js/feed-mgr/feeds/define-feed-ng2/summary/profile/history/profile-history.component.ts b/ui/ui-app/src/main/resources/static/js/feed-mgr/feeds/define-feed-ng2/summary/profile/history/profile-history.component.ts index bd267d861b..ed1a270a79 100644 --- a/ui/ui-app/src/main/resources/static/js/feed-mgr/feeds/define-feed-ng2/summary/profile/history/profile-history.component.ts +++ b/ui/ui-app/src/main/resources/static/js/feed-mgr/feeds/define-feed-ng2/summary/profile/history/profile-history.component.ts @@ -3,8 +3,12 @@ import {HttpClient} from '@angular/common/http'; import * as _ from 'underscore'; import * as angular from 'angular'; import {StateService} from "@uirouter/angular"; -import {FEED_DEFINITION_SECTION_STATE_NAME, FEED_DEFINITION_SUMMARY_STATE_NAME} from '../../../../../model/feed/feed-constants'; +import {FEED_DEFINITION_SUMMARY_STATE_NAME} from '../../../../../model/feed/feed-constants'; import {KyloIcons} from "../../../../../../kylo-utils/kylo-icons"; +import {IPageChangeEvent} from '@covalent/core/paging'; +import {ITdDataTableColumn} from '@covalent/core/data-table'; +import {TranslateService} from '@ngx-translate/core'; +import {TdTimeAgoPipe} from '@covalent/core/common'; @Component({ selector: 'profile-history', @@ -21,35 +25,48 @@ export class ProfileHistoryComponent implements OnInit { private hiveService: any; private utils: any; showSummary: boolean = true; - private profileSummary: Array = []; - public loading: boolean = false; + profileSummary: Array = []; + loading: boolean = false; showNoResults: boolean = false; + kyloIcons_Links_profile = KyloIcons.Links.profile; + currentPage: number = 1; + pageSize: number = 10; + filteredTotal = 0; + columns: ITdDataTableColumn[]; - public kyloIcons_Links_profile = KyloIcons.Links.profile; - - constructor(private $$angularInjector: Injector, private http: HttpClient, private state: StateService) { + constructor(private $$angularInjector: Injector, private http: HttpClient, private state: StateService, private translate: TranslateService) { this.hiveService = $$angularInjector.get("HiveService"); this.utils = $$angularInjector.get("Utils"); this.restUrlService = $$angularInjector.get("RestUrlService"); + + const timeAgo = new TdTimeAgoPipe(); + const DATE_FORMAT: (v: any) => any = (v: number) => timeAgo.transform(v); + + this.columns = [ + { name: 'TOTAL_COUNT', label: this.translate.instant('views.profile-history.Rows'), sortable: false}, + { name: 'VALID_COUNT', label: this.translate.instant('views.profile-history.Valid'), filter: false, sortable: false }, + { name: 'INVALID_COUNT', label: this.translate.instant('views.profile-history.Invalid'), filter: false, sortable: false}, + { name: 'DATE', label: this.translate.instant('views.profile-history.ProcessingDate'), filter: false, sortable: false, format: DATE_FORMAT}, + ]; } public ngOnInit(): void { this.feedId = this.stateParams ? this.stateParams.feedId : undefined; this.processingdttm = this.stateParams ? this.stateParams.processingdttm : undefined; - this.getProfileHistory(); + this.getProfileHistory(1); } - private getProfileHistory() { + private getProfileHistory(fromPage: number) { this.loading = true; this.showNoResults = false; const successFn = (response: any) => { - if (response.length == 0) { + if (response.content.length == 0) { this.showNoResults = true; } const dataMap: any = {}; let dataArr: any = []; - const columns: any = this.hiveService.getColumnNamesForQueryResult(response); + const columns: any = this.hiveService.getColumnNamesForQueryResult(response.content); if (columns != null) { //get the keys into the map for the different columns const dateColumn: any = _.find(columns, (column) => { @@ -65,7 +82,7 @@ export class ProfileHistoryComponent implements OnInit { }); //group on date column - angular.forEach(response, (row: any) => { + angular.forEach(response.content, (row: any) => { const date = row[dateColumn]; if (dataMap[date] == undefined) { const timeInMillis = this.hiveService.getUTCTime(date); @@ -92,13 +109,9 @@ export class ProfileHistoryComponent implements OnInit { }); //sort it desc - - // dataArr = _.sortBy(dataArr,dateColumn).reverse(); - dataArr = _.sortBy(dataArr, 'DATE_TIME').reverse(); - - this.profileSummary = dataArr; - - + this.profileSummary = _.sortBy(dataArr, 'DATE_TIME').reverse();; + this.filteredTotal = response.totalElements; + this.loading = false; } this.loading = false; @@ -108,7 +121,7 @@ export class ProfileHistoryComponent implements OnInit { this.loading = false; }; - this.http.get(this.restUrlService.FEED_PROFILE_SUMMARY_URL(this.feedId)).toPromise().then(successFn, errorFn); + this.http.get(this.restUrlService.FEED_PROFILE_SUMMARY_URL(this.feedId, fromPage, this.pageSize)).toPromise().then(successFn, errorFn); }; onValidCountClick(row: any) { @@ -119,13 +132,17 @@ export class ProfileHistoryComponent implements OnInit { this.goToResults(row, 'invalid'); } - public viewProfileResults(row: any) { - this.goToResults(row, 'stats'); + public viewProfileResults(event: any) { + this.goToResults(event.row, 'stats'); }; goToResults(row: any, type: string) { this.state.go(FEED_DEFINITION_SUMMARY_STATE_NAME+".profile.results", {processingdttm: row['PROCESSING_DTTM'], t: type}); } - + page(pagingEvent: IPageChangeEvent): void { + this.currentPage = pagingEvent.page; + this.pageSize = pagingEvent.pageSize; + this.getProfileHistory(this.currentPage); + } } diff --git a/ui/ui-app/src/main/resources/static/js/feed-mgr/services/RestUrlConstants.ts b/ui/ui-app/src/main/resources/static/js/feed-mgr/services/RestUrlConstants.ts index 39555d8620..5f35e20d00 100644 --- a/ui/ui-app/src/main/resources/static/js/feed-mgr/services/RestUrlConstants.ts +++ b/ui/ui-app/src/main/resources/static/js/feed-mgr/services/RestUrlConstants.ts @@ -192,8 +192,8 @@ export class RestUrlConstants { return RestUrlConstants.GET_FEEDS_URL + "/" + feedId + "/profile-stats"; } - static FEED_PROFILE_SUMMARY_URL(feedId:string) { - return RestUrlConstants.GET_FEEDS_URL + "/" + feedId + "/profile-summary"; + static FEED_PROFILE_SUMMARY_URL(feedId:string, pageNumber: number, pageSize: number) { + return RestUrlConstants.GET_FEEDS_URL + "/" + feedId + "/profile-summary?page=" + pageNumber + "&pageSize=" + pageSize; } static FEED_PROFILE_VALID_RESULTS_URL(feedId:string, processingDttm:string) {