Skip to content

Commit

Permalink
KYLO-3073 Profile page with a lot of entries is slow
Browse files Browse the repository at this point in the history
  • Loading branch information
@uralovs
uralovs committed Dec 7, 2018
1 parent b1967e0 commit 46be24f
Show file tree
Hide file tree
Showing 6 changed files with 94 additions and 81 deletions.
28 changes: 24 additions & 4 deletions ...oller/src/main/java/com/thinkbiganalytics/feedmgr/rest/controller/FeedRestController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@
import org.slf4j.LoggerFactory;
import org.springframework.dao.DataAccessException;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort;
import org.springframework.stereotype.Component;
Expand All @@ -103,6 +104,7 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Comparator;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -995,10 +997,27 @@ public Response mergeTemplate(@PathParam("feedId") String feedId, FeedMetadata f
@ApiResponse(code = 200, message = "Returns the profile summaries.", response = Map.class, responseContainer = "List"),
@ApiResponse(code = 500, message = "The profiles are unavailable.", response = RestResponseStatus.class)
})
public Response profileSummary(@PathParam("feedId") String feedId) {
public Response profileSummary(@PathParam("feedId") String feedId,
@QueryParam("page") Integer page,
@QueryParam("pageSize") Integer pageSize) {
if (page == null || page <= 0) {
page = 1;
}
if (pageSize == null || pageSize <= 0) {
pageSize = 10;
}


FeedMetadata feedMetadata = getMetadataService().getFeedById(feedId);
final String profileTable = HiveUtils.quoteIdentifier(feedMetadata.getProfileTableName());
String query = "SELECT * from " + profileTable + " where columnname = '(ALL)'";
String profileTableName = feedMetadata.getProfileTableName();
QueryResult tablePartitions = hiveService.getTablePartitions(profileTableName);
List<Map<String, Object>> partitions = tablePartitions.getRows();
Stream<Long> sortedPartitions = partitions.stream().map(row -> Long.parseLong(row.get("partition").toString().substring("processing_dttm=".length()))).sorted(Comparator.reverseOrder());
long totalPartitions = partitions.size();
List<String> partitionsPage = sortedPartitions.skip((page - 1) * pageSize).limit(pageSize).map(partition -> "'" + Long.toString(partition) + "'").collect(Collectors.toList());

final String profileTable = HiveUtils.quoteIdentifier(profileTableName);
String query = "SELECT * from " + profileTable + " where columnname = '(ALL)' and processing_dttm in (" + StringUtils.join(partitionsPage, ',') + ")";

List<Map<String, Object>> rows = new ArrayList<>();
try {
Expand Down Expand Up @@ -1032,7 +1051,8 @@ public Response profileSummary(@PathParam("feedId") String feedId) {
}
}

return Response.ok(rows).build();
PageImpl<Map<String, Object>> response = new PageImpl<>(rows, null, totalPartitions);
return Response.ok(response).build();
}

@GET
Expand Down
7 changes: 7 additions & 0 deletions ...rvice/thrift-proxy-core/src/main/java/com/thinkbiganalytics/hive/service/HiveService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,4 +272,11 @@ public void refreshHiveAccessCacheForImpersonatedUser() {
this.perUserAccessCache.refresh(currentUser);
}

public QueryResult getTablePartitions(String tableName) {
if (!isTableAccessibleByImpersonatedUser(tableName)) {
throw new RuntimeException(String.format("Table '%s' not found", tableName));
}

return new QueryRunner(jdbcTemplate).query("show partitions " + HiveUtils.quoteIdentifier(tableName));
}
}
2 changes: 2 additions & 0 deletions ui/ui-app/src/main/resources/static/js/feed-mgr/feeds/define-feed-ng2/define-feed.module.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ import {TruncatePipe} from './summary/profile/container/stats/truncate-pipe';
import {FeedStatsModule} from '../../../ops-mgr/feeds/feed-stats-ng2/feed-stats.module';
import {MetadataIndexingComponent} from "../../shared/metadata-indexing/metadata-indexing.component";
import {JitCompilerFactory} from '@angular/platform-browser-dynamic';
import {CovalentPagingModule} from '@covalent/core/paging';

export function createCompilerFn(c: CompilerFactory) {
return c.createCompiler([{useJit: true}]);
Expand Down Expand Up @@ -217,6 +218,7 @@ export function createCompilerFn(c: CompilerFactory) {
CovalentCommonModule,
CovalentChipsModule,
CovalentDataTableModule,
CovalentPagingModule,
CovalentDialogsModule,
CovalentLayoutModule,
CovalentLoadingModule,
Expand Down
73 changes: 20 additions & 53 deletions .../js/feed-mgr/feeds/define-feed-ng2/summary/profile/history/profile-history.component.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,59 +10,26 @@

<mat-card-content>
<div *ngIf="showSummary">

<mat-nav-list browser-height
browser-height-scroll-y="false"
browser-height-offset="265">

<td-virtual-scroll-container #virtualScroll [data]="profileSummary">
<ng-template let-row="row" let-last="isLast" tdVirtualScrollRow>

<a mat-list-item (click)="viewProfileResults(row)" class="profile-row">
<div fxFlex fxLayout="row" fxLayoutAlign="space-around center">

<div>
<div class="item-title" title="{{row.TOTAL_COUNT}}">
{{row.TOTAL_COUNT}}
</div>
<div class="hint" translate>views.profile-history.Rows</div>
</div>

<div>
<a (click)="onValidCountClick(row)" class="item-title primary-color-1" title="{{row.VALID_COUNT}}">
{{row.VALID_COUNT}}
</a>
<div class="hint" translate>views.profile-history.Valid</div>
</div>

<div>
<div (click)="onInvalidCountClick(row)" class="item-title primary-color-1" title="{{row.INVALID_COUNT}}">
{{row.INVALID_COUNT}}
</div>
<div class="hint" translate>views.profile-history.Invalid</div>
</div>

<div>
<div class="item-title" title="{{row.DATE_TIME}}">
{{row.DATE_TIME | timeAgo:reference}}
</div>
<div class="hint" translate>views.profile-history.ProcessingDate</div>
</div>

<div *ngIf="row.MIN_TIMESTAMP">
<div class="item-title" title="{{row.MIN_TIMESTAMP}} - {{row.MAX_TIMESTAMP}}">
{{row.MIN_TIMESTAMP}} to {{row.MAX_TIMESTAMP}}
</div>
<div class="hint" translate>views.profile-history.Dates Processed</div>
</div>

</div>
</a>

<mat-divider [inset]="true" *ngIf="!isLast"></mat-divider>
</ng-template>
</td-virtual-scroll-container>
</mat-nav-list>
<td-data-table
#dataTable
[data]="profileSummary"
[columns]="columns"
[clickable]="true"
(rowClick)="viewProfileResults($event)"
[style.min-height.px]="200">
</td-data-table>
<div class="md-padding" *ngIf="!dataTable.hasData" layout="row" layout-align="center center">
<h3>No results to display.</h3>
</div>
<td-paging-bar #pagingBar [pageSize]="pageSize" [total]="filteredTotal" (change)="page($event)">
<span hide-xs>Rows per page:</span>
<mat-select [style.width.px]="50" [(ngModel)]="pageSize">
<mat-option *ngFor="let size of [10, 50, 100, 200, 500]" [value]="size">
{{size}}
</mat-option>
</mat-select>
{{pagingBar.range}} <span hide-xs>of {{pagingBar.total}}</span>
</td-paging-bar>
</div>
<div *ngIf="showNoResults" class="no-data" fxLayout="column" fxLayoutAlign="center center">
<ng-md-icon class="tc-grey-500" icon="find_in_page" size="96"></ng-md-icon>
Expand Down
61 changes: 39 additions & 22 deletions ...ic/js/feed-mgr/feeds/define-feed-ng2/summary/profile/history/profile-history.component.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,12 @@ import {HttpClient} from '@angular/common/http';
import * as _ from 'underscore';
import * as angular from 'angular';
import {StateService} from "@uirouter/angular";
import {FEED_DEFINITION_SECTION_STATE_NAME, FEED_DEFINITION_SUMMARY_STATE_NAME} from '../../../../../model/feed/feed-constants';
import {FEED_DEFINITION_SUMMARY_STATE_NAME} from '../../../../../model/feed/feed-constants';
import {KyloIcons} from "../../../../../../kylo-utils/kylo-icons";
import {IPageChangeEvent} from '@covalent/core/paging';
import {ITdDataTableColumn} from '@covalent/core/data-table';
import {TranslateService} from '@ngx-translate/core';
import {TdTimeAgoPipe} from '@covalent/core/common';

@Component({
selector: 'profile-history',
Expand All @@ -21,35 +25,48 @@ export class ProfileHistoryComponent implements OnInit {
private hiveService: any;
private utils: any;
showSummary: boolean = true;
private profileSummary: Array<any> = [];
public loading: boolean = false;
profileSummary: Array<any> = [];
loading: boolean = false;
showNoResults: boolean = false;
kyloIcons_Links_profile = KyloIcons.Links.profile;
currentPage: number = 1;
pageSize: number = 10;
filteredTotal = 0;
columns: ITdDataTableColumn[];

public kyloIcons_Links_profile = KyloIcons.Links.profile;

constructor(private $$angularInjector: Injector, private http: HttpClient, private state: StateService) {
constructor(private $$angularInjector: Injector, private http: HttpClient, private state: StateService, private translate: TranslateService) {
this.hiveService = $$angularInjector.get("HiveService");
this.utils = $$angularInjector.get("Utils");
this.restUrlService = $$angularInjector.get("RestUrlService");

const timeAgo = new TdTimeAgoPipe();
const DATE_FORMAT: (v: any) => any = (v: number) => timeAgo.transform(v);

this.columns = [
{ name: 'TOTAL_COUNT', label: this.translate.instant('views.profile-history.Rows'), sortable: false},
{ name: 'VALID_COUNT', label: this.translate.instant('views.profile-history.Valid'), filter: false, sortable: false },
{ name: 'INVALID_COUNT', label: this.translate.instant('views.profile-history.Invalid'), filter: false, sortable: false},
{ name: 'DATE', label: this.translate.instant('views.profile-history.ProcessingDate'), filter: false, sortable: false, format: DATE_FORMAT},
];
}


public ngOnInit(): void {
this.feedId = this.stateParams ? this.stateParams.feedId : undefined;
this.processingdttm = this.stateParams ? this.stateParams.processingdttm : undefined;
this.getProfileHistory();
this.getProfileHistory(1);
}

private getProfileHistory() {
private getProfileHistory(fromPage: number) {
this.loading = true;
this.showNoResults = false;
const successFn = (response: any) => {
if (response.length == 0) {
if (response.content.length == 0) {
this.showNoResults = true;
}
const dataMap: any = {};
let dataArr: any = [];
const columns: any = this.hiveService.getColumnNamesForQueryResult(response);
const columns: any = this.hiveService.getColumnNamesForQueryResult(response.content);
if (columns != null) {
//get the keys into the map for the different columns
const dateColumn: any = _.find(columns, (column) => {
Expand All @@ -65,7 +82,7 @@ export class ProfileHistoryComponent implements OnInit {
});

//group on date column
angular.forEach(response, (row: any) => {
angular.forEach(response.content, (row: any) => {
const date = row[dateColumn];
if (dataMap[date] == undefined) {
const timeInMillis = this.hiveService.getUTCTime(date);
Expand All @@ -92,13 +109,9 @@ export class ProfileHistoryComponent implements OnInit {
});

//sort it desc

// dataArr = _.sortBy(dataArr,dateColumn).reverse();
dataArr = _.sortBy(dataArr, 'DATE_TIME').reverse();

this.profileSummary = dataArr;


this.profileSummary = _.sortBy(dataArr, 'DATE_TIME').reverse();;
this.filteredTotal = response.totalElements;
this.loading = false;
}
this.loading = false;

Expand All @@ -108,7 +121,7 @@ export class ProfileHistoryComponent implements OnInit {
this.loading = false;
};

this.http.get(this.restUrlService.FEED_PROFILE_SUMMARY_URL(this.feedId)).toPromise().then(successFn, errorFn);
this.http.get(this.restUrlService.FEED_PROFILE_SUMMARY_URL(this.feedId, fromPage, this.pageSize)).toPromise().then(successFn, errorFn);
};

onValidCountClick(row: any) {
Expand All @@ -119,13 +132,17 @@ export class ProfileHistoryComponent implements OnInit {
this.goToResults(row, 'invalid');
}

public viewProfileResults(row: any) {
this.goToResults(row, 'stats');
public viewProfileResults(event: any) {
this.goToResults(event.row, 'stats');
};

goToResults(row: any, type: string) {
this.state.go(FEED_DEFINITION_SUMMARY_STATE_NAME+".profile.results", {processingdttm: row['PROCESSING_DTTM'], t: type});
}


page(pagingEvent: IPageChangeEvent): void {
this.currentPage = pagingEvent.page;
this.pageSize = pagingEvent.pageSize;
this.getProfileHistory(this.currentPage);
}
}
4 changes: 2 additions & 2 deletions ui/ui-app/src/main/resources/static/js/feed-mgr/services/RestUrlConstants.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,8 +192,8 @@ export class RestUrlConstants {
return RestUrlConstants.GET_FEEDS_URL + "/" + feedId + "/profile-stats";
}

static FEED_PROFILE_SUMMARY_URL(feedId:string) {
return RestUrlConstants.GET_FEEDS_URL + "/" + feedId + "/profile-summary";
static FEED_PROFILE_SUMMARY_URL(feedId:string, pageNumber: number, pageSize: number) {
return RestUrlConstants.GET_FEEDS_URL + "/" + feedId + "/profile-summary?page=" + pageNumber + "&pageSize=" + pageSize;
}

static FEED_PROFILE_VALID_RESULTS_URL(feedId:string, processingDttm:string) {
Expand Down

0 comments on commit 46be24f

Please sign in to comment.