Add missing subdirectories after refactor/move of components (#261)

* made jobs listing more compact * add cloudbuild yam file * add cloudbuild yam file * added missing sub directories * Test cloudbuild * Test cloudbuild * Added licence headers to css files * Added licence headers to css files * Added copyright notices * rewired authentication - beta * alternate login, beta implementation * Added automatic login on gt projects button * added optio n to hide/show repartition nodes
GoogleCloudPlatform · Aug 6, 2019 · 08b8e8d · 08b8e8d
1 parent a5254f0
commit 08b8e8d
Show file tree

Hide file tree

Showing 38 changed files with 718 additions and 101 deletions.
diff --git a/tools/bq-visualizer/README.md b/tools/bq-visualizer/README.md
@@ -2,6 +2,14 @@
 
 This utility provides a web application which can be used to visualise the flow of execution stages within a BigQuery job. This may be useful in identifying problematic stages and provides greater usability for large query plans than the default query plan explanation in the Google Cloud Console.
 
+## Release Notes
+15 July 2019 - in order to have the appliction whitelisted on appspot.com, the automatic login
+had to be disabled to allow users to access the Terms and Privacy page prior to logging in.
+
+Treeview will by default hide reparttions.
+
+Added a Display Options card at the bottom where this can be changed. 
+
 ## Manual
 
 ### Overview
@@ -60,3 +68,6 @@ The timing Tab displaus a Gantt style view to quickly show how long the indivuda
 ## Known Limits
 
 The application will only display graphs for queries. Load jobs etc do not result in query stages being output.
+
+Clicking the get projects when not yet logged in will result in a login process being started instead. Users need to
+click on get projects button again after login was successful.
diff --git a/tools/bq-visualizer/cloudbuild.yaml b/tools/bq-visualizer/cloudbuild.yaml
@@ -0,0 +1,34 @@
+
+#  Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+steps:
+
+  # Install node packages
+  - name: 'gcr.io/cloud-builders/npm'
+    args: [ 'install' ]
+    dir: 'tools/bq-visualizer'
+
+  # build
+  - name: 'gcr.io/cloud-builders/npm'
+    args: ['run', 'build', '--', '--prod' ]
+    dir: 'tools/bq-visualizer'
+
+artifacts:
+  objects:
+    location: 'gs://${_BQVISUALISER_BUCKET}/deployment_$BUILD_ID'
+    paths: ['tools/bq-visualizer/dist/*']
+
+
diff --git a/tools/bq-visualizer/package.json b/tools/bq-visualizer/package.json
@@ -4,7 +4,7 @@
   "scripts": {
     "ng": "ng",
     "start": "ng serve",
-    "build": "ng build prod",
+    "build": "ng build",
     "test": "ng test",
     "e2e": "ng e2e",
     "lint": "tslint -t verbose src/**/*.ts",
@@ -65,4 +65,4 @@
     "tslint": "~5.9.1",
     "typescript": "~3.1.1"
   }
-}
+}
diff --git a/tools/bq-visualizer/src/app/app.component.css b/tools/bq-visualizer/src/app/app.component.css
@@ -1,4 +1,20 @@
-.main {
+/*
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ .main {
   overflow: hidden;
   height: 100vh;
 }

diff --git a/tools/bq-visualizer/src/app/app.component.html b/tools/bq-visualizer/src/app/app.component.html
@@ -1,6 +1,9 @@
 <div class="header">
-  <h1 class="header__title">BQ Visualizer</h1>
+  <h1 class="header__title">BQ Visualiser</h1>
   <span class="flex-spacer"></span>
+  <a mat-button [disabled]="this.isLoggedIn" (click)="login()">login</a>
+  <a mat-button [disabled]="!this.isLoggedIn" (click)="logout()">logout</a>
+
   <a mat-button href="/terms">Terms &amp; Privacy</a>
 </div>
 <div class="main">

diff --git a/tools/bq-visualizer/src/app/app.component.spec.ts b/tools/bq-visualizer/src/app/app.component.spec.ts
@@ -27,8 +27,10 @@ import {MatTableModule} from '@angular/material/table';
 import {MatTabsModule} from '@angular/material/tabs';
 import {BrowserAnimationsModule} from '@angular/platform-browser/animations';
 import {RouterTestingModule} from '@angular/router/testing';
-import {OAuthService, UrlHelperService} from 'angular-oauth2-oidc';
+import {OAuthModule, OAuthService, UrlHelperService} from 'angular-oauth2-oidc';
+
 import {environment} from '../environments/environment';
+
 import {AppComponent} from './app.component';
 import {MockOAuthService} from './google-auth.service';
 import {JobComponent} from './job/job.component';
@@ -46,6 +48,7 @@ describe('AppComponent', () => {
     TestBed
         .configureTestingModule({
           imports: [
+            OAuthModule.forRoot(),
             RouterTestingModule,
             HttpClientTestingModule,
             FormsModule,
@@ -105,6 +108,6 @@ describe('AppComponent', () => {
        fixture.detectChanges();
        const compiled = fixture.debugElement.nativeElement;
        expect(compiled.querySelector('h1').textContent)
-           .toContain('BQ Visualizer');
+           .toContain('BQ Visualiser');
      }));
 });
diff --git a/tools/bq-visualizer/src/app/app.component.ts b/tools/bq-visualizer/src/app/app.component.ts
@@ -14,52 +14,39 @@
  * limitations under the License.
  */
 import {Component, ViewChild} from '@angular/core';
-import {MatTabChangeEvent} from '@angular/material/tabs';
-
-import {BqJob} from './bq_job';
-import {BqQueryPlan} from './bq_query_plan';
+// import {MatTabChangeEvent} from '@angular/material/tabs';
 import {GoogleAuthService} from './google-auth.service';
-import {JobComponent} from './job/job.component';
-import {TimingDisplayComponent} from './timing-display/timing-display.component';
-import {VisDisplayComponent} from './vis-display/vis-display.component';
+
+
 
 @Component({
   selector: 'app-root',
   templateUrl: './app.component.html',
   styleUrls: ['./app.component.css']
 })
 export class AppComponent {
-  title = 'BQ Visualizer';
-  /*
-    @ViewChild('tabs') tabGroup;
-    @ViewChild('job') jobComponent: JobComponent;
-    @ViewChild('tree') visComponent: VisDisplayComponent;
-    @ViewChild('timing') timingComponent: TimingDisplayComponent;
-  */
-  // adding the authservice here causes the application to invoke authentication
-  constructor(private authService: GoogleAuthService) {}
-  /*
-    async ngOnInit() {
-      this.jobComponent.planSelected.subscribe(async plan => {
-        // Load the query plan into the display components.
-        this.visComponent.loadPlan(plan);
-        this.timingComponent.loadPlan(plan);
-
-        // Switch to the 'Tree' tab.
-        this.tabGroup.selectedIndex = 1;
-      });
+  title = 'BQ Visualiser';
+  isLoggedIn = false;
+  constructor(private googleAuthService: GoogleAuthService) {
+    this.googleAuthService.loginEvent.subscribe(
+        (isloggedIn: boolean) => this.register_login(isloggedIn));
 
-      this.tabGroup.selectedTabChange.subscribe((tab: MatTabChangeEvent) => {
-        switch (tab.index) {
-          case 1:
-            this.visComponent.draw();
-            break;
-          case 2:
-            this.timingComponent.draw();
-            break;
-        }
-      })
-    }
+    this.isLoggedIn = this.googleAuthService.isLoggedIn();
+    /*
+    console.log(
+        'isloggedin = ' + this.googleAuthService.getAccessToken() != null);
+    console.log('token = ' + this.googleAuthService.getAccessToken());
+    */
+  }
+  /* event handler to recognise a login or logout event has occurred */
+  private register_login(what: boolean) {
+    this.isLoggedIn = what;
+  }
+  public login() {
+    this.googleAuthService.login();
+  }
 
-  */
+  public logout() {
+    this.googleAuthService.logout();
+  }
 }
diff --git a/tools/bq-visualizer/src/app/big-query.service.spec.ts b/tools/bq-visualizer/src/app/big-query.service.spec.ts
@@ -15,7 +15,7 @@
  */
 import {HttpClientTestingModule, HttpTestingController} from '@angular/common/http/testing';
 import {async, inject, TestBed} from '@angular/core/testing';
-import {OAuthService, UrlHelperService} from 'angular-oauth2-oidc';
+import {OAuthModule, OAuthService, UrlHelperService} from 'angular-oauth2-oidc';
 import {of} from 'rxjs';
 import {take} from 'rxjs/operators';
 
@@ -32,7 +32,7 @@ describe('BigQueryService', () => {
 
   beforeEach(() => {
     TestBed.configureTestingModule({
-      imports: [HttpClientTestingModule],
+      imports: [HttpClientTestingModule, OAuthModule.forRoot()],
       providers: [
         BigQueryService,
         {provide: OAuthService, useClass: MockOAuthService},
@@ -47,14 +47,14 @@ describe('BigQueryService', () => {
   it('should get a single page of jobs', () => {
     const project = 'stephanmeyn-train-cbp';
     const jobId = 'bquxjob_7397faf7_1679db86d97';
-    let jobs: BqJob[] = [];
-    service.getJobs(`${project}.${jobId}`).pipe(take(10)).subscribe(job => {
+    const jobs: BqJob[] = [];
+    service.getJobs(`${project}.${jobId}`, 10).pipe(take(10)).subscribe(job => {
       jobs.push(job);
     }, console.error);
 
     const mockReq = httpMock.expectOne(
         `${environment.bqUrl}/${project}.${jobId}/jobs?` +
-        `access_token=fake-oauth-token&maxResults=200&projection=full`);
+        `access_token=fake-oauth-token&maxResults=200&allUsers=true&projection=full`);
     expect(mockReq.cancelled).toBeFalsy();
     expect(mockReq.request.responseType).toEqual('json');
     mockReq.flush(require('../assets/test/get_jobs.json'));
@@ -76,13 +76,14 @@ describe('BigQueryService', () => {
 
   it('should get a single job', () => {
     let job: Job;
-    service.getQueryPlan('projectid.foobar', 'abc1234').subscribe(res => {
-      job = res;
-    }, console.error);
+    service.getQueryPlan('projectid.foobar', 'abc1234', 'somelocation')
+        .subscribe(res => {
+          job = res;
+        }, console.error);
 
     const mockReq = httpMock.expectOne(
         environment.bqUrl +
-        '/projectid.foobar/jobs/abc1234?access_token=fake-oauth-token');
+        '/projectid.foobar/jobs/abc1234?access_token=fake-oauth-token&location=somelocation');
     expect(mockReq.cancelled).toBeFalsy();
     expect(mockReq.request.responseType).toEqual('json');
     mockReq.flush(require('../assets/test/small_query_plan.json'));
@@ -96,7 +97,7 @@ describe('BigQueryService', () => {
   });
 
   it('should get projects', () => {
-    let projects: BqProject[] = [];
+    const projects: BqProject[] = [];
     service.getProjects().subscribe(project => {
       projects.push(project);
     }, console.error);
@@ -122,8 +123,8 @@ describe('BigQueryService', () => {
       return of(require('../assets/test/get_jobs_page_1.json'));
     });
 
-    let jobs: BqJob[] = [];
-    service.getJobs('stephanmeyn-train-cbp.bquxjob_7397faf7_1679db86d97')
+    const jobs: BqJob[] = [];
+    service.getJobs('stephanmeyn-train-cbp.bquxjob_7397faf7_1679db86d97', 50)
         .subscribe(
             job => {
               jobs.push(job);

diff --git a/tools/bq-visualizer/src/app/big-query.service.ts b/tools/bq-visualizer/src/app/big-query.service.ts
@@ -15,13 +15,14 @@
  */
 import {HttpClient} from '@angular/common/http';
 import {Injectable} from '@angular/core';
-import {OAuthService} from 'angular-oauth2-oidc';
+// import {OAuthService} from 'angular-oauth2-oidc';
 import {concat, defer, EMPTY, from, Observable, of, Subscription} from 'rxjs';
 import {catchError, filter, map} from 'rxjs/operators';
 
 import {environment} from '../environments/environment';
 
 import {BqJob} from './bq_job';
+import {GoogleAuthService} from './google-auth.service';
 import {LogService} from './log.service';
 import {BqListJobResponse, BqProject, BqProjectListResponse, Job} from './rest_interfaces';
 
@@ -35,7 +36,8 @@ export class BigQueryService {
   lastProjectId: string;
 
   constructor(
-      private http: HttpClient, private oauthService: OAuthService,
+      private http: HttpClient,  // private oauthService: OAuthService,
+      private googleAuthService: GoogleAuthService,
       private logSvc: LogService) {}
 
   /** Get the detail of a job. */
@@ -46,8 +48,7 @@ export class BigQueryService {
     const realid = jobId.split('.').slice(-1)[0];
     this.logSvc.debug(`getQueryPlan: fetched query plan for jobid=${jobId}`);
 
-    const token = this.oauthService.getAccessToken();
-    console.log(token);
+    const token = this.googleAuthService.getAccessToken();
 
     const args = {access_token: token, location: location};
     const url = bqUrl(`/${projectId}/jobs/${realid}`, args);
@@ -60,7 +61,7 @@ export class BigQueryService {
   /** Get all jobs for a project. */
   getJobs(projectId: string, maxJobs: number): Observable<BqJob> {
     return Observable.create(async obs => {
-      const token = this.oauthService.getAccessToken();
+      const token = this.googleAuthService.getAccessToken();
       let nextPageToken = '';
       let totalJobs = 0;
       while (true) {
@@ -71,7 +72,6 @@ export class BigQueryService {
           projection: 'full',
           pageToken: nextPageToken,
         });
-        console.log('getJobs: ' + url);
 
         try {
           await new Promise((resolve, reject) => {
@@ -91,7 +91,7 @@ export class BigQueryService {
                   }
                   nextPageToken = res.nextPageToken;
                   totalJobs += res.jobs.length;
-                  console.log('totalJobs: ' + totalJobs);
+                  // console.log('totalJobs: ' + totalJobs);
                   if (totalJobs >= maxJobs) {
                     obs.complete();
                     return;
@@ -120,7 +120,17 @@ export class BigQueryService {
   /** Get all projects. */
   getProjects(): Observable<BqProject> {
     return Observable.create(async obs => {
-      const token = this.oauthService.getAccessToken();
+      if (this.googleAuthService.isLoggedIn() === false) {
+        await this.googleAuthService.login();
+        if (this.googleAuthService.isLoggedIn) {
+          this.logSvc.info('successfully Logged in');
+        } else {
+          this.logSvc.error  ('failed Logged in');
+          obs.error('No authentication token available.');
+        }
+      }
+      const token = this.googleAuthService.getAccessToken();
+
       let nextPageToken = '';
       while (true) {
         const url = bqUrl('', {

diff --git a/tools/bq-visualizer/src/app/bq_query_plan.spec.ts b/tools/bq-visualizer/src/app/bq_query_plan.spec.ts
@@ -64,7 +64,7 @@ describe('BqQueryPlan', () => {
     const stats = JSON.parse(statsString);
     expect(stats).toBeTruthy();
     expect(stats['steps']).toBeFalsy();
-    expect(stats.name).toEqual('S00: Input');
+    expect(stats['name           ']).toEqual('S00: Input');
   });
 
   it('should get colour for the max time', () => {

diff --git a/tools/bq-visualizer/src/app/bq_query_plan.ts b/tools/bq-visualizer/src/app/bq_query_plan.ts
@@ -76,6 +76,10 @@ export class BqQueryPlan {
     }
   }
 
+  /** return nodes minus repartition typwe nodes */
+  public nodesWithoutRepartitions() {
+    return this.nodes.filter(node => (node.name.indexOf('Repartition') < 0));
+  }
   /** extract all node ids that are read from */
   private getReads(node: QueryStage): string[] {
     if (!node.steps) {