Merge main and fix conflicts

Signed-off-by: Eugene Lee <eugenesk@amazon.com>
opensearch-project · Feb 24, 2022 · ddad215 · ddad215
2 parents 824c3c2 + f14c32c
commit ddad215
Show file tree

Hide file tree

Showing 71 changed files with 2,942 additions and 1,555 deletions.
diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml
@@ -0,0 +1,28 @@
+name: Backport
+on:
+  pull_request_target:
+    types:
+      - closed
+      - labeled
+
+jobs:
+  backport:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+    name: Backport
+    steps:
+      - name: GitHub App token
+        id: github_app_token
+        uses: tibdex/github-app-token@v1.5.0
+        with:
+          app_id: ${{ secrets.APP_ID }}
+          private_key: ${{ secrets.APP_PRIVATE_KEY }}
+          installation_id: 22958780
+
+      - name: Backport
+        uses: VachaShah/backport@v1.1.4
+        with:
+          github_token: ${{ steps.github_app_token.outputs.token }}
+          branch_name: backport/backport-${{ github.event.number }}
diff --git a/.github/workflows/delete_backport_branch.yml b/.github/workflows/delete_backport_branch.yml
@@ -0,0 +1,15 @@
+name: Delete merged branch of the backport PRs
+on: 
+  pull_request:
+    types:
+      - closed
+
+jobs:
+  delete-branch:
+    runs-on: ubuntu-latest
+    if: startsWith(github.event.pull_request.head.ref,'backport/')
+    steps:
+      - name: Delete merged branch
+        uses: SvanBoxel/delete-merged-branch@main
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md
@@ -43,3 +43,11 @@ Example output: `./build/observability*.zip`
 ### Submitting Changes
 
 See [CONTRIBUTING](CONTRIBUTING.md).
+
+### Backports
+
+The Github workflow in [`backport.yml`](.github/workflows/backport.yml) creates backport PRs automatically when the original PR
+with an appropriate label `backport <backport-branch-name>` is merged to main with the backport workflow run successfully on the
+PR. For example, if a PR on main needs to be backported to `1.x` branch, add a label `backport 1.x` to the PR and make sure the
+backport workflow runs on the PR along with other checks. Once this PR is merged to main, the workflow will create a backport PR
+to the `1.x` branch.
diff --git a/dashboards-observability/.cypress/integration/event_analytics.spec.js b/dashboards-observability/.cypress/integration/event_analytics.spec.js
@@ -386,3 +386,29 @@ describe('Search fields in sidebar', () => {
     cy.get('[data-test-subj="field-OriginAirportID"]').should('exist');
   });
 });
+
+describe('Switch on and off livetail', () => {
+  it('Switch on and off in live tail', () => {
+    landOnEventExplorer();
+    cy.wait(delay);
+
+    cy.get('[data-test-subj="searchAutocompleteTextArea"]').type(TEST_QUERIES[1].query);
+
+    cy.get('[data-test-subj=eventLiveTail]').click();
+    cy.get('[data-test-subj=eventLiveTail__delay10]').click();
+    cy.wait(delay * 2);
+    cy
+      .get('.euiToastHeader__title')
+      .contains('On')
+      .should('exist');
+
+    cy.get('[data-test-subj=eventLiveTail]').click();
+    cy.get('[data-test-subj=eventLiveTail__off').click();
+    cy.wait(delay * 2);
+    cy
+      .get('.euiToastHeader__title')
+      .contains('Off')
+      .should('exist');
+
+  });
+});
diff --git a/dashboards-observability/common/constants/autocomplete.ts b/dashboards-observability/common/constants/autocomplete.ts
@@ -2,6 +2,9 @@
  * Copyright OpenSearch Contributors
  * SPDX-License-Identifier: Apache-2.0
  */
+/* eslint-disable prettier/prettier */
+
+import { BaseItem } from '@algolia/autocomplete-core';
 
 export const firstCommand = [{ label: 'source' }];
 
@@ -42,7 +45,7 @@ export const numberTypes = [
   'unsigned_long',
 ];
 
-export interface AutocompleteItem {
+export interface AutocompleteItem extends BaseItem {
   input: string;
   itemName: string;
   label: string;
@@ -64,37 +67,119 @@ export interface DataItem {
   doc_count: any;
 }
 
-const JUST_SOURCE_REGEX = new RegExp('source = (\\S+)');
-const SOURCE_WHERE_REGEX = new RegExp('source = (\\S+) \\| where \\S+ = \\S+');
-const SOURCE_MATCH_REGEX = new RegExp('source = (\\S+) \\| where match\\(\\S+, \\S+\\)');
-export const EMPTY_REGEX = new RegExp('\\s+');
-export const MATCH_FIELD_AFTER_WHERE = new RegExp('\\s*where\\s*\\S*');
-export const EQUAL_AFTER_FIELD = new RegExp('\\s*where\\s*(\\S+)\\s+');
-export const DATA_AFTER_EQUAL = new RegExp('\\s*where\\s*\\S+\\s*=\\s*\\S*');
-export const PIPE_AFTER_WHERE = new RegExp('\\s*where\\s*\\S+\\s*=\\s*\\S+\\s+');
-export const FIELD_AFTER_MATCH = new RegExp('\\s*where\\s*match\\(\\s*\\S*');
-export const COMMA_AFTER_FIELD = new RegExp('\\s*where\\s*match\\(\\s*(\\S+)\\s+');
-export const DATA_AFTER_COMMA = new RegExp('\\s*where\\s*match\\(\\s*\\S+\\s*,\\s*');
-export const CLOSE_AFTER_DATA = new RegExp('\\s*where\\s*match\\(\\s*\\S+\\s*,\\s*\\S+\\s+');
-export const PIPE_AFTER_MATCH = new RegExp(
-  '\\s*where\\s*match\\(\\s*\\S+\\s*,\\s*\\S+\\s*\\S+\\s*\\)\\s*'
-);
-export const FIELD_AFTER_DEDUP = new RegExp('\\s*dedup\\s*\\S*');
-export const PIPE_AFTER_FIELD = new RegExp('\\s*dedup\\s*\\S+\\s+');
+const JUST_SEARCH_REGEX = /\s*search\s+source\s*=\s*(\S+)/;
+const SEARCH_WHERE_REGEX = /\s*search\s+source\s*=\s*(\S+)\s*\|\s*where\s+\S+\s*=\s*\S+/;
+const SEARCH_MATCH_REGEX = /\s*search\s+source\s*=\s*(\S+)\s*\|\s*where\s+match\(\S+,\s*\S+\)/;
+const JUST_SOURCE_REGEX = /\s*source\s*=\s*(\S+)/;
+const SOURCE_WHERE_REGEX = /\s*source\s*=\s*(\S+)\s*\|\s*where\s+\S+\s*=\s*\S+/;
+const SOURCE_MATCH_REGEX = /\s*source\s*=\s*(\S+)\s*\|\s*where\s+match\(\S+,\s*\S+\)/;
+export const EMPTY_REGEX = /^\s*\S*$/;
+export const FIELD_AFTER_COMMAND = /^\s*(dedup|eval|rare|top|rename|where\s+match\()\s+\S*$/;
+
+// Regex for where command
+export const MATCH_FIELD_AFTER_WHERE = /^\s*where\s+\S*$/;
+export const EQUAL_AFTER_WHERE_FIELD = /^\s*where\s+(\S+)\s+$/;
+export const DATA_AFTER_WHERE_EQUAL = /^\s*where\s+\S+\s*=\s*\S*$/;
+export const PIPE_AFTER_WHERE = /^\s*where\s+\S+\s*=\s*\S+\s+$/;
+export const COMMA_AFTER_FIELD = /^\s*where\s+match\(\s*(\S+)\s+$/;
+export const DATA_AFTER_COMMA = /^\s*where\s+match\(\s*\S+\s*,\s*$/;
+export const CLOSE_AFTER_DATA = /^\s*where\s+match\(\s*\S+\s*,\s*\S+\s+$/;
+export const PIPE_AFTER_MATCH = /^\s*where\s+match\(\s*\S+\s*,\s*\S+\s*\S+\s*\)\s*$/;
+
+// Regex for dedup command
+export const FIELD_IN_FIELD_LOOP = /^\s*dedup\s*\d*\s+\S+\s*(,\s*\S+\s*)*,\s*\S*$/;
+export const PIPE_COMMA_AFTER_FIELD = /^\s*dedup\s*\d*\s+\S+\s*(,\s*\S+\s*)*\s+$/;
+export const PIPE_AFTER_KEEP_EMPTY = /^\s*dedup\s*\d*\s+\S+\s*(,\s*\S+\s*)*\s*keepempty=true\s+$/;
+export const PIPE_AFTER_CONSECUTIVE = /^\s*dedup\s*\d*\s+\S+\s*(,\s*\S+\s*)*\s*consecutive=true\s+$/;
+
+// Regex for eval command
+export const EQUAL_AFTER_EVAL_FIELD = /^\s*eval\s+(\S+)\s+$/;
+export const FIELD_AFTER_EVAL_EQUAL = /^\s*eval\s+\S+\s*=\s*\S*$/;
+export const MATH_AFTER_FIELD = /^\s*eval\s+\S+\s*=\s*\S+\s+$/;
+export const PIPE_MATH_AFTER_EXPRESSIONS = /^\s*eval\s+(\S+\s*=\s*\S+(\s*(\+|\-|\*|\/)\s*\S+)+)+\s+$/;
+
+// Regex for fields command
+export const PLUS_MINUS_FIELD_AFTER_FIELDS = /^\s*fields\s+\S*$/;
+export const FIELD_AFTER_PLUS_MINUS = /^\s*fields\s+(\+|\-)\s*\S*$/;
+export const PIPE_COMMA_AFTER_FIELDS = /^\s*fields\s+((\+|\-)\s+)?\S+\s*(,\s*\S+\s*)*\s+$/;
+export const FIELD_IN_FIELDS_LOOP = /^\s*fields\s+((\+|\-)\s+)?\S+\s*(,\s*\S+\s*)*,\s*\S*$/;
+
+// Regex for rare/top command
+export const PIPE_COMMA_BY_AFTER_FIELD = /^\s*(rare|top(\s+\d+)?)\s+\S+\s*(,\s*\S+\s*)*\s+\S*$/;
+export const RARE_TOP_FIELD_LOOP = /^\s*(rare|top(\s+\d+)?)\s+\S+\s*(,\s*\S+\s*)*,\s*\S*$/;
+export const FIELD_AFTER_BY = /^\s*(rare|top(\s+\d+)?)\s+\S+\s*(,\s*\S+\s*)*\s+by\s+\S*$/;
+export const PIPE_AFTER_GROUP_BY = /^\s*(rare|top(\s+\d+)?)\s+\S+\s*(,\s*\S+\s*)*\s+by\s+\S+\s+$/;
+
+// Regex for rename command
+export const AS_AFTER_FIELD = /^\s*rename\s+((,\s*)?\S+\s+as\s+\S+\s*)*\s*(,\s*)?\S+\s+\S*$/;
+export const PIPE_COMMA_AFTER_RENAME_FIELD = /^\s*rename\s+((,\s*)?\S+\s+as\s+\S+\s*)+$/;
+export const FIELD_AFTER_COMMA = /^\s*rename\s+((,\s*)?\S+\s+as\s+\S+\s*)+\s*,\s+\S*$/;
+
+// Regex for head command
+export const PIPE_AFTER_HEAD = /^\s*head\s+\d+\s+/;
+
+// Regex for sort command
+export const PLUS_MINUS_FIELD_AFTER_SORT = /^\s*sort(\s+\d+)?\s+\S*$/;
+export const FIELD_AFTER_PLUS_MINUS_SORT = /^\s*sort(\s+\d+)?((,\s*)?\s+(\+|\-)?\s*\S+\s*)*\s+(\+|\-)\s*\S*$/;
+export const PIPE_COMMA_AFTER_SORT_FIELD = /^\s*sort(\s+\d+)?((,\s*)?\s+(\+|\-)?\s*\S+\s*)*\s+\S+\s+$/;
+export const PLUS_MINUS_FIELD_IN_FIELDS_LOOP = /^\s*sort(\s+\d+)?((,\s*)?\s+(\+|\-)?\s*\S+\s*)*,\s+\S*$/;
+
+// Regex for stats command
+export const FIELD_AFTER_STATS_GROUP_BY = /^\s*stats\s+((,\s*)?((sum|avg|max|min|var_samp|var_pop|stddev_samp|stddev_pop)\(\s*\S+\s*\)\s*)|((,\s*)?count\(\)\s*))+\s+by\s+\S*$/;
+export const FIELD_AFTER_AGGREGATION = /^\s*stats\s+((,\s*)?((sum|avg|max|min|var_samp|var_pop|stddev_samp|stddev_pop)\(\s*\S+\s*\)\s*)|((,\s*)?count\(\)\s*))*(,\s*)?(sum|avg|max|min|var_samp|var_pop|stddev_samp|stddev_pop)\(\s*\S*$/;
+export const CLOSE_AFTER_FIELD = /^\s*stats\s+((,\s*)?((sum|avg|max|min|var_samp|var_pop|stddev_samp|stddev_pop)\(\s*\S+\s*\)\s*)|((,\s*)?count\(\)\s*))*(,\s*)?(sum|avg|max|min|var_samp|var_pop|stddev_samp|stddev_pop)\(\s*\S+\s+$/;
+export const PIPE_COMMA_BY_AFTER_AGGREGATION = /^\s*stats\s+((,\s*)?((sum|avg|max|min|var_samp|var_pop|stddev_samp|stddev_pop)\(\s*\S+\s*\)\s*)|((,\s*)?count\(\)\s*))+\s+\S*$/;
+export const PIPE_AFTER_STATS_GROUP_BY = /^\s*stats\s+((,\s*)?((sum|avg|max|min|var_samp|var_pop|stddev_samp|stddev_pop)\(\s*\S+\s*\)\s*)|((,\s*)?count\(\)\s*))+\s+by\s+\S+\s+$/;
+export const AGGREGATION_FOR_STATS= /^\s*stats\s+(((,\s*)?((sum|avg|max|min|var_samp|var_pop|stddev_samp|stddev_pop)\(\s*\S+\s*\)\s*)|((,\s*)?count\(\)\s*))+\s+,\s*)?\S*$/;
 
 export const regexForSuggestion = [
-  PIPE_AFTER_FIELD,
-  FIELD_AFTER_DEDUP,
-  PIPE_AFTER_MATCH,
-  CLOSE_AFTER_DATA,
-  DATA_AFTER_COMMA,
-  COMMA_AFTER_FIELD,
-  FIELD_AFTER_MATCH,
-  PIPE_AFTER_WHERE,
-  DATA_AFTER_EQUAL,
-  EQUAL_AFTER_FIELD,
-  MATCH_FIELD_AFTER_WHERE,
   EMPTY_REGEX,
+  FIELD_AFTER_COMMAND,
+  MATCH_FIELD_AFTER_WHERE,
+  EQUAL_AFTER_WHERE_FIELD,
+  DATA_AFTER_WHERE_EQUAL,
+  PIPE_AFTER_WHERE,
+  COMMA_AFTER_FIELD,
+  DATA_AFTER_COMMA,
+  CLOSE_AFTER_DATA,
+  PIPE_AFTER_MATCH,
+  FIELD_IN_FIELD_LOOP,
+  PIPE_COMMA_AFTER_FIELD,
+  PIPE_AFTER_KEEP_EMPTY,
+  PIPE_AFTER_CONSECUTIVE,
+  EQUAL_AFTER_EVAL_FIELD,
+  FIELD_AFTER_EVAL_EQUAL,
+  MATH_AFTER_FIELD,
+  PIPE_MATH_AFTER_EXPRESSIONS,
+  PLUS_MINUS_FIELD_AFTER_FIELDS,
+  FIELD_AFTER_PLUS_MINUS,
+  PIPE_COMMA_AFTER_FIELDS,
+  FIELD_IN_FIELDS_LOOP,
+  PIPE_COMMA_BY_AFTER_FIELD,
+  RARE_TOP_FIELD_LOOP,
+  FIELD_AFTER_BY,
+  PIPE_AFTER_GROUP_BY,
+  PIPE_COMMA_AFTER_RENAME_FIELD,
+  FIELD_AFTER_COMMA,
+  AS_AFTER_FIELD,
+  PIPE_AFTER_HEAD,
+  PLUS_MINUS_FIELD_AFTER_SORT,
+  FIELD_AFTER_PLUS_MINUS_SORT,
+  PLUS_MINUS_FIELD_IN_FIELDS_LOOP,
+  PIPE_COMMA_AFTER_SORT_FIELD,
+  FIELD_AFTER_STATS_GROUP_BY,
+  FIELD_AFTER_AGGREGATION,
+  CLOSE_AFTER_FIELD,
+  PIPE_COMMA_BY_AFTER_AGGREGATION,
+  PIPE_AFTER_STATS_GROUP_BY,
+  AGGREGATION_FOR_STATS,
 ];
 
-export const regexForIndex = [JUST_SOURCE_REGEX, SOURCE_WHERE_REGEX, SOURCE_MATCH_REGEX];
+export const regexForIndex = [
+  JUST_SOURCE_REGEX,
+  SOURCE_WHERE_REGEX,
+  SOURCE_MATCH_REGEX,
+  JUST_SEARCH_REGEX,
+  SEARCH_WHERE_REGEX,
+  SEARCH_MATCH_REGEX,
+];
diff --git a/dashboards-observability/common/constants/explorer.ts b/dashboards-observability/common/constants/explorer.ts
@@ -5,6 +5,8 @@
 
 export const EVENT_ANALYTICS_DOCUMENTATION_URL =
   'https://opensearch.org/docs/latest/observability/event-analytics/';
+export const OPEN_TELEMETRY_LOG_CORRELATION_LINK =
+  'https://opentelemetry.io/docs/reference/specification/logs/overview/#log-correlation';
 export const RAW_QUERY = 'rawQuery';
 export const FINAL_QUERY = 'finalQuery';
 export const SELECTED_DATE_RANGE = 'selectedDateRange';
@@ -32,7 +34,8 @@ export const TAB_CREATED_TYPE = 'tabCreatedType';
 export const NEW_TAB = 'newTab';
 export const REDIRECT_TAB = 'redirect_tab';
 export const PAGE_SIZE = 50;
-
+export const DEFAULT_COLUMNS = ['', 'Time', '_source'];
+export const OTEL_TRACE_ID = 'traceId';
 export const DATE_PICKER_FORMAT = 'YYYY-MM-DD HH:mm:ss';
 export const TIME_INTERVAL_OPTIONS = [
   {

diff --git a/dashboards-observability/common/types/explorer.ts b/dashboards-observability/common/types/explorer.ts
@@ -92,6 +92,7 @@ export interface IExplorerProps {
   history: History;
   notifications: NotificationsStart;
   savedObjectId: string;
+  curSelectedTabId : React.MutableRefObject<undefined>;
   setToast: (
     title: string,
     color?: string,

diff --git a/dashboards-observability/common/utils/query_utils.ts b/dashboards-observability/common/utils/query_utils.ts
@@ -3,57 +3,56 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { isEmpty } from 'lodash';
-import datemath from '@elastic/datemath';
-import { DATE_PICKER_FORMAT } from '../../common/constants/explorer';
+import datemath from "@elastic/datemath";
+import { isEmpty } from "lodash";
+import { DATE_PICKER_FORMAT } from "../../common/constants/explorer";
 import {
-  PPL_INDEX_REGEX,
   PPL_INDEX_INSERT_POINT_REGEX,
-  PPL_NEWLINE_REGEX
-} from '../../common/constants/shared';
+  PPL_INDEX_REGEX,
+  PPL_NEWLINE_REGEX,
+} from "../../common/constants/shared";
 
-export const getIndexPatternFromRawQuery = (query: string) : string => {
+export const getIndexPatternFromRawQuery = (query: string): string => {
   const matches = query.match(PPL_INDEX_REGEX);
   if (matches) {
     return matches[2];
   }
-  return '';
+  return "";
 };
 
-const commandExists = (query: string, command: string): boolean => {
-  return new RegExp(`\\|\\s*${command}\\b`).test(query);
-}
-
 // insert time filter command and additional commands based on raw query
 export const preprocessQuery = ({
   rawQuery,
   startTime,
   endTime,
   timeField,
+  isLiveQuery,
 }: {
   rawQuery: string;
   startTime: string;
   endTime: string;
   timeField?: string;
+  isLiveQuery: boolean;
 }) => {
-
-  let finalQuery = '';
+  let finalQuery = "";
 
   if (isEmpty(rawQuery)) return finalQuery;
-    
+
   // convert to moment
   const start = datemath.parse(startTime)?.format(DATE_PICKER_FORMAT);
   const end = datemath.parse(endTime)?.format(DATE_PICKER_FORMAT);
   const tokens = rawQuery.replaceAll(PPL_NEWLINE_REGEX, '').match(PPL_INDEX_INSERT_POINT_REGEX);
 
   if (isEmpty(tokens)) return finalQuery;
 
-  let conditions = `| where ${timeField} >= '${start}' and ${timeField} <= '${end}'`;
-  if (commandExists(rawQuery, 'parse')) {
-    conditions += ` | sort - ${timeField} | head 10000`;
+  finalQuery = `${tokens![1]}=${
+    tokens![2]
+  } | where ${timeField} >= '${start}' and ${timeField} <= '${end}'${
+    tokens![3]
+  }`;
+  if (isLiveQuery){
+    finalQuery = finalQuery + ` | sort - ${timeField}`;
   }
-
-  finalQuery = `${tokens![1]}=${tokens![2]} ${conditions} ${tokens![3]}`;
-
   return finalQuery;
-};
+}
+
diff --git a/.../public/components/application_analytics/__tests__/__snapshots__/log_config.test.tsx.snap b/.../public/components/application_analytics/__tests__/__snapshots__/log_config.test.tsx.snap
@@ -333,6 +333,7 @@ exports[`Log Config component renders empty log config 1`] = `
                                   }
                                 >
                                   <Autocomplete
+                                    baseQuery=""
                                     dslService={
                                       Object {
                                         "fetch": [MockFunction],
@@ -348,6 +349,7 @@ exports[`Log Config component renders empty log config 1`] = `
                                     key="autocomplete-bar"
                                     onItemSelect={[Function]}
                                     query=""
+                                    tabId="application-analytics-tab"
                                     tempQuery=""
                                   >
                                     <div
@@ -803,6 +805,7 @@ exports[`Log Config component renders with query 1`] = `
                                   }
                                 >
                                   <Autocomplete
+                                    baseQuery=""
                                     dslService={
                                       Object {
                                         "fetch": [MockFunction],
@@ -818,6 +821,7 @@ exports[`Log Config component renders with query 1`] = `
                                     key="autocomplete-bar"
                                     onItemSelect={[Function]}
                                     query="source = openserach_dashboard_sample_logs"
+                                    tabId="application-analytics-tab"
                                     tempQuery=""
                                   >
                                     <div