Skip to content

Commit

Permalink
Merge branch 'develop' into revert-4120-fix/issue-4109
Browse files Browse the repository at this point in the history
  • Loading branch information
@makeavish
makeavish authored Dec 21, 2023
2 parents a814728 + 43d5ee9 commit f5f3e07
Show file tree
Hide file tree
Showing 9 changed files with 536 additions and 138 deletions.
14 changes: 8 additions & 6 deletions frontend/src/container/LogDetailedView/TableView.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import { ILog } from 'types/api/logs/log';
import ActionItem, { ActionItemProps } from './ActionItem';
import FieldRenderer from './FieldRenderer';
import {
filterKeyForField,
flattenObject,
jsonToDataNodes,
recursiveParseJSON,
Expand Down Expand Up @@ -98,11 +99,12 @@ function TableView({
title: 'Action',
width: 11,
render: (fieldData: Record<string, string>): JSX.Element | null => {
const fieldKey = fieldData.field.split('.').slice(-1);
if (!RESTRICTED_FIELDS.includes(fieldKey[0])) {
const fieldFilterKey = filterKeyForField(fieldData.field);

if (!RESTRICTED_FIELDS.includes(fieldFilterKey)) {
return (
<ActionItem
fieldKey={fieldKey[0]}
fieldKey={fieldFilterKey}
fieldValue={fieldData.value}
onClickActionItem={onClickActionItem}
/>
Expand All @@ -119,7 +121,6 @@ function TableView({
align: 'left',
ellipsis: true,
render: (field: string, record): JSX.Element => {
const fieldKey = field.split('.').slice(-1);
const renderedField = <FieldRenderer field={field} />;

if (record.field === 'trace_id') {
Expand Down Expand Up @@ -148,10 +149,11 @@ function TableView({
);
}

if (!RESTRICTED_FIELDS.includes(fieldKey[0])) {
const fieldFilterKey = filterKeyForField(field);
if (!RESTRICTED_FIELDS.includes(fieldFilterKey)) {
return (
<AddToQueryHOC
fieldKey={fieldKey[0]}
fieldKey={fieldFilterKey}
fieldValue={flattenLogData[field]}
onAddToQuery={onAddToQuery}
>
Expand Down
26 changes: 24 additions & 2 deletions frontend/src/container/LogDetailedView/utils.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,16 @@ export const generateFieldKeyForArray = (
export const removeObjectFromString = (str: string): string =>
str.replace(/\[object Object\]./g, '');

// Split `str` on the first occurrence of `delimiter`
// For example, will return `['a', 'b.c']` when splitting `'a.b.c'` at dots
const splitOnce = (str: string, delimiter: string): string[] => {
const parts = str.split(delimiter);
if (parts.length < 2) {
return parts;
}
return [parts[0], parts.slice(1).join(delimiter)];
};

export const getFieldAttributes = (field: string): IFieldAttributes => {
let dataType;
let newField;
Expand All @@ -140,18 +150,30 @@ export const getFieldAttributes = (field: string): IFieldAttributes => {
if (field.startsWith('attributes_')) {
logType = MetricsType.Tag;
const stringWithoutPrefix = field.slice('attributes_'.length);
const parts = stringWithoutPrefix.split('.');
const parts = splitOnce(stringWithoutPrefix, '.');
[dataType, newField] = parts;
} else if (field.startsWith('resources_')) {
logType = MetricsType.Resource;
const stringWithoutPrefix = field.slice('resources_'.length);
const parts = stringWithoutPrefix.split('.');
const parts = splitOnce(stringWithoutPrefix, '.');
[dataType, newField] = parts;
}

return { dataType, newField, logType };
};

// Returns key to be used when filtering for `field` via
// the query builder. This is useful for powering filtering
// by field values from log details view.
export const filterKeyForField = (field: string): string => {
// Must work for all 3 of the following types of cases
// timestamp -> timestamp
// attributes_string.log.file -> log.file
// resources_string.k8s.pod.name -> k8s.pod.name
const fieldAttribs = getFieldAttributes(field);
return fieldAttribs?.newField || field;
};

export const aggregateAttributesResourcesToString = (logData: ILog): string => {
const outputJson: ILogAggregateAttributesResources = {
body: logData.body,
Expand Down
156 changes: 107 additions & 49 deletions pkg/query-service/app/clickhouseReader/reader.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3423,6 +3423,26 @@ func (r *ClickHouseReader) GetTagsInfoInLastHeartBeatInterval(ctx context.Contex
return &tagsInfo, nil
}

// remove this after sometime
func removeUnderscoreDuplicateFields(fields []model.LogField) []model.LogField {
lookup := map[string]model.LogField{}
for _, v := range fields {
lookup[v.Name+v.DataType] = v
}

for k := range lookup {
if strings.Contains(k, ".") {
delete(lookup, strings.ReplaceAll(k, ".", "_"))
}
}

updatedFields := []model.LogField{}
for _, v := range lookup {
updatedFields = append(updatedFields, v)
}
return updatedFields
}

// GetDashboardsInfo returns analytics data for dashboards
func (r *ClickHouseReader) GetDashboardsInfo(ctx context.Context) (*model.DashboardsInfo, error) {
dashboardsInfo := model.DashboardsInfo{}
Expand Down Expand Up @@ -3540,6 +3560,10 @@ func (r *ClickHouseReader) GetLogFields(ctx context.Context) (*model.GetFieldsRe
return nil, &model.ApiError{Err: err, Typ: model.ErrorInternal}
}

//remove this code after sometime
attributes = removeUnderscoreDuplicateFields(attributes)
resources = removeUnderscoreDuplicateFields(resources)

statements := []model.ShowCreateTableStatement{}
query = fmt.Sprintf("SHOW CREATE TABLE %s.%s", r.logsDB, r.logsLocalTable)
err = r.db.Select(ctx, &statements, query)
Expand Down Expand Up @@ -3587,74 +3611,56 @@ func (r *ClickHouseReader) UpdateLogField(ctx context.Context, field *model.Upda
valueColName := fmt.Sprintf("%s_%s_value", field.Type, strings.ToLower(field.DataType))

// create materialized column
query := fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s ADD COLUMN IF NOT EXISTS %s %s MATERIALIZED %s[indexOf(%s, '%s')] CODEC(ZSTD(1))",
r.logsDB, r.logsLocalTable,
r.cluster,
colname, field.DataType,
valueColName,
keyColName,
field.Name,
)
err := r.db.Exec(ctx, query)
if err != nil {
return &model.ApiError{Err: err, Typ: model.ErrorInternal}
}

defaultValueDistributed := "-1"
if strings.ToLower(field.DataType) == "bool" {
defaultValueDistributed = "false"
field.IndexType = "set(2)"
}
query = fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s ADD COLUMN IF NOT EXISTS %s %s MATERIALIZED %s",
r.logsDB, r.logsTable,
r.cluster,
colname, field.DataType,
defaultValueDistributed,
)
err = r.db.Exec(ctx, query)
if err != nil {
return &model.ApiError{Err: err, Typ: model.ErrorInternal}
}
for _, table := range []string{r.logsLocalTable, r.logsTable} {
q := "ALTER TABLE %s.%s ON CLUSTER %s ADD COLUMN IF NOT EXISTS %s %s DEFAULT %s[indexOf(%s, '%s')] CODEC(ZSTD(1))"
query := fmt.Sprintf(q,
r.logsDB, table,
r.cluster,
colname, field.DataType,
valueColName,
keyColName,
field.Name,
)
err := r.db.Exec(ctx, query)
if err != nil {
return &model.ApiError{Err: err, Typ: model.ErrorInternal}
}

// create exists column
query = fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s ADD COLUMN IF NOT EXISTS %s_exists bool MATERIALIZED if(indexOf(%s, '%s') != 0, true, false) CODEC(ZSTD(1))",
r.logsDB, r.logsLocalTable,
r.cluster,
colname,
keyColName,
field.Name,
)
err = r.db.Exec(ctx, query)
if err != nil {
return &model.ApiError{Err: err, Typ: model.ErrorInternal}
query = fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s ADD COLUMN IF NOT EXISTS %s_exists bool DEFAULT if(indexOf(%s, '%s') != 0, true, false) CODEC(ZSTD(1))",
r.logsDB, table,
r.cluster,
colname,
keyColName,
field.Name,
)
err = r.db.Exec(ctx, query)
if err != nil {
return &model.ApiError{Err: err, Typ: model.ErrorInternal}
}
}

query = fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s ADD COLUMN IF NOT EXISTS %s_exists bool MATERIALIZED false",
r.logsDB, r.logsTable,
r.cluster,
colname,
)
err = r.db.Exec(ctx, query)
if err != nil {
return &model.ApiError{Err: err, Typ: model.ErrorInternal}
// create the index
if strings.ToLower(field.DataType) == "bool" {
// there is no point in creating index for bool attributes as the cardinality is just 2
return nil
}

// create the index
if field.IndexType == "" {
field.IndexType = constants.DefaultLogSkipIndexType
}
if field.IndexGranularity == 0 {
field.IndexGranularity = constants.DefaultLogSkipIndexGranularity
}
query = fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s ADD INDEX IF NOT EXISTS %s_idx (%s) TYPE %s GRANULARITY %d",
query := fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s ADD INDEX IF NOT EXISTS %s_idx (%s) TYPE %s GRANULARITY %d",
r.logsDB, r.logsLocalTable,
r.cluster,
colname,
colname,
field.IndexType,
field.IndexGranularity,
)
err = r.db.Exec(ctx, query)
err := r.db.Exec(ctx, query)
if err != nil {
return &model.ApiError{Err: err, Typ: model.ErrorInternal}
}
Expand Down Expand Up @@ -4571,12 +4577,64 @@ func (r *ClickHouseReader) GetListResultV3(ctx context.Context, query string) ([
row[columnNames[idx]] = v
}
}

// remove duplicate _ attributes for logs.
// remove this function after a month
removeDuplicateUnderscoreAttributes(row)

rowList = append(rowList, &v3.Row{Timestamp: t, Data: row})
}

return rowList, nil

}

func removeDuplicateUnderscoreAttributes(row map[string]interface{}) {
if val, ok := row["attributes_int64"]; ok {
attributes := val.(*map[string]int64)
for key := range *attributes {
if strings.Contains(key, ".") {
uKey := strings.ReplaceAll(key, ".", "_")
delete(*attributes, uKey)
}
}

}

if val, ok := row["attributes_float64"]; ok {
attributes := val.(*map[string]float64)
for key := range *attributes {
if strings.Contains(key, ".") {
uKey := strings.ReplaceAll(key, ".", "_")
delete(*attributes, uKey)
}
}

}

if val, ok := row["attributes_bool"]; ok {
attributes := val.(*map[string]bool)
for key := range *attributes {
if strings.Contains(key, ".") {
uKey := strings.ReplaceAll(key, ".", "_")
delete(*attributes, uKey)
}
}

}
for _, k := range []string{"attributes_string", "resources_string"} {
if val, ok := row[k]; ok {
attributes := val.(*map[string]string)
for key := range *attributes {
if strings.Contains(key, ".") {
uKey := strings.ReplaceAll(key, ".", "_")
delete(*attributes, uKey)
}
}

}
}
}
func (r *ClickHouseReader) CheckClickHouse(ctx context.Context) error {
rows, err := r.db.Query(ctx, "SELECT 1")
if err != nil {
Expand Down
62 changes: 62 additions & 0 deletions pkg/query-service/app/logs/v3/enrich_query_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,68 @@ var testEnrichParamsData = []struct {
},
},
},
{
Name: "Enriching query range v3 params with dot support",
Params: v3.QueryRangeParamsV3{
CompositeQuery: &v3.CompositeQuery{
BuilderQueries: map[string]*v3.BuilderQuery{
"test": {
QueryName: "test",
Expression: "test",
DataSource: v3.DataSourceLogs,
AggregateAttribute: v3.AttributeKey{
Key: "method.name",
},
Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "service.name"}, Value: "test", Operator: "="},
}},
GroupBy: []v3.AttributeKey{{Key: "host.name"}},
OrderBy: []v3.OrderBy{{ColumnName: "host.name"}},
},
},
},
},
Fields: map[string]v3.AttributeKey{
"method.name": {
Key: "method.name",
Type: v3.AttributeKeyTypeTag,
DataType: v3.AttributeKeyDataTypeString,
IsColumn: true,
},
"service.name": {
Key: "service.name",
Type: v3.AttributeKeyTypeTag,
DataType: v3.AttributeKeyDataTypeString,
},
"host.name": {
Key: "host.name",
Type: v3.AttributeKeyTypeTag,
DataType: v3.AttributeKeyDataTypeString,
},
},
Result: v3.QueryRangeParamsV3{
CompositeQuery: &v3.CompositeQuery{
BuilderQueries: map[string]*v3.BuilderQuery{
"test": {
QueryName: "test",
Expression: "test",
DataSource: v3.DataSourceLogs,
AggregateAttribute: v3.AttributeKey{
Key: "method.name",
Type: v3.AttributeKeyTypeTag,
DataType: v3.AttributeKeyDataTypeString,
IsColumn: true,
},
Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "service.name", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeString}, Value: "test", Operator: "="},
}},
GroupBy: []v3.AttributeKey{{Key: "host.name", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeString}},
OrderBy: []v3.OrderBy{{ColumnName: "host.name", Key: "host.name", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeString}},
},
},
},
},
},
}

func TestEnrichParams(t *testing.T) {
Expand Down
Loading

0 comments on commit f5f3e07

Please sign in to comment.