Fix bug to query header search attributes correctly in visibility

common/definition/indexedKeys.go

@@ -58,7 +58,7 @@ const (
 	// Attr is prefix of custom search attributes
 	Attr = "Attr"
 	// HeaderFormat is the format of context headers in search attributes
-	HeaderFormat = "Header.%s"
+	HeaderFormat = "Header_%s"
 )
 
 // defaultIndexedKeys defines all searchable keys

common/elasticsearch/validator/queryValidator_test.go

@@ -27,6 +27,7 @@ import (
 
 	"github.com/uber/cadence/common/definition"
 	"github.com/uber/cadence/common/dynamicconfig"
+	"github.com/uber/cadence/common/types"
 )
 
 func TestValidateQuery(t *testing.T) {
@@ -35,6 +36,7 @@ func TestValidateQuery(t *testing.T) {
 		query     string
 		validated string
 		err       string
+		dcValid   map[string]interface{}
 	}{
 		{
 			msg:       "empty query",
@@ -126,11 +128,54 @@ func TestValidateQuery(t *testing.T) {
 			query: "WorkflowID = 'wid' union select * from dummy",
 			err:   "Invalid select query.",
 		},
+		{
+			msg:       "valid custom search attribute",
+			query:     "CustomStringField = 'value'",
+			validated: "`Attr.CustomStringField` = 'value'",
+		},
+		{
+			msg:       "custom search attribute can contain underscore",
+			query:     "Custom_Field = 'value'",
+			validated: "`Attr.Custom_Field` = 'value'",
+			dcValid: map[string]interface{}{
+				"Custom_Field": types.IndexedValueTypeString,
+			},
+		},
+		{
+			msg:       "custom search attribute can contain number",
+			query:     "Custom_0 = 'value'",
+			validated: "`Attr.Custom_0` = 'value'",
+			dcValid: map[string]interface{}{
+				"Custom_0": types.IndexedValueTypeString,
+			},
+		},
+		{
+			msg:   "customg search attribute cannot contain dot",
+			query: "Custom.Field = 'value'",
+			err:   "invalid search attribute \"Field\"",
+			dcValid: map[string]interface{}{
+				"Custom.Field": types.IndexedValueTypeString,
+			},
+		},
+		{
+			msg:   "custom search attribute cannot contain dash",
+			query: "Custom-Field = 'value'",
+			err:   "invalid comparison expression",
+			dcValid: map[string]interface{}{
+				"Custom-Field": types.IndexedValueTypeString,
+			},
+		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.msg, func(t *testing.T) {
-			validSearchAttr := dynamicconfig.GetMapPropertyFn(definition.GetDefaultIndexedKeys())
+			validSearchAttr := func(opts ...dynamicconfig.FilterOption) map[string]interface{} {
+				valid := definition.GetDefaultIndexedKeys()
+				for k, v := range tt.dcValid {
+					valid[k] = v
+				}
+				return valid
+			}
 			validateSearchAttr := dynamicconfig.GetBoolPropertyFn(true)
 			qv := NewQueryValidator(validSearchAttr, validateSearchAttr)
 			validated, err := qv.ValidateQuery(tt.query)

service/history/task/transfer_active_task_executor_test.go

@@ -1543,7 +1543,7 @@ func (s *transferActiveTaskExecutorSuite) TestProcessRecordWorkflowStartedTaskWi
 	s.mockShard.GetConfig().EnableContextHeaderInVisibility = func(domain string) bool { return true }
 	s.mockShard.GetConfig().ValidSearchAttributes = func(opts ...dc.FilterOption) map[string]interface{} {
 		return map[string]interface{}{
-			"Header.contextKey": struct{}{},
+			"Header_context_key": struct{}{},
 		}
 	}
 
@@ -1626,7 +1626,7 @@ func (s *transferActiveTaskExecutorSuite) TestProcessUpsertWorkflowSearchAttribu
 	s.mockShard.GetConfig().EnableContextHeaderInVisibility = func(domain string) bool { return true }
 	s.mockShard.GetConfig().ValidSearchAttributes = func(opts ...dc.FilterOption) map[string]interface{} {
 		return map[string]interface{}{
-			"Header.contextKey": struct{}{},
+			"Header_context_key": struct{}{},
 		}
 	}
 
@@ -1782,7 +1782,7 @@ func createRecordWorkflowExecutionStartedRequest(
 			t.Fatal(err)
 		}
 		searchAttributes = map[string][]byte{
-			"Header.contextKey": contextValueJSONString,
+			"Header_context_key": contextValueJSONString,
 		}
 	}
 	return &persistence.RecordWorkflowExecutionStartedRequest{
@@ -1937,7 +1937,7 @@ func createUpsertWorkflowSearchAttributesRequest(
 			t.Fatal(err)
 		}
 		searchAttributes = map[string][]byte{
-			"Header.contextKey": contextValueJSONString,
+			"Header_context_key": contextValueJSONString,
 		}
 	}
 

service/history/task/transfer_standby_task_executor_test.go

@@ -815,7 +815,7 @@ func (s *transferStandbyTaskExecutorSuite) TestProcessRecordWorkflowStartedTaskW
 	}
 	s.mockShard.GetConfig().ValidSearchAttributes = func(opts ...dc.FilterOption) map[string]interface{} {
 		return map[string]interface{}{
-			"Header.contextKey": struct{}{},
+			"Header_context_key": struct{}{},
 		}
 	}
 
@@ -906,7 +906,7 @@ func (s *transferStandbyTaskExecutorSuite) TestProcessUpsertWorkflowSearchAttrib
 	}
 	s.mockShard.GetConfig().ValidSearchAttributes = func(opts ...dc.FilterOption) map[string]interface{} {
 		return map[string]interface{}{
-			"Header.contextKey": struct{}{},
+			"Header_context_key": struct{}{},
 		}
 	}
 

service/history/task/transfer_task_executor_base.go

@@ -24,6 +24,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"regexp"
 	"time"
 
 	"github.com/uber/cadence/client/matching"
@@ -50,6 +51,8 @@ const (
 	defaultDomainName              = "defaultDomainName"
 )
 
+var nonAlphanumericRegex = regexp.MustCompile(`[^a-zA-Z0-9]+`)
+
 type (
 	transferTaskExecutorBase struct {
 		shard          shard.Context
@@ -402,7 +405,7 @@ func getWorkflowMemo(
 
 func appendContextHeaderToSearchAttributes(attr, context map[string][]byte, allowedKeys map[string]interface{}) (map[string][]byte, error) {
 	for k, v := range context {
-		key := fmt.Sprintf(definition.HeaderFormat, k)
+		key := sanitizedHeaderKey(k)
 		if _, ok := attr[key]; ok { // skip if key already exists
 			continue
 		}
@@ -443,3 +446,8 @@ func isWorkflowNotExistError(err error) bool {
 	_, ok := err.(*types.EntityNotExistsError)
 	return ok
 }
+
+func sanitizedHeaderKey(key string) string {
+	sanitizedKey := nonAlphanumericRegex.ReplaceAllString(key, "_")
+	return fmt.Sprintf(definition.HeaderFormat, sanitizedKey)
+}

service/history/testing/workflow_util.go

@@ -65,8 +65,8 @@ func StartWorkflow(
 				ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(2),
 				TaskStartToCloseTimeoutSeconds:      common.Int32Ptr(1),
 				Header: &types.Header{Fields: map[string][]byte{
-					"contextKey":        []byte("contextValue"),
-					"invalidContextKey": []byte("invalidContextValue"),
+					"context-key":        []byte("contextValue"),
+					"invalid-context-key": []byte("invalidContextValue"),
 				}},
 			},
 			PartitionConfig: map[string]string{"userid": uuid.New()},

tools/cli/admin_cluster_commands.go

@@ -36,7 +36,7 @@ import (
 
 // An indirection for the prompt function so that it can be mocked in the unit tests
 var promptFn = prompt
-var validSearchAttributeKey = regexp.MustCompile(`^[a-zA-Z][a-zA-Z_.-0-9]*$`)
+var validSearchAttributeKey = regexp.MustCompile(`^[a-zA-Z][a-zA-Z_0-9]*$`)
 
 // AdminAddSearchAttribute to whitelist search attribute
 func AdminAddSearchAttribute(c *cli.Context) {

tools/cli/admin_cluster_commands_test.go

@@ -126,8 +126,8 @@ func TestValidSearchAttributeKey(t *testing.T) {
 	assert.NoError(t, validateSearchAttributeKey("cityId"))
 	assert.NoError(t, validateSearchAttributeKey("paymentProfileUUID"))
 	assert.NoError(t, validateSearchAttributeKey("job_type"))
-	assert.NoError(t, validateSearchAttributeKey("Header.ctx-tenancy"))
 
+	assert.Error(t, validateSearchAttributeKey("payments-biling-invoices-TransactionUUID"))
 	assert.Error(t, validateSearchAttributeKey("9lives"))
 	assert.Error(t, validateSearchAttributeKey("tax%"))
 }