From 78e5cfd3787d12e453acdeae98b3307f6b86e8f3 Mon Sep 17 00:00:00 2001 From: Yaron Schneider Date: Fri, 11 Oct 2024 09:03:50 -0700 Subject: [PATCH] Add PII scrubbing to conversational api (#8191) * add pii scrubbing to conversational api Signed-off-by: yaron2 * linter Signed-off-by: yaron2 * change proto fields, add test Signed-off-by: yaron2 * fix test Signed-off-by: yaron2 --------- Signed-off-by: yaron2 --- dapr/proto/runtime/v1/dapr.proto | 14 +- go.mod | 2 + go.sum | 4 + pkg/api/universal/conversation.go | 49 +++++- pkg/proto/runtime/v1/dapr.pb.go | 160 +++++++++-------- .../suite/daprd/conversation/grpc/scrubpii.go | 166 ++++++++++++++++++ .../suite/daprd/conversation/http/scrubpii.go | 135 ++++++++++++++ 7 files changed, 449 insertions(+), 81 deletions(-) create mode 100644 tests/integration/suite/daprd/conversation/grpc/scrubpii.go create mode 100644 tests/integration/suite/daprd/conversation/http/scrubpii.go diff --git a/dapr/proto/runtime/v1/dapr.proto b/dapr/proto/runtime/v1/dapr.proto index 6d7ab51462c..d721cd0efc8 100644 --- a/dapr/proto/runtime/v1/dapr.proto +++ b/dapr/proto/runtime/v1/dapr.proto @@ -1284,8 +1284,8 @@ message ConversationAlpha1Request { // The name of Coverstaion component string name = 1; - // Conversation context - the Id of an existing chat room (like in ChatGPT) - optional string conversationContext = 2; + // The ID of an existing chat (like in ChatGPT) + optional string contextID = 2; // Inputs for the conversation, support multiple input in one time. repeated ConversationInput inputs = 3; @@ -1295,6 +1295,9 @@ message ConversationAlpha1Request { // The metadata passing to conversation components. map metadata = 5; + + // Scrub PII data that comes back from the LLM + optional bool scrubPII = 6; } message ConversationInput { @@ -1303,6 +1306,9 @@ message ConversationInput { // The role to set for the message optional string role = 2; + + // Scrub PII data that goes into the LLM + optional bool scrubPII = 3; } // ConversationAlpha1Result is the result for one input. @@ -1315,8 +1321,8 @@ message ConversationAlpha1Result { // ConversationAlpha1Response is the response for Conversation. message ConversationAlpha1Response { - // Conversation context - the Id of an existing chat room (like in ChatGPT) - optional string conversationContext = 1; + // The ID of an existing chat (like in ChatGPT) + optional string contextID = 1; // An array of results. repeated ConversationAlpha1Result outputs = 2; diff --git a/go.mod b/go.mod index d99ee92de11..5d74edc8cc2 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( contrib.go.opencensus.io/exporter/prometheus v0.4.2 github.com/PaesslerAG/jsonpath v0.1.1 github.com/PuerkitoBio/purell v1.2.1 + github.com/aavaz-ai/pii-scrubber v0.0.0-20220812094047-3fa450ab6973 github.com/alphadose/haxmap v1.4.0 github.com/argoproj/argo-rollouts v1.4.1 github.com/cenkalti/backoff/v4 v4.3.0 @@ -148,6 +149,7 @@ require ( github.com/aliyun/aliyun-tablestore-go-sdk v1.7.10 // indirect github.com/aliyun/credentials-go v1.1.2 // indirect github.com/andybalholm/brotli v1.0.6 // indirect + github.com/anshal21/go-worker v1.1.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/apache/dubbo-getty v1.4.9-0.20220610060150-8af010f3f3dc // indirect github.com/apache/dubbo-go-hessian2 v1.11.5 // indirect diff --git a/go.sum b/go.sum index 53a9f64c877..d1a661086ef 100644 --- a/go.sum +++ b/go.sum @@ -148,6 +148,8 @@ github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/ github.com/Workiva/go-datastructures v1.0.52/go.mod h1:Z+F2Rca0qCsVYDS8z7bAGm8f3UkzuWYS/oBZz5a7VVA= github.com/Workiva/go-datastructures v1.0.53 h1:J6Y/52yX10Xc5JjXmGtWoSSxs3mZnGSaq37xZZh7Yig= github.com/Workiva/go-datastructures v1.0.53/go.mod h1:1yZL+zfsztete+ePzZz/Zb1/t5BnDuE2Ya2MMGhzP6A= +github.com/aavaz-ai/pii-scrubber v0.0.0-20220812094047-3fa450ab6973 h1:oMQQEYwu9V0O+IcRs7ButCPjsbOS0eEtMOImkvaeWLA= +github.com/aavaz-ai/pii-scrubber v0.0.0-20220812094047-3fa450ab6973/go.mod h1:Y5HcXSyXAlR6rvJ5kqvPEtQlRLhekJFI/w/9CRZkpHs= github.com/aerospike/aerospike-client-go/v6 v6.12.0 h1:4RKpcUlfINyIsjyOK8DBKn6NaBUl5UaHrWVY07/R99Q= github.com/aerospike/aerospike-client-go/v6 v6.12.0/go.mod h1:sBqeA3mli2vT5JInbp+XGFbajxEFg4kANUHsOl3meUk= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5 h1:rFw4nCn9iMW+Vajsk51NtYIcwSTkXr+JGrMd36kTDJw= @@ -215,6 +217,8 @@ github.com/alphadose/haxmap v1.4.0 h1:1yn+oGzy2THJj1DMuJBzRanE3sMnDAjJVbU0L31Jp3 github.com/alphadose/haxmap v1.4.0/go.mod h1:rjHw1IAqbxm0S3U5tD16GoKsiAd8FWx5BJ2IYqXwgmM= github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI= github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/anshal21/go-worker v1.1.0 h1:TPt2jBN/6dmPDPDTq8DHA0MtoXG8RWKGoJVHqED+s5g= +github.com/anshal21/go-worker v1.1.0/go.mod h1:6GiLOIr/VvVg80vfW65ytLuouSvndU2IoJTu+8M47lI= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= diff --git a/pkg/api/universal/conversation.go b/pkg/api/universal/conversation.go index d642f216d5e..e1a462db946 100644 --- a/pkg/api/universal/conversation.go +++ b/pkg/api/universal/conversation.go @@ -16,6 +16,8 @@ package universal import ( "context" + piiscrubber "github.com/aavaz-ai/pii-scrubber" + "github.com/dapr/components-contrib/conversation" "github.com/dapr/dapr/pkg/messages" runtimev1pb "github.com/dapr/dapr/pkg/proto/runtime/v1" @@ -51,9 +53,29 @@ func (a *Universal) ConverseAlpha1(ctx context.Context, req *runtimev1pb.Convers return nil, err } + scrubber, err := piiscrubber.NewDefaultScrubber() + if err != nil { + err := messages.ErrConversationMissingInputs.WithFormat(req.GetName()) + a.logger.Debug(err) + return &runtimev1pb.ConversationAlpha1Response{}, err + } + for _, i := range req.GetInputs() { + msg := i.GetMessage() + + if i.GetScrubPII() { + scrubbed, sErr := scrubber.ScrubTexts([]string{i.GetMessage()}) + if sErr != nil { + sErr = messages.ErrConversationInvoke.WithFormat(req.GetName(), sErr.Error()) + a.logger.Debug(sErr) + return &runtimev1pb.ConversationAlpha1Response{}, sErr + } + + msg = scrubbed[0] + } + c := conversation.ConversationInput{ - Message: i.GetMessage(), + Message: msg, Role: conversation.Role(i.GetRole()), } @@ -61,7 +83,7 @@ func (a *Universal) ConverseAlpha1(ctx context.Context, req *runtimev1pb.Convers } request.Parameters = req.GetParameters() - request.ConversationContext = req.GetConversationContext() + request.ConversationContext = req.GetContextID() // do call policyRunner := resiliency.NewRunner[*conversation.ConversationResponse](ctx, @@ -75,7 +97,7 @@ func (a *Universal) ConverseAlpha1(ctx context.Context, req *runtimev1pb.Convers if err != nil { err = messages.ErrConversationInvoke.WithFormat(req.GetName(), err.Error()) a.logger.Debug(err) - return nil, err + return &runtimev1pb.ConversationAlpha1Response{}, err } // handle response @@ -83,13 +105,26 @@ func (a *Universal) ConverseAlpha1(ctx context.Context, req *runtimev1pb.Convers a.logger.Debug(response) if resp != nil { if resp.ConversationContext != "" { - response.ConversationContext = &resp.ConversationContext + response.ContextID = &resp.ConversationContext } - for i := range resp.Outputs { + for _, o := range resp.Outputs { + res := o.Result + + if req.GetScrubPII() { + scrubbed, sErr := scrubber.ScrubTexts([]string{o.Result}) + if sErr != nil { + sErr = messages.ErrConversationInvoke.WithFormat(req.GetName(), sErr.Error()) + a.logger.Debug(sErr) + return &runtimev1pb.ConversationAlpha1Response{}, sErr + } + + res = scrubbed[0] + } + response.Outputs = append(response.GetOutputs(), &runtimev1pb.ConversationAlpha1Result{ - Result: resp.Outputs[i].Result, - Parameters: resp.Outputs[i].Parameters, + Result: res, + Parameters: o.Parameters, }) } } diff --git a/pkg/proto/runtime/v1/dapr.pb.go b/pkg/proto/runtime/v1/dapr.pb.go index 83e29b5fbd2..f6787143534 100644 --- a/pkg/proto/runtime/v1/dapr.pb.go +++ b/pkg/proto/runtime/v1/dapr.pb.go @@ -6849,14 +6849,16 @@ type ConversationAlpha1Request struct { // The name of Coverstaion component Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - // Conversation context - the Id of an existing chat room (like in ChatGPT) - ConversationContext *string `protobuf:"bytes,2,opt,name=conversationContext,proto3,oneof" json:"conversationContext,omitempty"` + // The ID of an existing chat (like in ChatGPT) + ContextID *string `protobuf:"bytes,2,opt,name=contextID,proto3,oneof" json:"contextID,omitempty"` // Inputs for the conversation, support multiple input in one time. Inputs []*ConversationInput `protobuf:"bytes,3,rep,name=inputs,proto3" json:"inputs,omitempty"` // Parameters for all custom fields. Parameters map[string]*anypb.Any `protobuf:"bytes,4,rep,name=parameters,proto3" json:"parameters,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` // The metadata passing to conversation components. Metadata map[string]string `protobuf:"bytes,5,rep,name=metadata,proto3" json:"metadata,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // Scrub PII data that comes back from the LLM + ScrubPII *bool `protobuf:"varint,6,opt,name=scrubPII,proto3,oneof" json:"scrubPII,omitempty"` } func (x *ConversationAlpha1Request) Reset() { @@ -6898,9 +6900,9 @@ func (x *ConversationAlpha1Request) GetName() string { return "" } -func (x *ConversationAlpha1Request) GetConversationContext() string { - if x != nil && x.ConversationContext != nil { - return *x.ConversationContext +func (x *ConversationAlpha1Request) GetContextID() string { + if x != nil && x.ContextID != nil { + return *x.ContextID } return "" } @@ -6926,6 +6928,13 @@ func (x *ConversationAlpha1Request) GetMetadata() map[string]string { return nil } +func (x *ConversationAlpha1Request) GetScrubPII() bool { + if x != nil && x.ScrubPII != nil { + return *x.ScrubPII + } + return false +} + type ConversationInput struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -6935,6 +6944,8 @@ type ConversationInput struct { Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"` // The role to set for the message Role *string `protobuf:"bytes,2,opt,name=role,proto3,oneof" json:"role,omitempty"` + // Scrub PII data that goes into the LLM + ScrubPII *bool `protobuf:"varint,3,opt,name=scrubPII,proto3,oneof" json:"scrubPII,omitempty"` } func (x *ConversationInput) Reset() { @@ -6983,6 +6994,13 @@ func (x *ConversationInput) GetRole() string { return "" } +func (x *ConversationInput) GetScrubPII() bool { + if x != nil && x.ScrubPII != nil { + return *x.ScrubPII + } + return false +} + // ConversationAlpha1Result is the result for one input. type ConversationAlpha1Result struct { state protoimpl.MessageState @@ -7047,8 +7065,8 @@ type ConversationAlpha1Response struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Conversation context - the Id of an existing chat room (like in ChatGPT) - ConversationContext *string `protobuf:"bytes,1,opt,name=conversationContext,proto3,oneof" json:"conversationContext,omitempty"` + // The ID of an existing chat (like in ChatGPT) + ContextID *string `protobuf:"bytes,1,opt,name=contextID,proto3,oneof" json:"contextID,omitempty"` // An array of results. Outputs []*ConversationAlpha1Result `protobuf:"bytes,2,rep,name=outputs,proto3" json:"outputs,omitempty"` } @@ -7085,9 +7103,9 @@ func (*ConversationAlpha1Response) Descriptor() ([]byte, []int) { return file_dapr_proto_runtime_v1_dapr_proto_rawDescGZIP(), []int{103} } -func (x *ConversationAlpha1Response) GetConversationContext() string { - if x != nil && x.ConversationContext != nil { - return *x.ConversationContext +func (x *ConversationAlpha1Response) GetContextID() string { + if x != nil && x.ContextID != nil { + return *x.ContextID } return "" } @@ -8161,72 +8179,74 @@ var file_dapr_proto_runtime_v1_dapr_proto_rawDesc = []byte{ 0x26, 0x0a, 0x10, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x13, 0x0a, 0x11, 0x44, 0x65, 0x6c, 0x65, 0x74, - 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x90, 0x04, 0x0a, + 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xa0, 0x04, 0x0a, 0x19, 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x73, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, - 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x35, - 0x0a, 0x13, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x73, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, - 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x13, 0x63, - 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x73, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, - 0x78, 0x74, 0x88, 0x01, 0x01, 0x12, 0x40, 0x0a, 0x06, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x18, - 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x64, 0x61, 0x70, 0x72, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, - 0x6e, 0x76, 0x65, 0x72, 0x73, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x52, - 0x06, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x12, 0x60, 0x0a, 0x0a, 0x70, 0x61, 0x72, 0x61, 0x6d, - 0x65, 0x74, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x64, 0x61, - 0x70, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, - 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x73, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x41, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x50, 0x61, - 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x70, - 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x5a, 0x0a, 0x08, 0x6d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x64, 0x61, - 0x70, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, - 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x73, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x41, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x4d, 0x65, - 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x6d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x1a, 0x53, 0x0a, 0x0f, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, - 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2a, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3b, 0x0a, 0x0d, 0x4d, 0x65, - 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, - 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x16, 0x0a, 0x14, 0x5f, 0x63, 0x6f, 0x6e, 0x76, - 0x65, 0x72, 0x73, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x22, - 0x4f, 0x0a, 0x11, 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x73, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, + 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x21, + 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x48, 0x00, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x49, 0x44, 0x88, 0x01, + 0x01, 0x12, 0x40, 0x0a, 0x06, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x28, 0x2e, 0x64, 0x61, 0x70, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x72, + 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72, + 0x73, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x52, 0x06, 0x69, 0x6e, 0x70, + 0x75, 0x74, 0x73, 0x12, 0x60, 0x0a, 0x0a, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, + 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x64, 0x61, 0x70, 0x72, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, + 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x73, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x6c, 0x70, 0x68, + 0x61, 0x31, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, + 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x70, 0x61, 0x72, 0x61, 0x6d, + 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x5a, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x64, 0x61, 0x70, 0x72, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, + 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x73, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x6c, 0x70, 0x68, + 0x61, 0x31, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x12, 0x1f, 0x0a, 0x08, 0x73, 0x63, 0x72, 0x75, 0x62, 0x50, 0x49, 0x49, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x08, 0x48, 0x01, 0x52, 0x08, 0x73, 0x63, 0x72, 0x75, 0x62, 0x50, 0x49, 0x49, 0x88, + 0x01, 0x01, 0x1a, 0x53, 0x0a, 0x0f, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2a, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3b, 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, + 0x49, 0x44, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x73, 0x63, 0x72, 0x75, 0x62, 0x50, 0x49, 0x49, 0x22, + 0x7d, 0x0a, 0x11, 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x73, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x17, 0x0a, 0x04, 0x72, 0x6f, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x04, - 0x72, 0x6f, 0x6c, 0x65, 0x88, 0x01, 0x01, 0x42, 0x07, 0x0a, 0x05, 0x5f, 0x72, 0x6f, 0x6c, 0x65, - 0x22, 0xe8, 0x01, 0x0a, 0x18, 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x73, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x41, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x16, 0x0a, - 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, - 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x5f, 0x0a, 0x0a, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, - 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3f, 0x2e, 0x64, 0x61, 0x70, 0x72, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, - 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x73, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x6c, - 0x70, 0x68, 0x61, 0x31, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x2e, 0x50, 0x61, 0x72, 0x61, 0x6d, - 0x65, 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x70, 0x61, 0x72, 0x61, - 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x1a, 0x53, 0x0a, 0x0f, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, - 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2a, 0x0a, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xb6, 0x01, 0x0a, 0x1a, + 0x72, 0x6f, 0x6c, 0x65, 0x88, 0x01, 0x01, 0x12, 0x1f, 0x0a, 0x08, 0x73, 0x63, 0x72, 0x75, 0x62, + 0x50, 0x49, 0x49, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x48, 0x01, 0x52, 0x08, 0x73, 0x63, 0x72, + 0x75, 0x62, 0x50, 0x49, 0x49, 0x88, 0x01, 0x01, 0x42, 0x07, 0x0a, 0x05, 0x5f, 0x72, 0x6f, 0x6c, + 0x65, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x73, 0x63, 0x72, 0x75, 0x62, 0x50, 0x49, 0x49, 0x22, 0xe8, + 0x01, 0x0a, 0x18, 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x73, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x41, + 0x6c, 0x70, 0x68, 0x61, 0x31, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x72, + 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x73, + 0x75, 0x6c, 0x74, 0x12, 0x5f, 0x0a, 0x0a, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, + 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3f, 0x2e, 0x64, 0x61, 0x70, 0x72, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x73, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x6c, 0x70, 0x68, - 0x61, 0x31, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x35, 0x0a, 0x13, 0x63, 0x6f, - 0x6e, 0x76, 0x65, 0x72, 0x73, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, - 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x13, 0x63, 0x6f, 0x6e, 0x76, 0x65, - 0x72, 0x73, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x88, 0x01, - 0x01, 0x12, 0x49, 0x0a, 0x07, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x64, 0x61, 0x70, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, - 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x76, 0x65, - 0x72, 0x73, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x52, 0x65, 0x73, - 0x75, 0x6c, 0x74, 0x52, 0x07, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x73, 0x42, 0x16, 0x0a, 0x14, - 0x5f, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x73, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, - 0x74, 0x65, 0x78, 0x74, 0x2a, 0x57, 0x0a, 0x16, 0x50, 0x75, 0x62, 0x73, 0x75, 0x62, 0x53, 0x75, + 0x61, 0x31, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x2e, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, + 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, + 0x74, 0x65, 0x72, 0x73, 0x1a, 0x53, 0x0a, 0x0f, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, + 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2a, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x98, 0x01, 0x0a, 0x1a, 0x43, 0x6f, + 0x6e, 0x76, 0x65, 0x72, 0x73, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x6c, 0x70, 0x68, 0x61, 0x31, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x21, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x74, + 0x65, 0x78, 0x74, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x09, 0x63, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x49, 0x44, 0x88, 0x01, 0x01, 0x12, 0x49, 0x0a, 0x07, 0x6f, + 0x75, 0x74, 0x70, 0x75, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x64, + 0x61, 0x70, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, + 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x73, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x41, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, 0x6f, + 0x75, 0x74, 0x70, 0x75, 0x74, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, + 0x78, 0x74, 0x49, 0x44, 0x2a, 0x57, 0x0a, 0x16, 0x50, 0x75, 0x62, 0x73, 0x75, 0x62, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x44, 0x45, 0x43, 0x4c, 0x41, 0x52, 0x41, 0x54, 0x49, 0x56, 0x45, 0x10, 0x01, 0x12, 0x10, 0x0a, 0x0c, diff --git a/tests/integration/suite/daprd/conversation/grpc/scrubpii.go b/tests/integration/suite/daprd/conversation/grpc/scrubpii.go new file mode 100644 index 00000000000..1e0a951f791 --- /dev/null +++ b/tests/integration/suite/daprd/conversation/grpc/scrubpii.go @@ -0,0 +1,166 @@ +/* +Copyright 2024 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implieh. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package http + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + rtv1 "github.com/dapr/dapr/pkg/proto/runtime/v1" + "github.com/dapr/dapr/tests/integration/framework" + "github.com/dapr/dapr/tests/integration/framework/process/daprd" + "github.com/dapr/dapr/tests/integration/suite" +) + +func init() { + suite.Register(new(scrubpii)) +} + +type scrubpii struct { + daprd *daprd.Daprd +} + +func (s *scrubpii) Setup(t *testing.T) []framework.Option { + s.daprd = daprd.New(t, daprd.WithResourceFiles(` +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: echo +spec: + type: conversation.echo + version: v1 + metadata: + - name: key + value: testkey +`)) + + return []framework.Option{ + framework.WithProcesses(s.daprd), + } +} + +func (s *scrubpii) Run(t *testing.T, ctx context.Context) { + s.daprd.WaitUntilRunning(t, ctx) + + client := s.daprd.GRPCClient(t, ctx) + + t.Run("scrub input phone number", func(t *testing.T) { + scrubInput := true + + resp, err := client.ConverseAlpha1(ctx, &rtv1.ConversationAlpha1Request{ + Name: "echo", + Inputs: []*rtv1.ConversationInput{ + { + Message: "well hello there, my phone number is +2222222222", + ScrubPII: &scrubInput, + }, + }, + }) + require.NoError(t, err) + require.Len(t, resp.GetOutputs(), 1) + require.Equal(t, "well hello there, my phone number is ", resp.GetOutputs()[0].GetResult()) + }) + + t.Run("scrub input great phone number", func(t *testing.T) { + scrubInput := true + resp, err := client.ConverseAlpha1(ctx, &rtv1.ConversationAlpha1Request{ + Name: "echo", + Inputs: []*rtv1.ConversationInput{ + { + Message: "well hello there, my phone number is +4422222222", + ScrubPII: &scrubInput, + }, + }, + }) + require.NoError(t, err) + require.Len(t, resp.GetOutputs(), 1) + require.Equal(t, "well hello there, my phone number is ", resp.GetOutputs()[0].GetResult()) + }) + + t.Run("scrub input email", func(t *testing.T) { + scrubInput := true + + resp, err := client.ConverseAlpha1(ctx, &rtv1.ConversationAlpha1Request{ + Name: "echo", + Inputs: []*rtv1.ConversationInput{ + { + Message: "well hello there, my email is test@test.com", + ScrubPII: &scrubInput, + }, + }, + }) + require.NoError(t, err) + require.Len(t, resp.GetOutputs(), 1) + require.Equal(t, "well hello there, my email is ", resp.GetOutputs()[0].GetResult()) + }) + + t.Run("scrub input ip address", func(t *testing.T) { + scrubInput := true + + resp, err := client.ConverseAlpha1(ctx, &rtv1.ConversationAlpha1Request{ + Name: "echo", + Inputs: []*rtv1.ConversationInput{ + { + Message: "well hello there from 10.8.9.1", + ScrubPII: &scrubInput, + }, + }, + }) + require.NoError(t, err) + require.Len(t, resp.GetOutputs(), 1) + require.Equal(t, "well hello there from ", resp.GetOutputs()[0].GetResult()) + }) + + t.Run("scrub all outputs for PII", func(t *testing.T) { + scrubOutput := true + + resp, err := client.ConverseAlpha1(ctx, &rtv1.ConversationAlpha1Request{ + Name: "echo", + Inputs: []*rtv1.ConversationInput{ + { + Message: "well hello there from 10.8.9.1", + }, + { + Message: "well hello there, my email is test@test.com", + }, + }, + ScrubPII: &scrubOutput, + }) + + require.NoError(t, err) + require.Len(t, resp.GetOutputs(), 2) + require.Equal(t, "well hello there from ", resp.GetOutputs()[0].GetResult()) + require.Equal(t, "well hello there, my email is ", resp.GetOutputs()[1].GetResult()) + }) + + t.Run("no scrubbing on good input", func(t *testing.T) { + scrubOutput := true + + resp, err := client.ConverseAlpha1(ctx, &rtv1.ConversationAlpha1Request{ + Name: "echo", + Inputs: []*rtv1.ConversationInput{ + { + Message: "well hello there", + ScrubPII: &scrubOutput, + }, + }, + ScrubPII: &scrubOutput, + }) + require.NoError(t, err) + require.Len(t, resp.GetOutputs(), 1) + require.Equal(t, "well hello there", resp.GetOutputs()[0].GetResult()) + }) +} diff --git a/tests/integration/suite/daprd/conversation/http/scrubpii.go b/tests/integration/suite/daprd/conversation/http/scrubpii.go new file mode 100644 index 00000000000..ca6b4116f1b --- /dev/null +++ b/tests/integration/suite/daprd/conversation/http/scrubpii.go @@ -0,0 +1,135 @@ +/* +Copyright 2024 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implieh. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package http + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/dapr/dapr/tests/integration/framework" + "github.com/dapr/dapr/tests/integration/framework/client" + "github.com/dapr/dapr/tests/integration/framework/process/daprd" + "github.com/dapr/dapr/tests/integration/suite" +) + +func init() { + suite.Register(new(scrubPII)) +} + +type scrubPII struct { + daprd *daprd.Daprd +} + +func (s *scrubPII) Setup(t *testing.T) []framework.Option { + s.daprd = daprd.New(t, daprd.WithResourceFiles(` +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: echo +spec: + type: conversation.echo + version: v1 + metadata: + - name: key + value: testkey +`)) + + return []framework.Option{ + framework.WithProcesses(s.daprd), + } +} + +func (s *scrubPII) Run(t *testing.T, ctx context.Context) { + s.daprd.WaitUntilRunning(t, ctx) + postURL := fmt.Sprintf("http://%s/v1.0-alpha1/conversation/echo/converse", s.daprd.HTTPAddress()) + + httpClient := client.HTTP(t) + + t.Run("scrub input phone number", func(t *testing.T) { + body := `{"inputs":[{"message":"well hello there, my phone number is +2222222222", "scrubPII": true}]}` + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, postURL, strings.NewReader(body)) + require.NoError(t, err) + resp, err := httpClient.Do(req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + respBody, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + require.Equal(t, `{"outputs":[{"result":"well hello there, my phone number is "}]}`, string(respBody)) + }) + + t.Run("scrub input email", func(t *testing.T) { + body := `{"inputs":[{"message":"well hello there, my email is test@test.com", "scrubPII": true}]}` + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, postURL, strings.NewReader(body)) + require.NoError(t, err) + resp, err := httpClient.Do(req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + respBody, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + require.Equal(t, `{"outputs":[{"result":"well hello there, my email is "}]}`, string(respBody)) + }) + + t.Run("scrub input ip address", func(t *testing.T) { + body := `{"inputs":[{"message":"well hello there from 10.8.9.1", "scrubPII": true}]}` + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, postURL, strings.NewReader(body)) + require.NoError(t, err) + resp, err := httpClient.Do(req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + respBody, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + require.Equal(t, `{"outputs":[{"result":"well hello there from "}]}`, string(respBody)) + }) + + t.Run("scrub all outputs for PII", func(t *testing.T) { + body := `{"inputs":[{"message":"well hello there from 10.8.9.1"},{"message":"well hello there, my email is test@test.com"}],"scrubPII": true}` + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, postURL, strings.NewReader(body)) + require.NoError(t, err) + resp, err := httpClient.Do(req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + respBody, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + require.Equal(t, `{"outputs":[{"result":"well hello there from "}, {"result":"well hello there, my email is "}]}`, string(respBody)) + }) + + t.Run("no scrubbing on good input", func(t *testing.T) { + body := `{"inputs":[{"message":"well hello there","scrubPII": true}],"scrubPII": true}` + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, postURL, strings.NewReader(body)) + require.NoError(t, err) + resp, err := httpClient.Do(req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + respBody, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + require.Equal(t, `{"outputs":[{"result":"well hello there"}]}`, string(respBody)) + }) +}