From 9237ddfe4342509617fea9cac5e6e59b54cc2bca Mon Sep 17 00:00:00 2001 From: Crimson <39024757+crimson-gao@users.noreply.github.com> Date: Thu, 13 Jun 2024 15:04:25 +0800 Subject: [PATCH] Feature storeview (#283) * feat: support storeView impl * chore: add const storeType * chore: use const in test * feat: add interface for storeView --- client_interface.go | 14 ++ client_store_view.go | 124 ++++++++++++++++++ client_store_view_test.go | 255 ++++++++++++++++++++++++++++++++++++ model.go | 47 +++++++ token_auto_update_client.go | 60 +++++++++ 5 files changed, 500 insertions(+) create mode 100644 client_store_view.go create mode 100644 client_store_view_test.go diff --git a/client_interface.go b/client_interface.go index ebfe2a64..93eb98c6 100644 --- a/client_interface.go +++ b/client_interface.go @@ -170,6 +170,20 @@ type ClientInterface interface { // ListEventStore returns all eventStore names of project p. ListEventStore(project string, offset, size int) ([]string, error) + // #################### StoreView Operations ##################### + // CreateStoreView creates a new storeView. + CreateStoreView(project string, storeView *StoreView) error + // UpdateStoreView updates a storeView. + UpdateStoreView(project string, storeView *StoreView) error + // DeleteStoreView deletes a storeView. + DeleteStoreView(project string, storeViewName string) error + // GetStoreView returns storeView. + GetStoreView(project string, storeViewName string) (*StoreView, error) + // ListStoreViews returns all storeView names of a project. + ListStoreViews(project string, req *ListStoreViewsRequest) (*ListStoreViewsResponse, error) + // GetStoreViewIndex returns all index config of logstores in the storeView, only support storeType logstore. + GetStoreViewIndex(project string, storeViewName string) (*GetStoreViewIndexResponse, error) + // #################### Logtail Operations ##################### // ListMachineGroup returns machine group name list and the total number of machine groups. // The offset starts from 0 and the size is the max number of machine groups could be returned. diff --git a/client_store_view.go b/client_store_view.go new file mode 100644 index 00000000..cba85768 --- /dev/null +++ b/client_store_view.go @@ -0,0 +1,124 @@ +package sls + +import ( + "encoding/json" + "fmt" + "io/ioutil" +) + +func (c *Client) CreateStoreView(project string, storeView *StoreView) error { + body, err := json.Marshal(storeView) + if err != nil { + return NewClientError(err) + } + h := map[string]string{ + "Content-Type": "application/json", + "x-log-bodyrawsize": fmt.Sprintf("%v", len(body)), + } + uri := "/storeviews" + r, err := c.request(project, "POST", uri, h, body) + if err != nil { + return err + } + r.Body.Close() + return nil +} + +func (c *Client) UpdateStoreView(project string, storeView *StoreView) error { + body, err := json.Marshal(storeView) + if err != nil { + return NewClientError(err) + } + h := map[string]string{ + "Content-Type": "application/json", + "x-log-bodyrawsize": fmt.Sprintf("%v", len(body)), + } + uri := "/storeviews/" + storeView.Name + r, err := c.request(project, "PUT", uri, h, body) + if err != nil { + return err + } + r.Body.Close() + return nil +} + +func (c *Client) DeleteStoreView(project string, storeViewName string) error { + h := map[string]string{ + "Content-Type": "application/json", + "x-log-bodyrawsize": "0", + } + uri := "/storeviews/" + storeViewName + r, err := c.request(project, "DELETE", uri, h, nil) + if err != nil { + return err + } + r.Body.Close() + return nil +} + +func (c *Client) GetStoreView(project string, storeViewName string) (*StoreView, error) { + h := map[string]string{ + "Content-Type": "application/json", + "x-log-bodyrawsize": "0", + } + uri := "/storeviews/" + storeViewName + r, err := c.request(project, "GET", uri, h, nil) + if err != nil { + return nil, err + } + defer r.Body.Close() + buf, err := ioutil.ReadAll(r.Body) + if err != nil { + return nil, err + } + res := &StoreView{} + if err = json.Unmarshal(buf, res); err != nil { + return nil, NewClientError(err) + } + res.Name = storeViewName + return res, nil +} + +func (c *Client) ListStoreViews(project string, req *ListStoreViewsRequest) (*ListStoreViewsResponse, error) { + h := map[string]string{ + "Content-Type": "application/json", + "x-log-bodyrawsize": "0", + } + uri := fmt.Sprintf("/storeviews?offset=%d&line=%d", req.Offset, req.Size) + r, err := c.request(project, "GET", uri, h, nil) + if err != nil { + return nil, err + } + defer r.Body.Close() + buf, err := ioutil.ReadAll(r.Body) + if err != nil { + return nil, err + } + res := &ListStoreViewsResponse{} + if err = json.Unmarshal(buf, res); err != nil { + return nil, NewClientError(err) + } + return res, nil +} + +func (c *Client) GetStoreViewIndex(project string, storeViewName string) (*GetStoreViewIndexResponse, error) { + h := map[string]string{ + "Content-Type": "application/json", + "x-log-bodyrawsize": "0", + } + uri := fmt.Sprintf("/storeviews/%s/index", storeViewName) + r, err := c.request(project, "GET", uri, h, nil) + if err != nil { + return nil, err + } + defer r.Body.Close() + buf, err := ioutil.ReadAll(r.Body) + if err != nil { + return nil, err + } + res := &GetStoreViewIndexResponse{} + if err = json.Unmarshal(buf, res); err != nil { + return nil, NewClientError(err) + } + return res, nil +} diff --git a/client_store_view_test.go b/client_store_view_test.go new file mode 100644 index 00000000..6baaddff --- /dev/null +++ b/client_store_view_test.go @@ -0,0 +1,255 @@ +package sls + +import ( + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type StoreViewTestSuite struct { + suite.Suite + endpoint string + project string + storeViewName string + accessKeyID string + accessKeySecret string + client Client +} + +func TestStoreView(t *testing.T) { + suite.Run(t, new(StoreViewTestSuite)) +} + +func (s *StoreViewTestSuite) SetupSuite() { + s.endpoint = os.Getenv("LOG_TEST_ENDPOINT") + s.project = os.Getenv("LOG_TEST_STORE_VIEW_PROJECT") + s.accessKeyID = os.Getenv("LOG_TEST_ACCESS_KEY_ID") + s.accessKeySecret = os.Getenv("LOG_TEST_ACCESS_KEY_SECRET") + s.storeViewName = fmt.Sprintf("storeview_%d", time.Now().Unix()) + s.client.AccessKeyID = s.accessKeyID + s.client.AccessKeySecret = s.accessKeySecret + s.client.Endpoint = s.endpoint + + require.NotEmpty(s.T(), s.endpoint) + require.NotEmpty(s.T(), s.project) + require.NotEmpty(s.T(), s.accessKeyID) + require.NotEmpty(s.T(), s.accessKeySecret) + s.client.DeleteProject(s.project) + + time.Sleep(time.Second * 15) + _, err := s.client.CreateProject(s.project, "test project") + if err != nil { + panic(err) + } + time.Sleep(time.Second * 1) + s.createStores() + time.Sleep(time.Second * 1) +} + +func (s *StoreViewTestSuite) TearDownSuite() { + err := s.client.DeleteProject(s.project) + if err != nil { + panic(err) + } +} + +func (s *StoreViewTestSuite) TestStoreViewCURD() { + // get + _, err := s.client.GetStoreView(s.project, s.storeViewName) + s.NotNil(err) + s.Require().True(strings.Contains(err.Error(), "not exist")) + + // list + r, err := s.client.ListStoreViews(s.project, &ListStoreViewsRequest{ + Offset: 0, + Size: 10, + }) + s.Require().NoError(err) + s.Require().Equal(0, r.Count) + s.Require().Equal(0, r.Total) + s.Require().Equal(0, len(r.StoreViews)) + + // update + err = s.client.UpdateStoreView(s.project, &StoreView{ + Name: s.storeViewName, + StoreType: STORE_VIEW_STORE_TYPE_LOGSTORE, + Stores: []*StoreViewStore{ + { + StoreName: "logstore-1", + Project: s.project, + }, + }, + }) + s.NotNil(err) + s.Require().True(strings.Contains(err.Error(), "not exist")) + + // delete + err = s.client.DeleteStoreView(s.project, s.storeViewName) + s.NotNil(err) + s.Require().True(strings.Contains(err.Error(), "not exist")) + + // create ok + err = s.client.CreateStoreView(s.project, &StoreView{ + Name: s.storeViewName, + StoreType: STORE_VIEW_STORE_TYPE_LOGSTORE, + Stores: []*StoreViewStore{ + { + StoreName: "logstore-1", + Project: s.project, + }, + }, + }) + s.Require().NoError(err) + + // get + storeView, err := s.client.GetStoreView(s.project, s.storeViewName) + s.Require().NoError(err) + s.Require().Equal(s.storeViewName, storeView.Name) + s.Require().Equal(STORE_VIEW_STORE_TYPE_LOGSTORE, storeView.StoreType) + s.Require().Equal(1, len(storeView.Stores)) + + // list + r, err = s.client.ListStoreViews(s.project, &ListStoreViewsRequest{ + Offset: 0, + Size: 10, + }) + s.Require().NoError(err) + s.Require().Equal(1, r.Count) + s.Require().Equal(1, r.Total) + s.Require().Equal(1, len(r.StoreViews)) + s.Require().Equal(s.storeViewName, r.StoreViews[0]) + + // update + err = s.client.UpdateStoreView(s.project, &StoreView{ + Name: s.storeViewName, + StoreType: STORE_VIEW_STORE_TYPE_LOGSTORE, + Stores: []*StoreViewStore{ + { + StoreName: "logstore-1", + Project: s.project, + }, + }, + }) + s.Require().NoError(err) + + // delete + err = s.client.DeleteStoreView(s.project, s.storeViewName) + s.Require().NoError(err) + + // get + _, err = s.client.GetStoreView(s.project, s.storeViewName) + s.NotNil(err) + s.Require().True(strings.Contains(err.Error(), "not exist")) + + // list + r, err = s.client.ListStoreViews(s.project, &ListStoreViewsRequest{ + Offset: 0, + Size: 10, + }) + s.Require().NoError(err) + s.Require().Equal(0, r.Count) + s.Require().Equal(0, r.Total) + s.Require().Equal(0, len(r.StoreViews)) +} + +func (s *StoreViewTestSuite) TestStoreViewTypes() { + err := s.client.CreateStoreView(s.project, &StoreView{ + Name: s.storeViewName + "_1", + StoreType: STORE_VIEW_STORE_TYPE_LOGSTORE, + Stores: []*StoreViewStore{ + { + Project: s.project, + StoreName: fmt.Sprintf("logstore-%d", 0), + }, + { + Project: s.project, + StoreName: fmt.Sprintf("logstore-%d", 1), + }, + }, + }) + s.Require().NoError(err) + + err = s.client.CreateStoreView(s.project, &StoreView{ + Name: s.storeViewName + "_2", + StoreType: STORE_VIEW_STORE_TYPE_METRICSTORE, + Stores: []*StoreViewStore{ + { + Project: s.project, + StoreName: fmt.Sprintf("metricstore-%d", 0), + }, + { + Project: s.project, + StoreName: fmt.Sprintf("metricstore-%d", 1), + }, + }, + }) + s.Require().NoError(err) + + storeview, err := s.client.GetStoreView(s.project, s.storeViewName+"_1") + s.Require().NoError(err) + s.Require().Equal(s.storeViewName+"_1", storeview.Name) + s.Require().Equal(STORE_VIEW_STORE_TYPE_LOGSTORE, storeview.StoreType) + s.Require().Equal(2, len(storeview.Stores)) + + storeview, err = s.client.GetStoreView(s.project, s.storeViewName+"_2") + s.Require().NoError(err) + s.Require().Equal(s.storeViewName+"_2", storeview.Name) + s.Require().Equal(STORE_VIEW_STORE_TYPE_METRICSTORE, storeview.StoreType) + s.Require().Equal(2, len(storeview.Stores)) + + // list + r, err := s.client.ListStoreViews(s.project, &ListStoreViewsRequest{ + Offset: 0, + Size: 10, + }) + s.Require().NoError(err) + s.Require().Equal(2, r.Count) + s.Require().Equal(2, r.Total) + s.Require().Equal(2, len(r.StoreViews)) + + // get index + r2, err := s.client.GetStoreViewIndex(s.project, s.storeViewName+"_1") + s.Require().NoError(err) + s.Require().Equal(2, len(r2.Indexes)) + s.Require().Equal(0, len(r2.StoreViewErrors)) + + _, err = s.client.GetStoreViewIndex(s.project, s.storeViewName+"_2") + s.Require().NotNil(err) + s.True(strings.Contains(err.Error(), "not support")) + + // delete + err = s.client.DeleteStoreView(s.project, s.storeViewName+"_1") + s.Require().NoError(err) + err = s.client.DeleteStoreView(s.project, s.storeViewName+"_2") + s.Require().NoError(err) +} + +func (s *StoreViewTestSuite) createStores() { + for i := 0; i < 2; i++ { + err := s.client.CreateLogStore(s.project, fmt.Sprintf("logstore-%d", i), 7, 2, false, 64) + s.Require().NoError(err) + err = s.client.CreateIndex(s.project, fmt.Sprintf("logstore-%d", i), Index{ + Line: &IndexLine{ + CaseSensitive: false, + Chn: false, + Token: []string{ + ",", " ", "'", "\"", ";", "\\", "$", "#", "!", "=", "(", ")", "[", "]", "{", "}", "?", "@", "&", "<", ">", "/", ":", "\n", "\t", "\r", + }, + }, + LogReduce: false, + }) + s.Require().NoError(err) + err = s.client.CreateMetricStore(s.project, &LogStore{ + Name: fmt.Sprintf("metricstore-%d", i), + TTL: 7, + ShardCount: 2, + }) + s.Require().NoError(err) + } + +} diff --git a/model.go b/model.go index b48f65b2..02ec8fca 100644 --- a/model.go +++ b/model.go @@ -327,3 +327,50 @@ type PostLogStoreLogsRequest struct { HashKey *string CompressType int } + +type StoreView struct { + Name string `json:"name"` + StoreType string `json:"storeType"` + Stores []*StoreViewStore `json:"stores"` +} + +// storeType of storeView +const ( + STORE_VIEW_STORE_TYPE_LOGSTORE = "logstore" + STORE_VIEW_STORE_TYPE_METRICSTORE = "metricstore" +) + +type StoreViewStore struct { + Project string `json:"project"` + StoreName string `json:"storeName"` + Query string `json:"query,omitempty"` +} + +type GetStoreViewIndexResponse struct { + Indexes []*StoreViewIndex `json:"indexes"` + StoreViewErrors []*StoreViewErrors `json:"storeViewErrors"` +} + +type StoreViewIndex struct { + ProjectName string `json:"projectName"` + LogStore string `json:"logstore"` + Index Index `json:"index"` +} + +type StoreViewErrors struct { + ProjectName string `json:"projectName"` + LogStore string `json:"logstore"` + Status string `json:"status"` + Message string `json:"message"` +} + +type ListStoreViewsRequest struct { + Offset int `json:"offset"` + Size int `json:"size"` +} + +type ListStoreViewsResponse struct { + Total int `json:"total"` + Count int `json:"count"` + StoreViews []string `json:"storeviews"` +} diff --git a/token_auto_update_client.go b/token_auto_update_client.go index ec633427..3b4f28e2 100644 --- a/token_auto_update_client.go +++ b/token_auto_update_client.go @@ -2020,3 +2020,63 @@ func (c *TokenAutoUpdateClient) PostLogStoreLogsV2(project, logstore string, req } return } + +func (c *TokenAutoUpdateClient) CreateStoreView(project string, storeView *StoreView) (err error) { + for i := 0; i < c.maxTryTimes; i++ { + err = c.logClient.CreateStoreView(project, storeView) + if !c.processError(err) { + return + } + } + return +} + +func (c *TokenAutoUpdateClient) UpdateStoreView(project string, storeView *StoreView) (err error) { + for i := 0; i < c.maxTryTimes; i++ { + err = c.logClient.UpdateStoreView(project, storeView) + if !c.processError(err) { + return + } + } + return +} + +func (c *TokenAutoUpdateClient) DeleteStoreView(project string, storeViewName string) (err error) { + for i := 0; i < c.maxTryTimes; i++ { + err = c.logClient.DeleteStoreView(project, storeViewName) + if !c.processError(err) { + return + } + } + return +} + +func (c *TokenAutoUpdateClient) GetStoreView(project string, storeViewName string) (storeView *StoreView, err error) { + for i := 0; i < c.maxTryTimes; i++ { + storeView, err = c.logClient.GetStoreView(project, storeViewName) + if !c.processError(err) { + return + } + } + return +} + +func (c *TokenAutoUpdateClient) ListStoreViews(project string, req *ListStoreViewsRequest) (resp *ListStoreViewsResponse, err error) { + for i := 0; i < c.maxTryTimes; i++ { + resp, err = c.logClient.ListStoreViews(project, req) + if !c.processError(err) { + return + } + } + return +} + +func (c *TokenAutoUpdateClient) GetStoreViewIndex(project string, storeViewName string) (resp *GetStoreViewIndexResponse, err error) { + for i := 0; i < c.maxTryTimes; i++ { + resp, err = c.logClient.GetStoreViewIndex(project, storeViewName) + if !c.processError(err) { + return + } + } + return +}