This repository was archived by the owner on May 15, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 2
create_file function and unit tests for it #57
Merged
Merged
Changes from all commits
Commits
Show all changes
29 commits
Select commit
Hold shift + click to select a range
c5f6ac1
Created create_file.go and create_file_test.go
alicjakwie be1071c
Created create_file.go which integrates previous file_gen and the Cre…
alicjakwie 2be9a17
Created create_file.go and create_file_test.go create_file_test mocks
alicjakwie 6e332fd
Updated CreateFile added metrics restructured packages
alicjakwie 411afc5
Updated create_test_file.go restructured packages
alicjakwie 3f2539c
Rename hermes/probe/create_file_test.go to hermes/probe/create/create…
alicjakwie 2ea3d26
Update create_file.go
alicjakwie 20c6fca
Update create_file.go
alicjakwie a423f07
Update create_file_test.go
alicjakwie c285e3d
Update create_file_test.go
alicjakwie 627da5a
Update create_file_test.go
alicjakwie eb8b893
Update hermes/probe/create/create_file_test.go
alicjakwie 347a6e7
Delete create_file.go
alicjakwie 7929719
Delete create_file_test.go
alicjakwie 2b10bb1
addressed all the comments from the PR
alicjakwie 354bcbb
Update create_file_test.go
alicjakwie e4e8404
Update create_file.go
alicjakwie e697a82
Update create_file_test.go
alicjakwie efe7113
Update and rename create_file.go to create.go
alicjakwie da3ba77
Update and rename create_file_test.go to create_test.go
alicjakwie 0a290f2
Update create.go
alicjakwie c770f75
Update create_test.go
alicjakwie 85e1aab
Update create.go
alicjakwie 4fc373c
Update create_test.go
alicjakwie 108a609
Update create.go
alicjakwie 565966e
Update create_test.go
alicjakwie ce55568
Update create.go
alicjakwie 1b81590
Update create.go
alicjakwie 3956f21
Update create.go
alicjakwie File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,202 @@ | ||
| // Copyright 2020 Google LLC | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // https://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
| // | ||
| // Author: Alicja Kwiecinska, GitHub: alicjakwie | ||
| // | ||
| // package create contains all of the logic necessary to create files with randomized contents in GCS. | ||
| // | ||
| // TODO(#76) change the type of fileID to int | ||
| package create | ||
|
|
||
| import ( | ||
| "context" | ||
| "crypto/sha1" | ||
| "fmt" | ||
| "io" | ||
| "math/rand" | ||
| "time" | ||
|
|
||
| "cloud.google.com/go/storage" | ||
| "github.com/google/cloudprober/logger" | ||
| "github.com/googleapis/google-cloud-go-testing/storage/stiface" | ||
| "github.com/googleinterns/step224-2020/hermes/probe" | ||
| "github.com/googleinterns/step224-2020/hermes/probe/metrics" | ||
| "google.golang.org/api/iterator" | ||
|
|
||
| pb "github.com/googleinterns/step224-2020/hermes/proto" | ||
| ) | ||
|
|
||
| const ( | ||
| FileNameFormat = "Hermes_%02d_%x" | ||
| minFileID = 1 | ||
| maxFileID = 50 | ||
| maxFileSizeBytes = 1000 | ||
| hermesAPILatencySeconds = "hermes_api_latency_s" | ||
| ) | ||
|
|
||
| type randomFile struct { | ||
| id int32 | ||
| sizeBytes int | ||
| } | ||
|
|
||
| type randomFileReader struct { | ||
| sizeBytes int | ||
| // currently reading this byte | ||
| i int | ||
| rand *rand.Rand | ||
| } | ||
|
|
||
| // Read implements the standard Read interface, it returns a random stream of bytes generated by a pseudo-random generator. | ||
| // Argument: | ||
| // buf: a byte slice serves as an output buffer | ||
| // Returns: | ||
| // n: the number of bytes read | ||
| // err: error it should be nil when not done reading and io.EOF once the whole file contents have been read | ||
| func (r *randomFileReader) Read(buf []byte) (n int, err error) { | ||
| // check whether Read is done | ||
| if r.i >= r.sizeBytes { | ||
| return 0, io.EOF | ||
| } | ||
| b := buf | ||
| if len(buf) > r.sizeBytes-r.i { | ||
| // if the length of buffer is greater than the number of bytes left to read make the sizeBytes of the buffer match the number of bytes left to read | ||
| b = buf[:r.sizeBytes-r.i] | ||
| } | ||
| // n is now the length of the buffer | ||
| n, err = r.rand.Read(b) | ||
| if err != nil { | ||
| // in this case n = 0 | ||
| return n, err | ||
| } | ||
| r.i += n | ||
| return n, err | ||
| } | ||
|
|
||
| func (f *randomFile) newReader() *randomFileReader { | ||
| // id will serve as a Seed and i - index of the currently read byte will be set to 0 automatically in the returned reader | ||
| return &randomFileReader{ | ||
| sizeBytes: f.sizeBytes, | ||
| rand: rand.New(rand.NewSource(int64(f.id))), | ||
| } | ||
| } | ||
|
|
||
| func newRandomFile(id int32, sizeBytes int) (*randomFile, error) { | ||
| if id < minFileID || id > maxFileID { | ||
| return nil, fmt.Errorf("invalid argument: id = %d; want %d <= id <= %d", id, minFileID, maxFileID) | ||
| } | ||
| if sizeBytes > maxFileSizeBytes || sizeBytes <= 0 { | ||
| return nil, fmt.Errorf("invalid argument: sizeBytes = %d; want 0 < sizeBytes <= %d", sizeBytes, maxFileSizeBytes) | ||
| } | ||
| return &randomFile{id: id, sizeBytes: sizeBytes}, nil | ||
| } | ||
|
|
||
| func (f *randomFile) checksum() ([]byte, error) { | ||
| r := f.newReader() | ||
| h := sha1.New() | ||
| if _, err := io.Copy(h, r); err != nil { | ||
| return nil, fmt.Errorf("io.Copy: %w", err) | ||
| } | ||
| return h.Sum(nil), nil | ||
| } | ||
|
|
||
| func (f *randomFile) fileName() (string, error) { | ||
| checksum, err := f.checksum() | ||
| if err != nil { | ||
| return "", fmt.Errorf("{%d, %d}.checksum = nil, %w", f.id, f.sizeBytes, err) | ||
| } | ||
| return fmt.Sprintf(FileNameFormat, f.id, checksum), nil | ||
| } | ||
|
|
||
| // CreateFile creates and stores a file with randomized contents in the target storage system. | ||
| // Before it creates and stores a file it logs an intent to do so in the target storage system's journal. | ||
| // It verifies that the creation and storage process was successful. | ||
| // Finally, it updates the filenames map in the target's journal and record the exit status in the logger. | ||
| // Arguments: | ||
| // ctx: it carries deadlines and cancellation signals that might orinate from the main probe located in probe.go. | ||
| // target: contains information about target storage system, carries an intent log in the form of a StateJournal and it used to export metrics. | ||
| // fileID: the unique identifer of every randomFile, it cannot be repeated. It needs to be in the range [minFileID, maxFileID]. FileID 0 is reserved for a special file called the NIL file. | ||
| // client: is a storage client. It is used as an interface to interact with the target storage system. | ||
| // logger: a cloudprober logger used to record the exit status of the CreateFile operation in a target bucket. The logger passed MUST be a valid logger. | ||
| // Returns: | ||
| // error: an error string with detailed information about the status and fileID. Nil is returned when the operation is successful. | ||
| func CreateFile(ctx context.Context, target *probe.Target, fileID int32, fileSize int, client stiface.Client, logger *logger.Logger) error { | ||
| f, err := newRandomFile(fileID, fileSize) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| fileName, err := f.fileName() | ||
| if err != nil { | ||
| return err | ||
| } | ||
| target.Journal.Intent = &pb.Intent{FileOperation: pb.Intent_CREATE, Filename: fileName} | ||
| var status metrics.ExitStatus | ||
| if _, ok := target.Journal.Filenames[fileID]; ok { | ||
| status = metrics.UnknownFileFound | ||
| return fmt.Errorf("CreateFile(fileID: %d).%q could not create file as file with this ID already exists", fileID, status) | ||
| } | ||
| r := f.newReader() | ||
| start := time.Now() | ||
| bucketName := target.Target.GetBucketName() | ||
| wc := client.Bucket(bucketName).Object(fileName).NewWriter(ctx) | ||
| if _, err = io.Copy(wc, r); err != nil { | ||
| switch err { | ||
| case storage.ErrBucketNotExist: | ||
| status = metrics.BucketMissing | ||
| default: | ||
| status = metrics.ProbeFailed | ||
| } | ||
| target.LatencyMetrics.APICallLatency[metrics.APICreateFile][status].Metric(hermesAPILatencySeconds).AddFloat64(time.Now().Sub(start).Seconds()) | ||
| return fmt.Errorf("CreateFile(id: %d).%q: could not create file %q: %w", fileID, status, fileName, err) | ||
| } | ||
| if err := wc.Close(); err != nil { | ||
| status = metrics.WriterCloseFailed | ||
| target.LatencyMetrics.APICallLatency[metrics.APICreateFile][status].Metric(hermesAPILatencySeconds).AddFloat64(time.Now().Sub(start).Seconds()) | ||
| return fmt.Errorf("Writer.Close: %w with status %q", err, status) | ||
| } | ||
| status = metrics.Success | ||
| target.LatencyMetrics.APICallLatency[metrics.APICreateFile][status].Metric(hermesAPILatencySeconds).AddFloat64(time.Now().Sub(start).Seconds()) | ||
|
|
||
| // Verify that the file that has just been created is in fact present in the target system | ||
| fileNamePrefix := fmt.Sprintf(FileNameFormat, fileID, "") | ||
| query := &storage.Query{Prefix: fileNamePrefix} | ||
| start = time.Now() | ||
| objIter := client.Bucket(bucketName).Objects(ctx, query) | ||
| var namesFound []string | ||
| for { | ||
| obj, err := objIter.Next() | ||
| if err == iterator.Done { | ||
| break | ||
| } | ||
| if err != nil { | ||
| return fmt.Errorf("CreateFile check failed due to: %w", err) | ||
| } | ||
| namesFound = append(namesFound, obj.Name) | ||
| } | ||
| finish := time.Now() | ||
| if len(namesFound) == 0 { | ||
| target.LatencyMetrics.APICallLatency[metrics.APIListFiles][metrics.FileMissing].Metric(hermesAPILatencySeconds).AddFloat64(finish.Sub(start).Seconds()) | ||
| return fmt.Errorf("CreateFile check failed no files with prefix %q found", fileNamePrefix) | ||
| } | ||
| if len(namesFound) != 1 { | ||
| return fmt.Errorf("expected exactly one file in bucket %q with prefix %q; found %d: %v", bucketName, fileNamePrefix, len(namesFound), namesFound) | ||
| } | ||
| if namesFound[0] != fileName { | ||
| fmt.Errorf("CreateFile check failed: filename matching %q prefix: %q; want %q", fileNamePrefix, namesFound[0], filaName) | ||
| } | ||
| target.LatencyMetrics.APICallLatency[metrics.APIListFiles][metrics.Success].Metric(hermesAPILatencySeconds).AddFloat64(finish.Sub(start).Seconds()) | ||
|
|
||
| target.Journal.Filenames[fileID] = fileName | ||
| logger.Infof("Object %q added in bucket %q.", fileName, bucketName) | ||
| return nil | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,180 @@ | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // https://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
| // | ||
| // Author: Alicja Kwiecinska GitHub: alicjakwie | ||
| // | ||
| // TODO (#72) change error types to be compatible with ProbeError | ||
|
|
||
| package create | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "strings" | ||
| "testing" | ||
|
|
||
| "github.com/golang/protobuf/proto" | ||
| "github.com/googleinterns/step224-2020/hermes/probe" | ||
| "github.com/googleinterns/step224-2020/hermes/probe/fakegcs" | ||
| "github.com/googleinterns/step224-2020/hermes/probe/metrics" | ||
|
|
||
| metricpb "github.com/google/cloudprober/metrics/proto" | ||
| probepb "github.com/googleinterns/step224-2020/config/proto" | ||
| journalpb "github.com/googleinterns/step224-2020/hermes/proto" | ||
| ) | ||
|
|
||
| var fileNamePrefixLength = len(fmt.Sprintf(FileNameFormat, 0, "")) | ||
|
|
||
| func TestNewRandomFile(t *testing.T) { | ||
| tests := []struct { | ||
| fileID int32 | ||
| fileSize int | ||
| want *randomFile | ||
| wantErr bool | ||
| }{ | ||
| {51, 12, nil, true}, | ||
| {0, 50, nil, true}, | ||
| {3, 100, &randomFile{3, 100}, false}, | ||
| {12, 100, &randomFile{12, 100}, false}, | ||
| {3, 0, nil, true}, | ||
| {3, 1001, nil, true}, | ||
| } | ||
| for _, tc := range tests { | ||
| got, err := newRandomFile(tc.fileID, tc.fileSize) | ||
| if tc.want == nil && got != nil { | ||
| t.Errorf("{%d, %d}.newRandomFile = {%d, %d} expected nil", tc.fileID, tc.fileSize, got.id, got.sizeBytes) | ||
| } | ||
| if got == nil && tc.want != nil { | ||
| t.Errorf("{%d, %d}.newRandomFile = nil expected {%d, %d}", tc.fileID, tc.fileSize, tc.want.id, tc.want.sizeBytes) | ||
| } | ||
| if err != nil && !tc.wantErr { | ||
| t.Errorf("{%d, %d}.newRandomFile() failed and returned an unexpected error %w", tc.fileID, tc.fileSize, err) | ||
| } | ||
| if err == nil && tc.wantErr { | ||
| t.Errorf("{%d, %d}.newRandomFile() failed expected an error got nil", tc.fileID, tc.fileSize) | ||
| } | ||
| if got != nil && (got.id != tc.want.id || got.sizeBytes != tc.want.sizeBytes) { | ||
| t.Errorf("{%d, %d}.newRandomFile() = {%d,%d}, expected {%d,%d}", tc.fileID, tc.fileSize, got.id, got.sizeBytes, tc.want.id, tc.want.sizeBytes) | ||
| } | ||
|
|
||
| } | ||
|
|
||
| } | ||
|
|
||
| func TestFileName(t *testing.T) { | ||
| tests := []struct { | ||
| file *randomFile | ||
| want string | ||
| }{ | ||
| {&randomFile{3, 100}, "Hermes_03_"}, | ||
| {&randomFile{12, 100}, "Hermes_12_"}, | ||
| {&randomFile{8, 20}, "Hermes_08_"}, | ||
| } | ||
|
|
||
| for _, tc := range tests { | ||
| got, err := tc.file.fileName() | ||
| if err != nil { | ||
| t.Errorf("{%d, %d}.fileName() failed and returned an unexpected error %w", tc.file.id, tc.file.sizeBytes, err) | ||
| } | ||
| if !strings.HasPrefix(got, tc.want) { | ||
| t.Errorf("{%d, %d}.fileName() = %qchecksum expected %qchecksum", tc.file.id, tc.file.sizeBytes, got[:fileNamePrefixLength], tc.want) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func TestChecksum(t *testing.T) { | ||
| file := randomFile{11, 100} | ||
| checksum, err := file.checksum() | ||
| if err != nil { | ||
| t.Error(err) | ||
| } | ||
| otherFile := randomFile{13, 100} | ||
| otherChecksum, err := otherFile.checksum() | ||
| if err != nil { | ||
| t.Error(err) | ||
| } | ||
| if fmt.Sprintf("%x", checksum) == fmt.Sprintf("%x", otherChecksum) { | ||
| t.Errorf("{%d, %d}.checksum = {%d,%d}.checksum expected {%d, %d}.checksum != {%d, %d}.checksum ", file.id, file.sizeBytes, otherFile.id, otherFile.sizeBytes, file.id, file.sizeBytes, otherFile.id, otherFile.sizeBytes) | ||
|
|
||
| } | ||
| file = randomFile{11, 100} | ||
| checksumAgain, err := file.checksum() | ||
| if err != nil { | ||
| t.Error(err) | ||
| } | ||
| if fmt.Sprintf("%x", checksum) != fmt.Sprintf("%x", checksumAgain) { | ||
| t.Errorf("{%d, %d}.checksum != {%d, %d}.checksum expected {%d, %d}.checksum = {%d, %d}.checksum", file.id, file.sizeBytes, file.id, file.sizeBytes, file.id, file.sizeBytes, file.id, file.sizeBytes) | ||
| } | ||
| } | ||
|
|
||
| func TestCreateFile(t *testing.T) { | ||
| ctx := context.Background() | ||
| client := fakegcs.NewClient() | ||
| bucketName := "test_bucket_probe0" | ||
| fakeBucketHandle := client.Bucket(bucketName) | ||
| if err := fakeBucketHandle.Create(ctx, bucketName, nil); err != nil { | ||
| t.Error(err) | ||
| } | ||
| fileID := int32(6) | ||
| fileSize := 50 | ||
| target := &probe.Target{ | ||
| &probepb.Target{ | ||
| Name: "hermes", | ||
| TargetSystem: probepb.Target_GOOGLE_CLOUD_STORAGE, | ||
| TotalSpaceAllocatedMib: int64(1000), | ||
| BucketName: "test_bucket_probe0", | ||
| }, | ||
| &journalpb.StateJournal{ | ||
| Filenames: make(map[int32]string), | ||
| }, | ||
| &metrics.Metrics{}, | ||
| } | ||
| hp := &probepb.HermesProbeDef{ | ||
| ProbeName: proto.String("createfile_test"), | ||
| Targets: []*probepb.Target{ | ||
| &probepb.Target{ | ||
| Name: "hermes", | ||
| TargetSystem: probepb.Target_GOOGLE_CLOUD_STORAGE, | ||
| TotalSpaceAllocatedMib: int64(100), | ||
| BucketName: "test_bucket_probe0", | ||
| }, | ||
| }, | ||
| TargetSystem: probepb.HermesProbeDef_GCS.Enum(), | ||
| IntervalSec: proto.Int32(3600), | ||
| TimeoutSec: proto.Int32(60), | ||
| ProbeLatencyDistribution: &metricpb.Dist{ | ||
| Buckets: &metricpb.Dist_ExplicitBuckets{ | ||
| ExplicitBuckets: "0.1,0.2,0.4,0.6,0.8,1.6,3.2,6.4,12.8,1", | ||
| }, | ||
| }, | ||
| ApiCallLatencyDistribution: &metricpb.Dist{ | ||
| Buckets: &metricpb.Dist_ExplicitBuckets{ | ||
| ExplicitBuckets: "0.1,0.2,0.4,0.6,0.8,1.6,3.2,6.4,12.8,1", | ||
| }, | ||
| }, | ||
| } | ||
| probeTarget := &probepb.Target{ | ||
| Name: "hermes", | ||
| TargetSystem: probepb.Target_GOOGLE_CLOUD_STORAGE, | ||
| TotalSpaceAllocatedMib: int64(100), | ||
| BucketName: "test_bucket_probe0", | ||
| } | ||
|
|
||
| var err error | ||
| if target.LatencyMetrics, err = metrics.NewMetrics(hp, probeTarget); err != nil { | ||
| t.Fatalf("metrics.NewMetrics(): %v", err) | ||
| } | ||
| logger := fakegcs.NewLogger(ctx).Logger | ||
| if err := CreateFile(ctx, target, fileID, fileSize, client, logger); err != nil { | ||
| t.Error(err) | ||
| } | ||
| } |
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.