Skip to content
This repository was archived by the owner on May 15, 2023. It is now read-only.
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
c5f6ac1
Created create_file.go and create_file_test.go
alicjakwie Sep 10, 2020
be1071c
Created create_file.go which integrates previous file_gen and the Cre…
alicjakwie Sep 10, 2020
2be9a17
Created create_file.go and create_file_test.go create_file_test mocks
alicjakwie Sep 15, 2020
6e332fd
Updated CreateFile added metrics restructured packages
alicjakwie Sep 15, 2020
411afc5
Updated create_test_file.go restructured packages
alicjakwie Sep 15, 2020
3f2539c
Rename hermes/probe/create_file_test.go to hermes/probe/create/create…
alicjakwie Sep 16, 2020
2ea3d26
Update create_file.go
alicjakwie Sep 16, 2020
20c6fca
Update create_file.go
alicjakwie Sep 16, 2020
a423f07
Update create_file_test.go
alicjakwie Sep 16, 2020
c285e3d
Update create_file_test.go
alicjakwie Sep 16, 2020
627da5a
Update create_file_test.go
alicjakwie Sep 16, 2020
eb8b893
Update hermes/probe/create/create_file_test.go
alicjakwie Sep 16, 2020
347a6e7
Delete create_file.go
alicjakwie Sep 16, 2020
7929719
Delete create_file_test.go
alicjakwie Sep 16, 2020
2b10bb1
addressed all the comments from the PR
alicjakwie Sep 16, 2020
354bcbb
Update create_file_test.go
alicjakwie Sep 16, 2020
e4e8404
Update create_file.go
alicjakwie Sep 16, 2020
e697a82
Update create_file_test.go
alicjakwie Sep 16, 2020
efe7113
Update and rename create_file.go to create.go
alicjakwie Sep 18, 2020
da3ba77
Update and rename create_file_test.go to create_test.go
alicjakwie Sep 18, 2020
0a290f2
Update create.go
alicjakwie Sep 18, 2020
c770f75
Update create_test.go
alicjakwie Sep 18, 2020
85e1aab
Update create.go
alicjakwie Sep 18, 2020
4fc373c
Update create_test.go
alicjakwie Sep 18, 2020
108a609
Update create.go
alicjakwie Sep 18, 2020
565966e
Update create_test.go
alicjakwie Sep 18, 2020
ce55568
Update create.go
alicjakwie Sep 18, 2020
1b81590
Update create.go
alicjakwie Sep 18, 2020
3956f21
Update create.go
alicjakwie Sep 18, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
202 changes: 202 additions & 0 deletions hermes/probe/create/create.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,202 @@
// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Alicja Kwiecinska, GitHub: alicjakwie
//
// package create contains all of the logic necessary to create files with randomized contents in GCS.
//
// TODO(#76) change the type of fileID to int
package create

import (
"context"
"crypto/sha1"
"fmt"
"io"
"math/rand"
"time"

"cloud.google.com/go/storage"
"github.com/google/cloudprober/logger"
"github.com/googleapis/google-cloud-go-testing/storage/stiface"
"github.com/googleinterns/step224-2020/hermes/probe"
"github.com/googleinterns/step224-2020/hermes/probe/metrics"
"google.golang.org/api/iterator"

pb "github.com/googleinterns/step224-2020/hermes/proto"
)

const (
FileNameFormat = "Hermes_%02d_%x"
minFileID = 1
maxFileID = 50
maxFileSizeBytes = 1000
hermesAPILatencySeconds = "hermes_api_latency_s"
)

type randomFile struct {
id int32
sizeBytes int
}

type randomFileReader struct {
sizeBytes int
// currently reading this byte
i int
rand *rand.Rand
}

// Read implements the standard Read interface, it returns a random stream of bytes generated by a pseudo-random generator.
// Argument:
// buf: a byte slice serves as an output buffer
// Returns:
// n: the number of bytes read
// err: error it should be nil when not done reading and io.EOF once the whole file contents have been read
func (r *randomFileReader) Read(buf []byte) (n int, err error) {
// check whether Read is done
if r.i >= r.sizeBytes {
return 0, io.EOF
}
b := buf
if len(buf) > r.sizeBytes-r.i {
// if the length of buffer is greater than the number of bytes left to read make the sizeBytes of the buffer match the number of bytes left to read
b = buf[:r.sizeBytes-r.i]
}
// n is now the length of the buffer
n, err = r.rand.Read(b)
if err != nil {
// in this case n = 0
return n, err
}
r.i += n
return n, err
}

func (f *randomFile) newReader() *randomFileReader {
// id will serve as a Seed and i - index of the currently read byte will be set to 0 automatically in the returned reader
return &randomFileReader{
sizeBytes: f.sizeBytes,
rand: rand.New(rand.NewSource(int64(f.id))),
}
}

func newRandomFile(id int32, sizeBytes int) (*randomFile, error) {
if id < minFileID || id > maxFileID {
return nil, fmt.Errorf("invalid argument: id = %d; want %d <= id <= %d", id, minFileID, maxFileID)
}
if sizeBytes > maxFileSizeBytes || sizeBytes <= 0 {
return nil, fmt.Errorf("invalid argument: sizeBytes = %d; want 0 < sizeBytes <= %d", sizeBytes, maxFileSizeBytes)
}
return &randomFile{id: id, sizeBytes: sizeBytes}, nil
}

func (f *randomFile) checksum() ([]byte, error) {
r := f.newReader()
h := sha1.New()
if _, err := io.Copy(h, r); err != nil {
return nil, fmt.Errorf("io.Copy: %w", err)
}
return h.Sum(nil), nil
}

func (f *randomFile) fileName() (string, error) {
checksum, err := f.checksum()
if err != nil {
return "", fmt.Errorf("{%d, %d}.checksum = nil, %w", f.id, f.sizeBytes, err)
}
return fmt.Sprintf(FileNameFormat, f.id, checksum), nil
}

// CreateFile creates and stores a file with randomized contents in the target storage system.
// Before it creates and stores a file it logs an intent to do so in the target storage system's journal.
// It verifies that the creation and storage process was successful.
// Finally, it updates the filenames map in the target's journal and record the exit status in the logger.
// Arguments:
// ctx: it carries deadlines and cancellation signals that might orinate from the main probe located in probe.go.
// target: contains information about target storage system, carries an intent log in the form of a StateJournal and it used to export metrics.
// fileID: the unique identifer of every randomFile, it cannot be repeated. It needs to be in the range [minFileID, maxFileID]. FileID 0 is reserved for a special file called the NIL file.
// client: is a storage client. It is used as an interface to interact with the target storage system.
// logger: a cloudprober logger used to record the exit status of the CreateFile operation in a target bucket. The logger passed MUST be a valid logger.
// Returns:
// error: an error string with detailed information about the status and fileID. Nil is returned when the operation is successful.
func CreateFile(ctx context.Context, target *probe.Target, fileID int32, fileSize int, client stiface.Client, logger *logger.Logger) error {
f, err := newRandomFile(fileID, fileSize)
if err != nil {
return err
}
fileName, err := f.fileName()
if err != nil {
return err
}
target.Journal.Intent = &pb.Intent{FileOperation: pb.Intent_CREATE, Filename: fileName}
var status metrics.ExitStatus
if _, ok := target.Journal.Filenames[fileID]; ok {
status = metrics.UnknownFileFound
return fmt.Errorf("CreateFile(fileID: %d).%q could not create file as file with this ID already exists", fileID, status)
}
r := f.newReader()
start := time.Now()
bucketName := target.Target.GetBucketName()
wc := client.Bucket(bucketName).Object(fileName).NewWriter(ctx)
if _, err = io.Copy(wc, r); err != nil {
switch err {
case storage.ErrBucketNotExist:
status = metrics.BucketMissing
default:
status = metrics.ProbeFailed
}
target.LatencyMetrics.APICallLatency[metrics.APICreateFile][status].Metric(hermesAPILatencySeconds).AddFloat64(time.Now().Sub(start).Seconds())
return fmt.Errorf("CreateFile(id: %d).%q: could not create file %q: %w", fileID, status, fileName, err)
}
if err := wc.Close(); err != nil {
status = metrics.WriterCloseFailed
target.LatencyMetrics.APICallLatency[metrics.APICreateFile][status].Metric(hermesAPILatencySeconds).AddFloat64(time.Now().Sub(start).Seconds())
return fmt.Errorf("Writer.Close: %w with status %q", err, status)
}
status = metrics.Success
target.LatencyMetrics.APICallLatency[metrics.APICreateFile][status].Metric(hermesAPILatencySeconds).AddFloat64(time.Now().Sub(start).Seconds())

// Verify that the file that has just been created is in fact present in the target system
fileNamePrefix := fmt.Sprintf(FileNameFormat, fileID, "")
query := &storage.Query{Prefix: fileNamePrefix}
start = time.Now()
objIter := client.Bucket(bucketName).Objects(ctx, query)
var namesFound []string
for {
obj, err := objIter.Next()
if err == iterator.Done {
break
}
if err != nil {
return fmt.Errorf("CreateFile check failed due to: %w", err)
}
namesFound = append(namesFound, obj.Name)
}
finish := time.Now()
if len(namesFound) == 0 {
target.LatencyMetrics.APICallLatency[metrics.APIListFiles][metrics.FileMissing].Metric(hermesAPILatencySeconds).AddFloat64(finish.Sub(start).Seconds())
return fmt.Errorf("CreateFile check failed no files with prefix %q found", fileNamePrefix)
}
if len(namesFound) != 1 {
return fmt.Errorf("expected exactly one file in bucket %q with prefix %q; found %d: %v", bucketName, fileNamePrefix, len(namesFound), namesFound)
}
if namesFound[0] != fileName {
fmt.Errorf("CreateFile check failed: filename matching %q prefix: %q; want %q", fileNamePrefix, namesFound[0], filaName)
}
target.LatencyMetrics.APICallLatency[metrics.APIListFiles][metrics.Success].Metric(hermesAPILatencySeconds).AddFloat64(finish.Sub(start).Seconds())

target.Journal.Filenames[fileID] = fileName
logger.Infof("Object %q added in bucket %q.", fileName, bucketName)
return nil
}
180 changes: 180 additions & 0 deletions hermes/probe/create/create_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,180 @@
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Alicja Kwiecinska GitHub: alicjakwie
//
// TODO (#72) change error types to be compatible with ProbeError

package create

import (
"context"
"fmt"
"strings"
"testing"

"github.com/golang/protobuf/proto"
"github.com/googleinterns/step224-2020/hermes/probe"
"github.com/googleinterns/step224-2020/hermes/probe/fakegcs"
"github.com/googleinterns/step224-2020/hermes/probe/metrics"

metricpb "github.com/google/cloudprober/metrics/proto"
probepb "github.com/googleinterns/step224-2020/config/proto"
journalpb "github.com/googleinterns/step224-2020/hermes/proto"
)

var fileNamePrefixLength = len(fmt.Sprintf(FileNameFormat, 0, ""))

func TestNewRandomFile(t *testing.T) {
tests := []struct {
fileID int32
fileSize int
want *randomFile
wantErr bool
}{
{51, 12, nil, true},
{0, 50, nil, true},
{3, 100, &randomFile{3, 100}, false},
{12, 100, &randomFile{12, 100}, false},
{3, 0, nil, true},
{3, 1001, nil, true},
}
for _, tc := range tests {
got, err := newRandomFile(tc.fileID, tc.fileSize)
if tc.want == nil && got != nil {
t.Errorf("{%d, %d}.newRandomFile = {%d, %d} expected nil", tc.fileID, tc.fileSize, got.id, got.sizeBytes)
}
if got == nil && tc.want != nil {
t.Errorf("{%d, %d}.newRandomFile = nil expected {%d, %d}", tc.fileID, tc.fileSize, tc.want.id, tc.want.sizeBytes)
}
if err != nil && !tc.wantErr {
t.Errorf("{%d, %d}.newRandomFile() failed and returned an unexpected error %w", tc.fileID, tc.fileSize, err)
}
if err == nil && tc.wantErr {
t.Errorf("{%d, %d}.newRandomFile() failed expected an error got nil", tc.fileID, tc.fileSize)
}
if got != nil && (got.id != tc.want.id || got.sizeBytes != tc.want.sizeBytes) {
t.Errorf("{%d, %d}.newRandomFile() = {%d,%d}, expected {%d,%d}", tc.fileID, tc.fileSize, got.id, got.sizeBytes, tc.want.id, tc.want.sizeBytes)
}

}

}

func TestFileName(t *testing.T) {
tests := []struct {
file *randomFile
want string
}{
{&randomFile{3, 100}, "Hermes_03_"},
{&randomFile{12, 100}, "Hermes_12_"},
{&randomFile{8, 20}, "Hermes_08_"},
}

for _, tc := range tests {
got, err := tc.file.fileName()
if err != nil {
t.Errorf("{%d, %d}.fileName() failed and returned an unexpected error %w", tc.file.id, tc.file.sizeBytes, err)
}
if !strings.HasPrefix(got, tc.want) {
t.Errorf("{%d, %d}.fileName() = %qchecksum expected %qchecksum", tc.file.id, tc.file.sizeBytes, got[:fileNamePrefixLength], tc.want)
}
}
}

func TestChecksum(t *testing.T) {
file := randomFile{11, 100}
checksum, err := file.checksum()
if err != nil {
t.Error(err)
}
otherFile := randomFile{13, 100}
otherChecksum, err := otherFile.checksum()
if err != nil {
t.Error(err)
}
if fmt.Sprintf("%x", checksum) == fmt.Sprintf("%x", otherChecksum) {
t.Errorf("{%d, %d}.checksum = {%d,%d}.checksum expected {%d, %d}.checksum != {%d, %d}.checksum ", file.id, file.sizeBytes, otherFile.id, otherFile.sizeBytes, file.id, file.sizeBytes, otherFile.id, otherFile.sizeBytes)

}
file = randomFile{11, 100}
checksumAgain, err := file.checksum()
if err != nil {
t.Error(err)
}
if fmt.Sprintf("%x", checksum) != fmt.Sprintf("%x", checksumAgain) {
t.Errorf("{%d, %d}.checksum != {%d, %d}.checksum expected {%d, %d}.checksum = {%d, %d}.checksum", file.id, file.sizeBytes, file.id, file.sizeBytes, file.id, file.sizeBytes, file.id, file.sizeBytes)
}
}

func TestCreateFile(t *testing.T) {
ctx := context.Background()
client := fakegcs.NewClient()
bucketName := "test_bucket_probe0"
fakeBucketHandle := client.Bucket(bucketName)
if err := fakeBucketHandle.Create(ctx, bucketName, nil); err != nil {
t.Error(err)
}
fileID := int32(6)
fileSize := 50
target := &probe.Target{
&probepb.Target{
Name: "hermes",
TargetSystem: probepb.Target_GOOGLE_CLOUD_STORAGE,
TotalSpaceAllocatedMib: int64(1000),
BucketName: "test_bucket_probe0",
},
&journalpb.StateJournal{
Filenames: make(map[int32]string),
},
&metrics.Metrics{},
}
hp := &probepb.HermesProbeDef{
ProbeName: proto.String("createfile_test"),
Targets: []*probepb.Target{
&probepb.Target{
Name: "hermes",
TargetSystem: probepb.Target_GOOGLE_CLOUD_STORAGE,
TotalSpaceAllocatedMib: int64(100),
BucketName: "test_bucket_probe0",
},
},
TargetSystem: probepb.HermesProbeDef_GCS.Enum(),
IntervalSec: proto.Int32(3600),
TimeoutSec: proto.Int32(60),
ProbeLatencyDistribution: &metricpb.Dist{
Buckets: &metricpb.Dist_ExplicitBuckets{
ExplicitBuckets: "0.1,0.2,0.4,0.6,0.8,1.6,3.2,6.4,12.8,1",
},
},
ApiCallLatencyDistribution: &metricpb.Dist{
Buckets: &metricpb.Dist_ExplicitBuckets{
ExplicitBuckets: "0.1,0.2,0.4,0.6,0.8,1.6,3.2,6.4,12.8,1",
},
},
}
probeTarget := &probepb.Target{
Name: "hermes",
TargetSystem: probepb.Target_GOOGLE_CLOUD_STORAGE,
TotalSpaceAllocatedMib: int64(100),
BucketName: "test_bucket_probe0",
}

var err error
if target.LatencyMetrics, err = metrics.NewMetrics(hp, probeTarget); err != nil {
t.Fatalf("metrics.NewMetrics(): %v", err)
}
logger := fakegcs.NewLogger(ctx).Logger
if err := CreateFile(ctx, target, fileID, fileSize, client, logger); err != nil {
t.Error(err)
}
}