Skip to content

Commit

Permalink
Sprint changes (#1355)
Browse files Browse the repository at this point in the history
* Update sprint 1.12 (#1341)

* wait for repair and increase numBlocks (#1338)

* reorder wait group done (#1340)

---------

Co-authored-by: Ebrahim Gomaa <ebrahim.gomaa.hg@gmail.com>

* fix trailing whitespace (#1343)

* Merge staging changes (#1346)

* wait for repair and increase numBlocks (#1338)

* reorder wait group done (#1340)

* hotfix / remove hard coded prefix handling of encrypted upload (#1344)

* remove hard coded prefix handling of encrypted upload

* fix for other parts of the file

---------

Co-authored-by: Hitenjain14 <57557631+Hitenjain14@users.noreply.github.com>
Co-authored-by: Ebrahim Gomaa <ebrahim.gomaa.hg@gmail.com>

* fix upload select (#1351)

* fix workdir in mobile sdk (#1345)

* fix mobile workdir

* set multi op batch size

* set option to download to disk in wasm (#1348)

* fix panic in hash chan (#1352)

* Fix merge conflict in sprint-1.12 (#1354)

* wait for repair and increase numBlocks (#1338)

* reorder wait group done (#1340)

* hotfix / remove hard coded prefix handling of encrypted upload (#1344)

* remove hard coded prefix handling of encrypted upload

* fix for other parts of the file

* increase batch size (#1349)

---------

Co-authored-by: Ebrahim Gomaa <ebrahim.gomaa.hg@gmail.com>

* repair in batches (#1347)

* repair in batches

* fix lint

* fix unit test

* fix batch size

---------

Co-authored-by: Yury <yuderbasov@gmail.com>

---------

Co-authored-by: Ebrahim Gomaa <ebrahim.gomaa.hg@gmail.com>
Co-authored-by: peterlimg <54137706+peterlimg@users.noreply.github.com>
Co-authored-by: Yury <yuderbasov@gmail.com>
  • Loading branch information
4 people authored Jan 4, 2024
1 parent ea5f217 commit d5aa3ea
Show file tree
Hide file tree
Showing 14 changed files with 375 additions and 180 deletions.
2 changes: 1 addition & 1 deletion core/version/version.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@
//====== THIS IS AUTOGENERATED FILE. DO NOT MODIFY ========

package version
const VERSIONSTR = "v1.10.0-141-gf81a4b84"
const VERSIONSTR = "v1.11-3-gad762e49"

12 changes: 12 additions & 0 deletions mobilesdk/zbox/storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -850,3 +850,15 @@ func GetRemoteFileMap(allocationID string) (string, error) {

return string(retBytes), nil
}

// SetWorkingDir set working dir
//
// ## Inputs
// - workDir
func SetWorkingDir(workDir string) {
sdk.Workdir = workDir
}

func SetMultiOpBatchSize(size int) {
sdk.SetMultiOpBatchSize(size)
}
18 changes: 14 additions & 4 deletions wasmsdk/blobber.go
Original file line number Diff line number Diff line change
Expand Up @@ -368,16 +368,26 @@ func multiDownload(allocationID, jsonMultiDownloadOptions, authTicket, callbackF
fileName := strings.Replace(path.Base(option.RemotePath), "/", "-", -1)
localPath := allocationID + "_" + fileName
option.LocalPath = localPath
statusBar := &StatusBar{wg: wg, localPath: localPath}
statusBar := &StatusBar{wg: wg}
allStatusBar[ind] = statusBar
if useCallback {
callback := js.Global().Get(callbackFuncName)
statusBar.callback = func(totalBytes, completedBytes int, filename, objURL, err string) {
callback.Invoke(totalBytes, completedBytes, filename, objURL, err)
}
}
fs, _ := sys.Files.Open(localPath)
mf, _ := fs.(*sys.MemFile)
var mf sys.File
if option.DownloadToDisk {
mf, err = jsbridge.NewFileWriter(fileName)
if err != nil {
PrintError(err.Error())
return "", err
}
} else {
statusBar.localPath = localPath
fs, _ := sys.Files.Open(localPath)
mf, _ = fs.(*sys.MemFile)
}

var downloader sdk.Downloader
if option.DownloadOp == 1 {
Expand Down Expand Up @@ -467,6 +477,7 @@ type MultiDownloadOption struct {
NumBlocks int `json:"numBlocks"`
RemoteFileName string `json:"remoteFileName"` //Required only for file download with auth ticket
RemoteLookupHash string `json:"remoteLookupHash,omitempty"` //Required only for file download with auth ticket
DownloadToDisk bool `json:"downloadToDisk"`
}

// MultiOperation - do copy, move, delete and createdir operation together
Expand Down Expand Up @@ -677,7 +688,6 @@ func uploadWithJsFuncs(allocationID, remotePath string, readChunkFuncName string
}
wg.Add(1)


fileReader := jsbridge.NewFileReader(readChunkFuncName, fileSize)

mimeType, err := zboxutil.GetFileContentType(fileReader)
Expand Down
5 changes: 3 additions & 2 deletions wasmsdk/demo/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<head>
<!-- for zcn.wasm-->
<script src="https://cdn.jsdelivr.net/gh/herumi/bls-wasm@v1.0.0/browser/bls.js"></script>
<script src="https://cdn.jsdelivr.net/gh/golang/go@go1.21.0/misc/wasm/wasm_exec.js"></script>
<script src="https://cdn.jsdelivr.net/gh/golang/go@go1.21.5/misc/wasm/wasm_exec.js"></script>
<script src="zcn.js"></script>


Expand Down Expand Up @@ -211,7 +211,7 @@ <h2>please download zcn.wasm from https://github.com/0chain/gosdk/releases/lates

let network = query.get('network')
if (!network || network == 'undefined') {
network = "dev.zus.network"
network = "test.zus.network"
}

const blockWorker = 'https://' + network + '/dns';
Expand Down Expand Up @@ -802,6 +802,7 @@ <h2>please download zcn.wasm from https://github.com/0chain/gosdk/releases/lates
remotePath: path,
downloadOp: 1,
numBlocks: 0,
downloadToDisk: true,
})
let stringifiedArray = JSON.stringify(objects);

Expand Down
88 changes: 88 additions & 0 deletions wasmsdk/jsbridge/file_writer.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
//go:build js && wasm
// +build js,wasm

package jsbridge

import (
"errors"
"io/fs"
"sync"
"syscall/js"
)

var jsFileWriterMutex sync.Mutex

type FileWriter struct {
writableStream js.Value
}

func (w *FileWriter) Write(p []byte) (int, error) {
//js.Value doesn't work in parallel invoke
jsFileWriterMutex.Lock()
defer jsFileWriterMutex.Unlock()

uint8Array := js.Global().Get("Uint8Array").New(len(p))
js.CopyBytesToJS(uint8Array, p)
_, err := Await(w.writableStream.Call("write", uint8Array))
if len(err) > 0 && !err[0].IsNull() {
return 0, errors.New("file_writer: " + err[0].String())
}
return len(p), nil
}

func (w *FileWriter) Close() error {
_, err := Await(w.writableStream.Call("close"))
if len(err) > 0 && !err[0].IsNull() {
return errors.New("file_writer: " + err[0].String())
}
return nil
}

func (w *FileWriter) Read(p []byte) (int, error) {
return 0, errors.New("file_writer: not supported")
}

func (w *FileWriter) Seek(offset int64, whence int) (int64, error) {
return 0, nil
}

func (w *FileWriter) Sync() error {
return nil
}

func (w *FileWriter) Stat() (fs.FileInfo, error) {
return nil, nil
}

func NewFileWriter(filename string) (*FileWriter, error) {
// writableStream := js.Global().Get("writableStream")
// stream, err := Await(writableStream.Call("create", filename))
// if len(err) > 0 && !err[0].IsNull() {
// return nil, errors.New("file_writer: " + err[0].String())
// }
// return &FileWriter{
// writableStream: stream[0],
// }, nil
if !js.Global().Get("window").Get("showSaveFilePicker").Truthy() || !js.Global().Get("window").Get("WritableStream").Truthy() {
return nil, errors.New("file_writer: not supported")
}

showSaveFilePicker := js.Global().Get("window").Get("showSaveFilePicker")
//create options with suggested name
options := js.Global().Get("Object").New()
options.Set("suggestedName", filename)

//request a file handle
fileHandle, err := Await(showSaveFilePicker.Invoke(options))
if len(err) > 0 && !err[0].IsNull() {
return nil, errors.New("file_writer: " + err[0].String())
}
//create a writable stream
writableStream, err := Await(fileHandle[0].Call("createWritable"))
if len(err) > 0 && !err[0].IsNull() {
return nil, errors.New("file_writer: " + err[0].String())
}
return &FileWriter{
writableStream: writableStream[0],
}, nil
}
62 changes: 43 additions & 19 deletions zboxcore/sdk/allocation.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ var (
notInitialized = errors.New("sdk_not_initialized", "Please call InitStorageSDK Init and use GetAllocation to get the allocation object")
IsWasm = false
MultiOpBatchSize = 50
Workdir string
)

const (
Expand Down Expand Up @@ -223,13 +224,16 @@ type OperationRequest struct {
DestName string // Required only for rename operation
DestPath string // Required for copy and move operation
IsUpdate bool
IsRepair bool // Required for repair operation
IsWebstreaming bool

// Required for uploads
Workdir string
FileMeta FileMeta
FileReader io.Reader
Opts []ChunkedUploadOption
Workdir string
FileMeta FileMeta
FileReader io.Reader
Mask *zboxutil.Uint128 // Required for delete repair operation
DownloadFile bool // Required for upload repair operation
Opts []ChunkedUploadOption
}

func GetReadPriceRange() (PriceRange, error) {
Expand All @@ -239,6 +243,10 @@ func GetWritePriceRange() (PriceRange, error) {
return getPriceRange("max_write_price")
}

func SetMultiOpBatchSize(size int) {
MultiOpBatchSize = size
}

func SetWasm() {
IsWasm = true
BatchSize = 5
Expand Down Expand Up @@ -372,10 +380,11 @@ func (a *Allocation) UploadFile(workdir, localpath string, remotepath string,
return a.StartChunkedUpload(workdir, localpath, remotepath, status, false, false, "", false, false)
}

func (a *Allocation) RepairFile(file sys.File, remotepath string,
status StatusCallback, mask zboxutil.Uint128, ref *fileref.FileRef) error {

func (a *Allocation) RepairFile(file sys.File, remotepath string, statusCallback StatusCallback, mask zboxutil.Uint128, ref *fileref.FileRef) *OperationRequest {
idr, _ := homedir.Dir()
if Workdir != "" {
idr = Workdir
}
mask = mask.Not().And(zboxutil.NewUint128(1).Lsh(uint64(len(a.Blobbers))).Sub64(1))
fileMeta := FileMeta{
ActualSize: ref.ActualFileSize,
Expand All @@ -387,22 +396,26 @@ func (a *Allocation) RepairFile(file sys.File, remotepath string,
if ref.EncryptedKey != "" {
opts = []ChunkedUploadOption{
WithMask(mask),
WithStatusCallback(status),
WithEncrypt(true),
WithStatusCallback(statusCallback),
WithEncryptedPoint(ref.EncryptedKeyPoint),
}
} else {
opts = []ChunkedUploadOption{
WithMask(mask),
WithStatusCallback(status),
WithStatusCallback(statusCallback),
}
}
connectionID := zboxutil.NewConnectionId()
chunkedUpload, err := CreateChunkedUpload(idr, a, fileMeta, file, false, true, false, connectionID, opts...)
if err != nil {
return err
op := &OperationRequest{
OperationType: constants.FileOperationInsert,
IsRepair: true,
RemotePath: remotepath,
Workdir: idr,
FileMeta: fileMeta,
Opts: opts,
FileReader: file,
}
return chunkedUpload.Start()
return op
}

// UpdateFileWithThumbnail [Deprecated]please use CreateChunkedUpload
Expand Down Expand Up @@ -787,7 +800,7 @@ func (a *Allocation) RepairRequired(remotepath string) (zboxutil.Uint128, zboxut
return found, deleteMask, !found.Equals(uploadMask), fileRef, nil
}

func (a *Allocation) DoMultiOperation(operations []OperationRequest) error {
func (a *Allocation) DoMultiOperation(operations []OperationRequest, opts ...MultiOperationOption) error {
if len(operations) == 0 {
return nil
}
Expand All @@ -808,6 +821,9 @@ func (a *Allocation) DoMultiOperation(operations []OperationRequest) error {
consensusThresh: a.consensusThreshold,
fullconsensus: a.fullconsensus,
}
for _, opt := range opts {
opt(&mo)
}
previousPaths := make(map[string]bool)
connectionErrors := make([]error, len(mo.allocationObj.Blobbers))

Expand Down Expand Up @@ -844,6 +860,11 @@ func (a *Allocation) DoMultiOperation(operations []OperationRequest) error {
break
}
op := operations[i]
op.RemotePath = strings.TrimSpace(op.RemotePath)
if op.FileMeta.RemotePath != "" {
op.FileMeta.RemotePath = strings.TrimSpace(op.FileMeta.RemotePath)
op.FileMeta.RemoteName = strings.TrimSpace(op.FileMeta.RemoteName)
}
remotePath := op.RemotePath
parentPaths := GenerateParentPaths(remotePath)

Expand All @@ -870,13 +891,16 @@ func (a *Allocation) DoMultiOperation(operations []OperationRequest) error {
operation = NewMoveOperation(op.RemotePath, op.DestPath, mo.operationMask, mo.maskMU, mo.consensusThresh, mo.fullconsensus, mo.ctx)

case constants.FileOperationInsert:
operation, newConnectionID, err = NewUploadOperation(op.Workdir, mo.allocationObj, mo.connectionID, op.FileMeta, op.FileReader, false, op.IsWebstreaming, op.Opts...)
operation, newConnectionID, err = NewUploadOperation(op.Workdir, mo.allocationObj, mo.connectionID, op.FileMeta, op.FileReader, false, op.IsWebstreaming, op.IsRepair, op.DownloadFile, op.Opts...)

case constants.FileOperationDelete:
operation = NewDeleteOperation(op.RemotePath, mo.operationMask, mo.maskMU, mo.consensusThresh, mo.fullconsensus, mo.ctx)

if op.Mask != nil {
operation = NewDeleteOperation(op.RemotePath, *op.Mask, mo.maskMU, mo.consensusThresh, mo.fullconsensus, mo.ctx)
} else {
operation = NewDeleteOperation(op.RemotePath, mo.operationMask, mo.maskMU, mo.consensusThresh, mo.fullconsensus, mo.ctx)
}
case constants.FileOperationUpdate:
operation, newConnectionID, err = NewUploadOperation(op.Workdir, mo.allocationObj, mo.connectionID, op.FileMeta, op.FileReader, true, op.IsWebstreaming, op.Opts...)
operation, newConnectionID, err = NewUploadOperation(op.Workdir, mo.allocationObj, mo.connectionID, op.FileMeta, op.FileReader, true, op.IsWebstreaming, op.IsRepair, op.DownloadFile, op.Opts...)

case constants.FileOperationCreateDir:
operation = NewDirOperation(op.RemotePath, mo.operationMask, mo.maskMU, mo.consensusThresh, mo.fullconsensus, mo.ctx)
Expand Down
Loading

0 comments on commit d5aa3ea

Please sign in to comment.