Skip to content

Commit

Permalink
fix(controller): prevent closing input in aliyun oss
Browse files Browse the repository at this point in the history
  • Loading branch information
jialeicui committed Feb 7, 2023
1 parent 5708997 commit a39a16e
Show file tree
Hide file tree
Showing 3 changed files with 97 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ var record = reader.read();
this.recordMap.put(record.get(this.schema.getKeyColumn()), record);
}
}
} catch (IOException e) {
} catch (IOException | RuntimeException e) {
throw new SwProcessException(ErrorType.DATASTORE, "failed to load " + this.tableName, e);
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
/*
* Copyright 2022 Starwhale, Inc. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package ai.starwhale.mlops.storage;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;

public class NopCloserInputStream extends InputStream {
private final InputStream inputStream;

public NopCloserInputStream(InputStream inputStream) {
this.inputStream = inputStream;
}

@Override
public int read() throws IOException {
return this.inputStream.read();
}

@Override
public int read(byte[] b) throws IOException {
return inputStream.read(b);
}

@Override
public int read(byte[] b, int off, int len) throws IOException {
return inputStream.read(b, off, len);
}

@Override
public byte[] readAllBytes() throws IOException {
return inputStream.readAllBytes();
}

@Override
public byte[] readNBytes(int len) throws IOException {
return inputStream.readNBytes(len);
}

@Override
public int readNBytes(byte[] b, int off, int len) throws IOException {
return inputStream.readNBytes(b, off, len);
}

@Override
public long skip(long n) throws IOException {
return inputStream.skip(n);
}

@Override
public int available() throws IOException {
return inputStream.available();
}

@Override
public void close() throws IOException {
}

@Override
public void mark(int readLimit) {
inputStream.mark(readLimit);
}

@Override
public void reset() throws IOException {
inputStream.reset();
}

@Override
public boolean markSupported() {
return inputStream.markSupported();
}

@Override
public long transferTo(OutputStream out) throws IOException {
return inputStream.transferTo(out);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package ai.starwhale.mlops.storage.aliyun;

import ai.starwhale.mlops.storage.LengthAbleInputStream;
import ai.starwhale.mlops.storage.NopCloserInputStream;
import ai.starwhale.mlops.storage.StorageAccessService;
import ai.starwhale.mlops.storage.StorageObjectInfo;
import ai.starwhale.mlops.storage.s3.S3Config;
Expand Down Expand Up @@ -76,7 +77,8 @@ public StorageObjectInfo head(String path) throws IOException {
public void put(String path, InputStream inputStream, long size) throws IOException {
var meta = new ObjectMetadata();
meta.setContentLength(size);
this.ossClient.putObject(this.bucket, path, inputStream, meta);
var is = new NopCloserInputStream(inputStream);
this.ossClient.putObject(this.bucket, path, is, meta);
}

@Override
Expand Down

0 comments on commit a39a16e

Please sign in to comment.