Closed
Description
Hi,
We use prestosql 324 but HadoopIllegalArgumentException occurred in the table with HDFS erasure coding.
Any suggestion?
DDL
CREATE EXTERNAL TABLE hoge.piyo (
hh STRING,
mmss STRING,
log STRING COMMENT 'json'
)
PARTITIONED BY (service STRING, yyyymmdd STRING)
STORED AS RCFILE
LOCATION
'hdfs://...'
;
query
SELECT count(*) FROM hoge.piyo WHERE ... AND yyyymmdd='20190930' LIMIT 100
stacktrace
io.prestosql.spi.PrestoException: Failed to read RC file: hdfs://.../yyyymmdd=20190930/000044_0
at io.prestosql.plugin.hive.rcfile.RcFilePageSource.getNextPage(RcFilePageSource.java:152)
at io.prestosql.plugin.hive.HivePageSource.getNextPage(HivePageSource.java:244)
at io.prestosql.operator.TableScanOperator.getOutput(TableScanOperator.java:286)
at io.prestosql.operator.Driver.processInternal(Driver.java:379)
at io.prestosql.operator.Driver.lambda$processFor$8(Driver.java:283)
at io.prestosql.operator.Driver.tryWithLock(Driver.java:675)
at io.prestosql.operator.Driver.processFor(Driver.java:276)
at io.prestosql.execution.SqlTaskExecution$DriverSplitRunner.processFor(SqlTaskExecution.java:1075)
at io.prestosql.execution.executor.PrioritizedSplitRunner.process(PrioritizedSplitRunner.java:163)
at io.prestosql.execution.executor.TaskExecutor$TaskRunner.run(TaskExecutor.java:484)
at io.prestosql.$gen.Presto_324____20191204_075334_2.run(Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.hadoop.HadoopIllegalArgumentException: Invalid buffer, not of length 951251
at org.apache.hadoop.io.erasurecode.rawcoder.ByteBufferDecodingState.checkOutputBuffers(ByteBufferDecodingState.java:137)
at org.apache.hadoop.io.erasurecode.rawcoder.ByteBufferDecodingState.<init>(ByteBufferDecodingState.java:48)
at org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder.decode(RawErasureDecoder.java:86)
at org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder.decode(RawErasureDecoder.java:170)
at org.apache.hadoop.hdfs.StripeReader.decodeAndFillBuffer(StripeReader.java:433)
at org.apache.hadoop.hdfs.PositionStripeReader.decode(PositionStripeReader.java:74)
at org.apache.hadoop.hdfs.StripeReader.readStripe(StripeReader.java:390)
at org.apache.hadoop.hdfs.DFSStripedInputStream.fetchBlockByteRange(DFSStripedInputStream.java:507)
at org.apache.hadoop.hdfs.DFSInputStream.pread(DFSInputStream.java:1360)
at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:1324)
at org.apache.hadoop.fs.FSInputStream.readFully(FSInputStream.java:121)
at org.apache.hadoop.fs.FSDataInputStream.readFully(FSDataInputStream.java:111)
at org.apache.hadoop.fs.FSDataInputStream.readFully(FSDataInputStream.java:111)
at io.prestosql.plugin.hive.rcfile.HdfsRcFileDataSource.readFully(HdfsRcFileDataSource.java:82)
at io.prestosql.rcfile.RcFileReader$DataSourceSliceLoader.load(RcFileReader.java:693)
at io.prestosql.rcfile.RcFileReader$DataSourceSliceLoader.load(RcFileReader.java:667)
at io.airlift.slice.ChunkedSliceInput$InternalLoader.load(ChunkedSliceInput.java:383)
at io.airlift.slice.ChunkedSliceInput.ensureAvailable(ChunkedSliceInput.java:111)
at io.airlift.slice.ChunkedSliceInput.readBytes(ChunkedSliceInput.java:259)
at io.airlift.slice.ChunkedSliceInput.read(ChunkedSliceInput.java:242)
at com.google.common.io.ByteStreams.skipUpTo(ByteStreams.java:819)
at com.google.common.io.ByteStreams.skipFully(ByteStreams.java:795)
at io.prestosql.rcfile.RcFileReader.advance(RcFileReader.java:410)
at io.prestosql.plugin.hive.rcfile.RcFilePageSource.getNextPage(RcFilePageSource.java:122)
... 13 more
Metadata
Metadata
Assignees
Labels
No labels