Skip to content

ARROW-400: set struct length on load #234

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wesm wants to merge 2 commits into from
Closed

ARROW-400: set struct length on load #234

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
741ff71
ARROW-400: set struct length on json load
julienledem Dec 9, 2016
f516ba1
Add unit test for ARROW-400
wesm Dec 9, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
237 changes: 237 additions & 0 deletions integration/data/struct_example.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,237 @@
{
"schema": {
"fields": [
{
"name": "struct_nullable",
"type": {
"name": "struct"
},
"nullable": true,
"children": [
{
"name": "f1",
"type": {
"name": "int",
"isSigned": true,
"bitWidth": 32
},
"nullable": true,
"children": [],
"typeLayout": {
"vectors": [
{
"type": "VALIDITY",
"typeBitWidth": 1
},
{
"type": "DATA",
"typeBitWidth": 32
}
]
}
},
{
"name": "f2",
"type": {
"name": "utf8"
},
"nullable": true,
"children": [],
"typeLayout": {
"vectors": [
{
"type": "VALIDITY",
"typeBitWidth": 1
},
{
"type": "OFFSET",
"typeBitWidth": 32
},
{
"type": "DATA",
"typeBitWidth": 8
}
]
}
}
],
"typeLayout": {
"vectors": [
{
"type": "VALIDITY",
"typeBitWidth": 1
}
]
}
}
]
},
"batches": [
{
"count": 7,
"columns": [
{
"name": "struct_nullable",
"count": 7,
"VALIDITY": [
0,
1,
1,
1,
0,
1,
0
],
"children": [
{
"name": "f1",
"count": 7,
"VALIDITY": [
1,
0,
1,
1,
1,
0,
0
],
"DATA": [
1402032511,
290876774,
137773603,
410361374,
1959836418,
1995074679,
-163525262
]
},
{
"name": "f2",
"count": 7,
"VALIDITY": [
0,
1,
1,
1,
0,
1,
0
],
"OFFSET": [
0,
0,
7,
14,
21,
21,
28,
28
],
"DATA": [
"",
"MhRNxD4",
"3F9HBxK",
"aVd88fp",
"",
"3loZrRf",
""
]
}
]
}
]
},
{
"count": 10,
"columns": [
{
"name": "struct_nullable",
"count": 10,
"VALIDITY": [
0,
1,
1,
0,
1,
0,
0,
1,
1,
1
],
"children": [
{
"name": "f1",
"count": 10,
"VALIDITY": [
0,
0,
0,
0,
0,
0,
1,
0,
0,
0
],
"DATA": [
-2041500147,
1715692943,
-35444996,
1425496657,
112765084,
1760754983,
413888857,
2039738337,
-1924327700,
670528518
]
},
{
"name": "f2",
"count": 10,
"VALIDITY": [
1,
0,
0,
1,
1,
1,
1,
1,
1,
0
],
"OFFSET": [
0,
7,
7,
7,
14,
21,
28,
35,
42,
49,
49
],
"DATA": [
"AS5oARE",
"",
"",
"JGdagcX",
"78SLiRw",
"vbGf7OY",
"5uh5fTs",
"0ilsf82",
"LjS9MbU",
""
]
}
]
}
]
}
]
}
2 changes: 1 addition & 1 deletion java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ public class MapVector extends AbstractMapVector {
private final SingleMapReaderImpl reader = new SingleMapReaderImpl(this);
private final Accessor accessor = new Accessor();
private final Mutator mutator = new Mutator();
int valueCount;
public int valueCount;

public MapVector(String name, BufferAllocator allocator, CallBack callBack) {
super(name, allocator, callBack);
Expand Down
4 changes: 4 additions & 0 deletions java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
import org.apache.arrow.vector.ValueVector.Mutator;
import org.apache.arrow.vector.VarCharVector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.complex.NullableMapVector;
import org.apache.arrow.vector.schema.ArrowVectorType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.Schema;
Expand Down Expand Up @@ -153,6 +154,9 @@ private void readVector(Field field, FieldVector vector) throws JsonParseExcepti
}
readToken(END_ARRAY);
}
if (vector instanceof NullableMapVector) {
((NullableMapVector)vector).valueCount = count;
}
}
readToken(END_OBJECT);
}
Expand Down
20 changes: 20 additions & 0 deletions java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,13 @@
import java.io.IOException;

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.FieldVector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.complex.MapVector;
import org.apache.arrow.vector.complex.NullableMapVector;
import org.apache.arrow.vector.file.BaseFileTest;
import org.apache.arrow.vector.types.pojo.Schema;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -117,4 +119,22 @@ public void testWriteReadUnionJSON() throws IOException {
}
}

@Test
public void testSetStructLength() throws IOException {
File file = new File("../../integration/data/struct_example.json");
try (
BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
) {
JsonFileReader reader = new JsonFileReader(file, readerAllocator);
Schema schema = reader.start();
LOGGER.debug("reading schema: " + schema);

// initialize vectors
try (VectorSchemaRoot root = reader.read();) {
FieldVector vector = root.getVector("struct_nullable");
Assert.assertEquals(7, vector.getAccessor().getValueCount());
}
}
}

}