Skip to content

ARROW-287: Make nullable vectors use a BitVecor instead of UInt1Vector for bits #134

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

ARROW-287: Make nullable vectors use a BitVecor instead of UInt1Vector for bits #134

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
15fde9d
ARROW-287: Make nullable vectors use a BitVecor instead of UInt1Vecto…
julienledem Sep 9, 2016
d4e5084
add nullable vector test that verifies Bit based buffers
julienledem Sep 13, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions java/vector/src/main/codegen/templates/NullableValueVectors.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
private final String valuesField = "$values$";
private final Field field;

final UInt1Vector bits = new UInt1Vector(bitsField, allocator);
final BitVector bits = new BitVector(bitsField, allocator);
final ${valuesName} values;

private final Mutator mutator;
Expand Down Expand Up @@ -446,7 +446,7 @@ public void copyFromSafe(int fromIndex, int thisIndex, Nullable${minor.class}Vec
}

public final class Accessor extends BaseDataValueVector.BaseAccessor <#if type.major = "VarLen">implements VariableWidthVector.VariableWidthAccessor</#if> {
final UInt1Vector.Accessor bAccessor = bits.getAccessor();
final BitVector.Accessor bAccessor = bits.getAccessor();
final ${valuesName}.Accessor vAccessor = values.getAccessor();

/**
Expand Down Expand Up @@ -545,7 +545,7 @@ public void setIndexDefined(int index){
public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value) {
setCount++;
final ${valuesName}.Mutator valuesMutator = values.getMutator();
final UInt1Vector.Mutator bitsMutator = bits.getMutator();
final BitVector.Mutator bitsMutator = bits.getMutator();
<#if type.major == "VarLen">
for (int i = lastSet + 1; i < index; i++) {
valuesMutator.set(i, emptyByteArray);
Expand Down
16 changes: 14 additions & 2 deletions java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@
*/
package org.apache.arrow.vector;

import io.netty.buffer.ArrowBuf;

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.complex.reader.FieldReader;
Expand All @@ -29,6 +27,8 @@
import org.apache.arrow.vector.util.OversizedAllocationException;
import org.apache.arrow.vector.util.TransferPair;

import io.netty.buffer.ArrowBuf;

/**
* Bit implements a vector of bit-width values. Elements in the vector are accessed by position from the logical start
* of the vector. The width of each element is 1 bit. The equivalent Java primitive is an int containing the value '0'
Expand Down Expand Up @@ -435,6 +435,18 @@ public final void generateTestData(int values) {
setValueCount(values);
}

public void generateTestDataAlt(int size) {
setValueCount(size);
boolean even = true;
final int valueCount = getAccessor().getValueCount();
for(int i = 0; i < valueCount; i++, even = !even) {
if(even){
set(i, (byte) 1);
}else{
set(i, (byte) 0);
}
}
}
}

@Override
Expand Down
6 changes: 3 additions & 3 deletions java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,9 @@
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.AddOrGetResult;
import org.apache.arrow.vector.BaseDataValueVector;
import org.apache.arrow.vector.BitVector;
import org.apache.arrow.vector.BufferBacked;
import org.apache.arrow.vector.FieldVector;
import org.apache.arrow.vector.UInt1Vector;
import org.apache.arrow.vector.UInt4Vector;
import org.apache.arrow.vector.ValueVector;
import org.apache.arrow.vector.ZeroVector;
Expand All @@ -55,7 +55,7 @@
public class ListVector extends BaseRepeatedValueVector implements FieldVector {

final UInt4Vector offsets;
final UInt1Vector bits;
final BitVector bits;
private final List<BufferBacked> innerVectors;
private Mutator mutator = new Mutator();
private Accessor accessor = new Accessor();
Expand All @@ -65,7 +65,7 @@ public class ListVector extends BaseRepeatedValueVector implements FieldVector {

public ListVector(String name, BufferAllocator allocator, CallBack callBack) {
super(name, allocator);
this.bits = new UInt1Vector("$bits$", allocator);
this.bits = new BitVector("$bits$", allocator);
this.offsets = getOffsetVector();
this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(bits, offsets));
this.writer = new UnionListWriter(this);
Expand Down
8 changes: 4 additions & 4 deletions java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,10 @@

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.BaseDataValueVector;
import org.apache.arrow.vector.BitVector;
import org.apache.arrow.vector.BufferBacked;
import org.apache.arrow.vector.FieldVector;
import org.apache.arrow.vector.NullableVectorDefinitionSetter;
import org.apache.arrow.vector.UInt1Vector;
import org.apache.arrow.vector.ValueVector;
import org.apache.arrow.vector.complex.impl.NullableMapReaderImpl;
import org.apache.arrow.vector.complex.reader.FieldReader;
Expand All @@ -45,7 +45,7 @@ public class NullableMapVector extends MapVector implements FieldVector {

private final NullableMapReaderImpl reader = new NullableMapReaderImpl(this);

protected final UInt1Vector bits;
protected final BitVector bits;

private final List<BufferBacked> innerVectors;

Expand All @@ -54,7 +54,7 @@ public class NullableMapVector extends MapVector implements FieldVector {

public NullableMapVector(String name, BufferAllocator allocator, CallBack callBack) {
super(name, checkNotNull(allocator), callBack);
this.bits = new UInt1Vector("$bits$", allocator);
this.bits = new BitVector("$bits$", allocator);
this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(bits));
this.accessor = new Accessor();
this.mutator = new Mutator();
Expand Down Expand Up @@ -186,7 +186,7 @@ public boolean allocateNewSafe() {
return success;
}
public final class Accessor extends MapVector.Accessor {
final UInt1Vector.Accessor bAccessor = bits.getAccessor();
final BitVector.Accessor bAccessor = bits.getAccessor();

@Override
public Object getObject(int index) {
Expand Down
67 changes: 63 additions & 4 deletions java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,17 +17,23 @@
*/
package org.apache.arrow.vector;

import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import java.nio.charset.Charset;
import java.util.List;

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.schema.TypeLayout;
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.util.OversizedAllocationException;
import org.apache.arrow.vector.types.pojo.Field;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.nio.charset.Charset;

import static org.junit.Assert.*;
import io.netty.buffer.ArrowBuf;


public class TestValueVector {
Expand Down Expand Up @@ -223,6 +229,59 @@ public void testNullableFloat() {
}
}

@Test
public void testNullableInt() {
// Create a new value vector for 1024 integers
try (final NullableIntVector vector = (NullableIntVector) MinorType.INT.getNewVector(EMPTY_SCHEMA_PATH, allocator, null)) {
final NullableIntVector.Mutator m = vector.getMutator();
vector.allocateNew(1024);

// Put and set a few values.
m.set(0, 1);
m.set(1, 2);
m.set(100, 3);
m.set(1022, 4);
m.set(1023, 5);

m.setValueCount(1024);

final NullableIntVector.Accessor accessor = vector.getAccessor();
assertEquals(1, accessor.get(0));
assertEquals(2, accessor.get(1));
assertEquals(3, accessor.get(100));
assertEquals(4, accessor.get(1022));
assertEquals(5, accessor.get(1023));

// Ensure null values.
assertTrue(vector.getAccessor().isNull(3));

Field field = vector.getField();
TypeLayout typeLayout = field.getTypeLayout();

List<ArrowBuf> buffers = vector.getFieldBuffers();

assertEquals(2, typeLayout.getVectors().size());
assertEquals(2, buffers.size());

ArrowBuf validityVectorBuf = buffers.get(0);
assertEquals(128, validityVectorBuf.readableBytes());
assertEquals(3, validityVectorBuf.getByte(0)); // 1st and second bit defined
for (int i = 1; i < 12; i++) {
assertEquals(0, validityVectorBuf.getByte(i)); // nothing defined until 100
}
assertEquals(16, validityVectorBuf.getByte(12)); // 100th bit is defined (12 * 8 + 4)
for (int i = 13; i < 127; i++) {
assertEquals(0, validityVectorBuf.getByte(i)); // nothing defined between 100th and 1022nd
}
assertEquals(-64, validityVectorBuf.getByte(127)); // 1022nd and 1023rd bit defined

vector.allocateNew(2048);
// vector has been erased
assertTrue(vector.getAccessor().isNull(0));
}
}


@Test
public void testBitVector() {
// Create a new value vector for 1024 integers
Expand Down