fix extraneous bits

Author: julienledem

java/vector/src/main/java/org/apache/arrow/vector/BitVector.java

@@ -54,20 +54,27 @@ public void load(ArrowFieldNode fieldNode, ArrowBuf data) {
     // When the vector is all nulls or all defined, the content of the buffer can be omitted
     if (data.readableBytes() == 0 && fieldNode.getLength() != 0) {
       data.release();
-      allocateNew(fieldNode.getLength());
-      int n = getSizeFromCount(fieldNode.getLength());
+      int count = fieldNode.getLength();
+      allocateNew(count);
+      int n = getSizeFromCount(count);
       if (fieldNode.getNullCount() == 0) {
         // all defined
         // create an all 1s buffer
-        for (int i = 0; i < n; ++i) {
+        // set full bytes
+        int fullBytesCount = count / 8;
+        for (int i = 0; i < fullBytesCount; ++i) {
           this.data.setByte(i, 0xFF);
         }
+        int remainder = count % 8;
+        // set remaining bits
+        if (remainder > 0) {
+          byte bitMask = (byte) (0xFFL >>> ((8 - remainder) & 7));;
+          this.data.setByte(fullBytesCount, bitMask);
+        }
       } else if (fieldNode.getNullCount() == fieldNode.getLength()) {
         // all null
         // create an all 0s buffer
-        for (int i = 0; i < n; ++i) {
-          this.data.setByte(i, 0x00);
-        }
+        zeroVector();
       } else {
         throw new IllegalArgumentException("The buffer can be empty only if there's no data or it's all null or all defined");
       }

java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java

@@ -19,8 +19,8 @@
 
 import static java.util.Arrays.asList;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -128,17 +128,30 @@ public void testLoadEmptyValidityBuffer() throws IOException {
 
       vectorLoader.load(recordBatch);
 
-      FieldReader intDefinedReader = newRoot.getVector("intDefined").getReader();
-      FieldReader intNullReader = newRoot.getVector("intNull").getReader();
+      NullableIntVector intDefinedVector = (NullableIntVector)newRoot.getVector("intDefined");
+      NullableIntVector intNullVector = (NullableIntVector)newRoot.getVector("intNull");
       for (int i = 0; i < count; i++) {
-        intDefinedReader.setPosition(i);
-        intNullReader.setPosition(i);
-        Integer defined = intDefinedReader.readInteger();
-        assertNotNull("#" + i, defined);
-        assertEquals("#" + i, i, defined.intValue());
-        Integer nullVal = intNullReader.readInteger();
-        assertNull("#" + i, nullVal);
+        assertFalse("#" + i, intDefinedVector.getAccessor().isNull(i));
+        assertEquals("#" + i, i, intDefinedVector.getAccessor().get(i));
+        assertTrue("#" + i, intNullVector.getAccessor().isNull(i));
       }
+      intDefinedVector.getMutator().setSafe(count + 10, 1234);
+      assertTrue(intDefinedVector.getAccessor().isNull(count + 1));
+      // empty slots should still default to unset
+      intDefinedVector.getMutator().setSafe(count + 1, 789);
+      assertFalse(intDefinedVector.getAccessor().isNull(count + 1));
+      assertEquals(789, intDefinedVector.getAccessor().get(count + 1));
+      assertTrue(intDefinedVector.getAccessor().isNull(count));
+      assertTrue(intDefinedVector.getAccessor().isNull(count + 2));
+      assertTrue(intDefinedVector.getAccessor().isNull(count + 3));
+      assertTrue(intDefinedVector.getAccessor().isNull(count + 4));
+      assertTrue(intDefinedVector.getAccessor().isNull(count + 5));
+      assertTrue(intDefinedVector.getAccessor().isNull(count + 6));
+      assertTrue(intDefinedVector.getAccessor().isNull(count + 7));
+      assertTrue(intDefinedVector.getAccessor().isNull(count + 8));
+      assertTrue(intDefinedVector.getAccessor().isNull(count + 9));
+      assertFalse(intDefinedVector.getAccessor().isNull(count + 10));
+      assertEquals(1234, intDefinedVector.getAccessor().get(count + 10));
     } finally {
       values.release();
     }