Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Java8-11 changes consistent with 17 #576

Merged
merged 12 commits into from
Aug 15, 2024
11 changes: 1 addition & 10 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,6 @@
-->

[![Maven Central](https://maven-badges.herokuapp.com/maven-central/org.apache.datasketches/datasketches-java/badge.svg)](https://maven-badges.herokuapp.com/maven-central/org.apache.datasketches/datasketches-java)
[![Language grade: Java](https://img.shields.io/lgtm/grade/java/g/apache/datasketches-java.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/apache/datasketches-java/context:java)
[![Total alerts](https://img.shields.io/lgtm/alerts/g/apache/datasketches-java.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/apache/datasketches-java/alerts/)
[![Coverage Status](https://coveralls.io/repos/github/apache/datasketches-java/badge.svg)](https://coveralls.io/github/apache/datasketches-java)

=================
Expand Down Expand Up @@ -48,22 +46,15 @@ However, if your application also directly relies on the APIs of the *datasketch
you may need additional JVM arguments.
Please refer to the [datasketches-memory README](https://github.com/apache/datasketches-memory/blob/master/README.md) for details.

If your application uses Maven, you can also use the *pom.xml* of this component as an example of how to automatically
configure the JVM arguments for compilation and testing based on the version of the JDK.
If your application uses Maven, you can also use the *pom.xml* of this component as an example of how to automatically configure the JVM arguments for compilation and testing based on the version of the JDK.

### Recommended Build Tool
This DataSketches component is structured as a Maven project and Maven is the recommended Build Tool.

There are two types of tests: normal unit tests and tests run by the strict profile.

To run normal unit tests:

$ mvn clean test

To run the strict profile tests (only supported in Java 8):

$ mvn clean test -P strict

To install jars built from the downloaded source:

$ mvn clean install -DskipTests=true
Expand Down
55 changes: 26 additions & 29 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -83,9 +83,7 @@ under the License.
</developers>

<properties>
<!-- UNIQUE FOR THIS JAVA COMPONENT -->
<datasketches-memory.version>2.2.1</datasketches-memory.version>
<!-- END:UNIQUE FOR THIS JAVA COMPONENT -->
<datasketches-memory.version>3.0.0</datasketches-memory.version>

<!-- Test -->
<testng.version>7.5.1</testng.version>
Expand All @@ -95,8 +93,8 @@ under the License.
<testng.check-cpp-historical-files>check_cpp_historical_files</testng.check-cpp-historical-files>

<!-- System-wide properties -->
<maven.version>3.6.3</maven.version> <!-- may override parent, used in enforcer plugin -->
<java.version>1.8</java.version>
<maven.version>3.6.3</maven.version>
<java.version>8</java.version>
<maven.compiler.source>${java.version}</maven.compiler.source>
<maven.compiler.target>${java.version}</maven.compiler.target>
<argLine>-Xmx4g -Duser.language=en -Duser.country=US -Dfile.encoding=UTF-8</argLine>
Expand All @@ -106,41 +104,40 @@ under the License.
<project.reporting.outputEncoding>${charset.encoding}</project.reporting.outputEncoding>
<maven.build.timestamp.format>yyyy-MM-dd'T'HH-mm-ss'Z'</maven.build.timestamp.format>

<!-- org.mojohaus plugins: used for checking & updating dependency versions -->
<versions-maven-plugin.version>2.16.2</versions-maven-plugin.version>

<!-- Maven Plugins -->
<maven-assembly-plugin.version>3.7.1</maven-assembly-plugin.version> <!-- may override parent -->
<maven-compiler-plugin.version>3.13.0</maven-compiler-plugin.version> <!-- may override parent -->
<maven-deploy-plugin.version>3.1.1</maven-deploy-plugin.version> <!-- may override parent -->
<maven-enforcer-plugin.version>3.4.1</maven-enforcer-plugin.version> <!-- may override parent -->
<maven-gpg-plugin.version>3.2.3</maven-gpg-plugin.version> <!-- may override parent -->
<maven-jar-plugin.version>3.4.0</maven-jar-plugin.version> <!-- may override parent -->
<maven-javadoc-plugin.version>3.6.3</maven-javadoc-plugin.version> <!-- may override parent -->
<maven-release-plugin.version>3.0.1</maven-release-plugin.version> <!-- may override parent -->
<maven-remote-resources-plugin.version>3.2.0</maven-remote-resources-plugin.version> <!-- may override parent -->
<maven-source-plugin.version>3.3.1</maven-source-plugin.version> <!-- may override parent -->
<maven-surefire-failsafe-plugins.version>3.2.5</maven-surefire-failsafe-plugins.version><!-- for surefire, failsafe and surefire-report, may override parent-->
<!-- Apache Plugins -->
<apache-rat-plugin.version>0.16.1</apache-rat-plugin.version> <!-- may override parent -->
<!-- org.jacoco Maven Plugins -->
<jacoco-maven-plugin.version>0.8.12</jacoco-maven-plugin.version>
<!-- org.eluder Maven Plugins -->
<!-- org.apache.maven plugins -->
<maven-assembly-plugin.version>3.7.1</maven-assembly-plugin.version>
<maven-compiler-plugin.version>3.13.0</maven-compiler-plugin.version>
<maven-deploy-plugin.version>3.1.2</maven-deploy-plugin.version>
<maven-enforcer-plugin.version>3.4.1</maven-enforcer-plugin.version>
<maven-gpg-plugin.version>3.2.3</maven-gpg-plugin.version>
<maven-jar-plugin.version>3.4.1</maven-jar-plugin.version>
<maven-javadoc-plugin.version>3.6.3</maven-javadoc-plugin.version>
<maven-release-plugin.version>3.0.1</maven-release-plugin.version>
<maven-remote-resources-plugin.version>3.2.0</maven-remote-resources-plugin.version>
<maven-source-plugin.version>3.3.1</maven-source-plugin.version>
<maven-surefire-failsafe-plugins.version>3.2.5</maven-surefire-failsafe-plugins.version> <!-- for surefire, failsafe and surefire-report-->
<!-- com.github plugins -->
<git-commit-id-plugin.version>4.9.10</git-commit-id-plugin.version>
<!-- org.apache.creadur plugins -->
<apache-rat-plugin.version>0.16.1</apache-rat-plugin.version>
<!-- org.eluder maven plugins -->
<coveralls-repo-token></coveralls-repo-token>
<coveralls-maven-plugin.version>4.3.0</coveralls-maven-plugin.version>
<!-- org.jacoco maven plugins -->
<jacoco-maven-plugin.version>0.8.12</jacoco-maven-plugin.version>
<!-- org.mojohaus plugins -->
<versions-maven-plugin.version>2.16.2</versions-maven-plugin.version>
<!-- other -->
<lifecycle-mapping.version>1.0.0</lifecycle-mapping.version> <!-- not used -->
<git-commit-id-plugin.version>4.9.10</git-commit-id-plugin.version>
<lifecycle-mapping.version>1.0.0</lifecycle-mapping.version>
</properties>

<dependencies>
<!-- UNIQUE FOR THIS JAVA COMPONENT -->
<dependency>
<groupId>org.apache.datasketches</groupId>
<artifactId>datasketches-memory</artifactId>
<version>${datasketches-memory.version}</version>
</dependency>
<!-- END: UNIQUE FOR THIS JAVA COMPONENT -->

<!-- Test Scope -->
<dependency>
<groupId>org.testng</groupId>
Expand Down
3 changes: 2 additions & 1 deletion src/main/java/org/apache/datasketches/common/Util.java
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,8 @@ public static String zeroPad(final String s, final int fieldLength) {

/**
* Prepend or postpend the given string with the given character to fill the given field length.
* If the given string is equal to or greater than the given field length, it will be returned without modification.
* If the given string is equal to or greater than the given field length, it will be returned
* without modification.
* @param s the given string
* @param fieldLength the desired field length
* @param padChar the desired pad character
Expand Down
13 changes: 5 additions & 8 deletions src/main/java/org/apache/datasketches/theta/Sketch.java
Original file line number Diff line number Diff line change
Expand Up @@ -292,14 +292,11 @@ public double getLowerBound(final int numStdDev) {

/**
* Returns the maximum number of storage bytes required for a CompactSketch with the given
* number of actual entries. Note that this assumes the worse case of the sketch in
* estimation mode, which requires storing theta and count.
* @param numberOfEntries the actual number of entries stored with the CompactSketch.
* number of actual entries.
* @param numberOfEntries the actual number of retained entries stored in the sketch.
* @return the maximum number of storage bytes required for a CompactSketch with the given number
* of entries.
* @deprecated as a public method. Use {@link #getCompactSketchMaxBytes(int) instead}
* of retained entries.
*/
@Deprecated
public static int getMaxCompactSketchBytes(final int numberOfEntries) {
if (numberOfEntries == 0) { return 8; }
if (numberOfEntries == 1) { return 16; }
Expand All @@ -314,8 +311,8 @@ public static int getMaxCompactSketchBytes(final int numberOfEntries) {
* nomEntries.
*/
public static int getCompactSketchMaxBytes(final int lgNomEntries) {
return (int)((2 << lgNomEntries) * ThetaUtil.REBUILD_THRESHOLD)
+ Family.QUICKSELECT.getMaxPreLongs() * Long.BYTES;
return (int)((2 << lgNomEntries) * ThetaUtil.REBUILD_THRESHOLD
+ Family.QUICKSELECT.getMaxPreLongs()) * Long.BYTES;
}

/**
Expand Down
10 changes: 3 additions & 7 deletions src/main/java/org/apache/datasketches/theta/Sketches.java
Original file line number Diff line number Diff line change
Expand Up @@ -80,15 +80,11 @@ public static int getMaxAnotBResultBytes(final int maxNomEntries) {

/**
* Returns the maximum number of storage bytes required for a CompactSketch with the given
* number of actual entries. Note that this assumes the worse case of the sketch in
* estimation mode, which requires storing theta and count.
* @param numberOfEntries the actual number of entries stored with the CompactSketch.
* number of actual entries.
* @param numberOfEntries the actual number of retained entries stored in the sketch.
* @return the maximum number of storage bytes required for a CompactSketch with the given number
* of entries.
* @see Sketch#getMaxCompactSketchBytes(int)
* @deprecated as a public method. Use {@link #getCompactSketchMaxBytes(int) instead}
* of retained entries.
*/
@Deprecated
public static int getMaxCompactSketchBytes(final int numberOfEntries) {
return Sketch.getMaxCompactSketchBytes(numberOfEntries);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@
import org.apache.datasketches.common.SketchesArgumentException;
import org.apache.datasketches.common.SketchesReadOnlyException;
import org.apache.datasketches.memory.Memory;
import org.apache.datasketches.memory.WritableHandle;
import org.apache.datasketches.memory.WritableMemory;
import org.testng.annotations.Test;

Expand All @@ -51,8 +50,7 @@ public void createNewFilterTest() throws Exception {
assertFalse(bf1.isDirect());
assertFalse(bf1.isReadOnly());

try (WritableHandle wh = WritableMemory.allocateDirect(sizeBytes)) {
final WritableMemory wmem = wh.getWritable();
try (WritableMemory wmem = WritableMemory.allocateDirect(sizeBytes)) {
final BloomFilter bf2 = new BloomFilter(numBits, numHashes, seed, wmem);
assertTrue(bf2.isEmpty());
assertTrue(bf2.hasMemory());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.apache.datasketches.common.SketchesArgumentException;
import org.apache.datasketches.common.SketchesReadOnlyException;
import org.apache.datasketches.memory.Memory;
import org.apache.datasketches.memory.MemoryBoundsException;
import org.apache.datasketches.memory.WritableMemory;
import org.testng.annotations.Test;

Expand Down Expand Up @@ -142,8 +143,8 @@ public void bitAddressOutOfBoundsNonEmptyTest() {

final Memory mem = bitArrayToMemory(hba);
DirectBitArrayR dba = DirectBitArrayR.wrap(mem, hba.isEmpty());
assertThrows(AssertionError.class, () -> dba.getBit(-10));
assertThrows(AssertionError.class, () -> dba.getBit(2048));
assertThrows(MemoryBoundsException.class, () -> dba.getBit(-10));
assertThrows(MemoryBoundsException.class, () -> dba.getBit(2048));
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import static org.testng.Assert.assertTrue;

import org.apache.datasketches.common.SketchesArgumentException;
import org.apache.datasketches.memory.MemoryBoundsException;
import org.apache.datasketches.memory.WritableMemory;
import org.testng.annotations.Test;

Expand Down Expand Up @@ -139,7 +140,7 @@ public void basicWritableWrapTest() {
@Test
public void countWritableWrappedBitsWhenDirty() {
// like basicOperationTest but with setBit which does
// not neecssarily track numBitsSet_
// not necessarily track numBitsSet_
final HeapBitArray hba = new HeapBitArray(128);
assertFalse(hba.getAndSetBit(1));
assertFalse(hba.getAndSetBit(2));
Expand Down Expand Up @@ -172,12 +173,12 @@ public void bitAddresOutOfBoundsNonEmptyTest() {
dba.getAndSetBit(i);
}

assertThrows(AssertionError.class, () -> dba.getBit(-10));
assertThrows(AssertionError.class, () -> dba.getBit(2048));
assertThrows(AssertionError.class, () -> dba.setBit(-20));
assertThrows(AssertionError.class, () -> dba.setBit(4096));
assertThrows(AssertionError.class, () -> dba.getAndSetBit(-30));
assertThrows(AssertionError.class, () -> dba.getAndSetBit(8192));
assertThrows(MemoryBoundsException.class, () -> dba.getBit(-10));
assertThrows(MemoryBoundsException.class, () -> dba.getBit(2048));
assertThrows(MemoryBoundsException.class, () -> dba.setBit(-20));
assertThrows(MemoryBoundsException.class, () -> dba.setBit(4096));
assertThrows(MemoryBoundsException.class, () -> dba.getAndSetBit(-30));
assertThrows(MemoryBoundsException.class, () -> dba.getAndSetBit(8192));
}

@Test
Expand Down
83 changes: 38 additions & 45 deletions src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@
import org.testng.annotations.Test;

import org.apache.datasketches.memory.Memory;
import org.apache.datasketches.memory.WritableHandle;
import org.apache.datasketches.memory.WritableMemory;


Expand All @@ -50,51 +49,45 @@ public void checkGrow() {
int n = 8; //put lgConfigK == 4 into HLL mode
int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType);
HllSketch hllSketch;
try (WritableHandle handle = WritableMemory.allocateDirect(bytes,
ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())) {
WritableMemory wmem = handle.getWritable();
hllSketch = new HllSketch(lgConfigK, tgtHllType, wmem);
for (int i = 0; i < n; i++) {
hllSketch.update(i);
}
hllSketch.couponUpdate(HllUtil.pair(7, 15)); //mock extreme values
hllSketch.couponUpdate(HllUtil.pair(8, 15));
hllSketch.couponUpdate(HllUtil.pair(9, 15));
//println(hllSketch.toString(true, true, true, true));
DirectHllArray dha = (DirectHllArray) hllSketch.hllSketchImpl;
assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 2);
assertTrue(hllSketch.isMemory());
assertTrue(hllSketch.isOffHeap());
assertTrue(hllSketch.isSameResource(wmem));

//Check heapify
byte[] byteArray = hllSketch.toCompactByteArray();
HllSketch hllSketch2 = HllSketch.heapify(byteArray);
HllArray ha = (HllArray) hllSketch2.hllSketchImpl;
assertEquals(ha.getAuxHashMap().getLgAuxArrInts(), 2);
assertEquals(ha.getAuxHashMap().getAuxCount(), 3);

//Check wrap
byteArray = hllSketch.toUpdatableByteArray();
WritableMemory wmem2 = WritableMemory.writableWrap(byteArray);
hllSketch2 = HllSketch.writableWrap(wmem2);
//println(hllSketch2.toString(true, true, true, true));
DirectHllArray dha2 = (DirectHllArray) hllSketch2.hllSketchImpl;
assertEquals(dha2.getAuxHashMap().getLgAuxArrInts(), 2);
assertEquals(dha2.getAuxHashMap().getAuxCount(), 3);

//Check grow to on-heap
hllSketch.couponUpdate(HllUtil.pair(10, 15)); //puts it over the edge, must grow
//println(hllSketch.toString(true, true, true, true));
dha = (DirectHllArray) hllSketch.hllSketchImpl;
assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 3);
assertEquals(dha.getAuxHashMap().getAuxCount(), 4);
assertTrue(hllSketch.isMemory());
assertFalse(hllSketch.isOffHeap());
assertFalse(hllSketch.isSameResource(wmem));
} catch (final Exception e) {
throw new RuntimeException(e);
WritableMemory wmem = WritableMemory.allocateDirect(bytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
hllSketch = new HllSketch(lgConfigK, tgtHllType, wmem);
for (int i = 0; i < n; i++) {
hllSketch.update(i);
}
hllSketch.couponUpdate(HllUtil.pair(7, 15)); //mock extreme values
hllSketch.couponUpdate(HllUtil.pair(8, 15));
hllSketch.couponUpdate(HllUtil.pair(9, 15));
//println(hllSketch.toString(true, true, true, true));
DirectHllArray dha = (DirectHllArray) hllSketch.hllSketchImpl;
assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 2);
assertTrue(hllSketch.isMemory());
assertTrue(hllSketch.isOffHeap());
assertTrue(hllSketch.isSameResource(wmem));

//Check heapify
byte[] byteArray = hllSketch.toCompactByteArray();
HllSketch hllSketch2 = HllSketch.heapify(byteArray);
HllArray ha = (HllArray) hllSketch2.hllSketchImpl;
assertEquals(ha.getAuxHashMap().getLgAuxArrInts(), 2);
assertEquals(ha.getAuxHashMap().getAuxCount(), 3);

//Check wrap
byteArray = hllSketch.toUpdatableByteArray();
WritableMemory wmem2 = WritableMemory.writableWrap(byteArray);
hllSketch2 = HllSketch.writableWrap(wmem2);
//println(hllSketch2.toString(true, true, true, true));
DirectHllArray dha2 = (DirectHllArray) hllSketch2.hllSketchImpl;
assertEquals(dha2.getAuxHashMap().getLgAuxArrInts(), 2);
assertEquals(dha2.getAuxHashMap().getAuxCount(), 3);

//Check grow to on-heap
hllSketch.couponUpdate(HllUtil.pair(10, 15)); //puts it over the edge, must grow
//println(hllSketch.toString(true, true, true, true));
dha = (DirectHllArray) hllSketch.hllSketchImpl;
assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 3);
assertEquals(dha.getAuxHashMap().getAuxCount(), 4);
assertTrue(hllSketch.isMemory());
assertFalse(hllSketch.isOffHeap());
}

@Test
Expand Down
Loading
Loading