Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/master'
Browse files Browse the repository at this point in the history
  • Loading branch information
peter-lawrey committed Jul 15, 2013
2 parents 5ea7fcf + 29e013c commit 4bf8cba
Show file tree
Hide file tree
Showing 8 changed files with 452 additions and 28 deletions.

Large diffs are not rendered by default.

6 changes: 6 additions & 0 deletions chronicle/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,12 @@
<version>4.11</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>net.sf.trove4j</groupId>
<artifactId>trove4j</artifactId>
<version>3.0.3</version>
<scope>test</scope>
</dependency>
</dependencies>

<build>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import com.higherfrequencytrading.chronicle.Chronicle;
import com.higherfrequencytrading.chronicle.Excerpt;
import com.higherfrequencytrading.chronicle.ExcerptMarshallable;
import com.higherfrequencytrading.chronicle.tcp.InProcessChronicleSink;
import com.higherfrequencytrading.chronicle.tools.ChronicleTools;

import java.io.Closeable;
Expand Down Expand Up @@ -57,14 +58,17 @@ public DataStore(final Chronicle chronicle, ModelMode mode) {

case READ_ONLY:
final String name = chronicle.name();
updater = Executors.newSingleThreadExecutor(new ThreadFactory() {
@Override
public Thread newThread(Runnable r) {
Thread t = new Thread(r, name + "data store updater");
t.setDaemon(true);
return t;
}
});
if (chronicle instanceof InProcessChronicleSink)
updater = Executors.newSingleThreadExecutor(new ThreadFactory() {
@Override
public Thread newThread(Runnable r) {
Thread t = new Thread(r, name + "data store updater");
t.setDaemon(true);
return t;
}
});
else
updater = null;
break;

default:
Expand Down Expand Up @@ -157,24 +161,25 @@ public void start(final long lastEvent) {
break;

case READ_ONLY:
updater.submit(new Runnable() {
@Override
public void run() {
excerpt = chronicle.createExcerpt();
while (!closed) {
boolean found = excerpt.nextIndex();
if (found) {
processNextEvent(excerpt.index() <= lastEvent);

} else {
for (Wrapper wrapper : wrappersArray) {
wrapper.notifyOff(false);
wrapper.inSync();
if (updater != null)
updater.submit(new Runnable() {
@Override
public void run() {
excerpt = chronicle.createExcerpt();
while (!closed) {
boolean found = excerpt.nextIndex();
if (found) {
processNextEvent(excerpt.index() <= lastEvent);

} else {
for (Wrapper wrapper : wrappersArray) {
wrapper.notifyOff(false);
wrapper.inSync();
}
}
}
}
}
});
});
break;

default:
Expand Down Expand Up @@ -233,4 +238,18 @@ public void close() {
updater.shutdown();
chronicle.close();
}

/**
* Should only be used for plain IndexChronicle.
*
* @return was a new entry found.
*/
public boolean nextEvent() {
assert updater == null;
if (excerpt.nextIndex()) {
processNextEvent(false);
return true;
}
return false;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
package com.higherfrequencytrading.chronicle.impl;

import com.higherfrequencytrading.chronicle.tools.ChronicleTools;

import java.io.IOException;
import java.nio.ByteOrder;

/**
* @author jkubrynski@gmail.com
* @since 2013-06-30
*/
public class ChronicleBuilder {

public static IndexedChronicleBuilder newIndexedChronicleBuilder(String basePath) {
return new IndexedChronicleBuilder(basePath);
}

public static IntIndexedChronicleBuilder newIntIndexedChronicleBuilder(String basePath) {
return new IntIndexedChronicleBuilder(basePath);
}

public static class IndexedChronicleBuilder {

protected String basePath;
protected int dataBitSizeHint =
ChronicleTools.is64Bit() ? IndexedChronicle.DEFAULT_DATA_BITS_SIZE : IndexedChronicle.DEFAULT_DATA_BITS_SIZE32;
protected ByteOrder byteOrder = ByteOrder.nativeOrder();
protected boolean minimiseByteBuffers = !ChronicleTools.is64Bit();
protected boolean synchronousMode = false;
protected boolean useUnsafe = false;

public IndexedChronicleBuilder(String basePath) {
this.basePath = basePath;
}

public IndexedChronicleBuilder dataBitSizeHint(int dataBitSizeHint) {
this.dataBitSizeHint = dataBitSizeHint;
return this;
}

public IndexedChronicleBuilder byteOrder(ByteOrder byteOrder) {
this.byteOrder = byteOrder;
return this;
}

public IndexedChronicleBuilder minimiseByteBuffers(boolean minimiseByteBuffers) {
this.minimiseByteBuffers = minimiseByteBuffers;
return this;
}

public IndexedChronicleBuilder useSynchronousMode(boolean synchronousMode) {
this.synchronousMode = synchronousMode;
return this;
}

public IndexedChronicleBuilder useUnsafe(boolean useUnsafe) {
this.useUnsafe = useUnsafe;
return this;
}

public IndexedChronicle build() throws IOException {
IndexedChronicle indexedChronicle =
new IndexedChronicle(basePath, dataBitSizeHint, byteOrder, minimiseByteBuffers, synchronousMode);
indexedChronicle.useUnsafe(useUnsafe);
return indexedChronicle;
}
}

public static class IntIndexedChronicleBuilder extends IndexedChronicleBuilder {

public IntIndexedChronicleBuilder(String basePath) {
super(basePath);
}

@Override
public IntIndexedChronicle build() throws IOException {
IntIndexedChronicle intIndexedChronicle = new IntIndexedChronicle(basePath, dataBitSizeHint, byteOrder);
intIndexedChronicle.useUnsafe(useUnsafe);
return intIndexedChronicle;
}
}


}
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
package com.higherfrequencytrading.chronicle.examples;

import com.higherfrequencytrading.chronicle.Chronicle;
import com.higherfrequencytrading.chronicle.Excerpt;
import com.higherfrequencytrading.chronicle.impl.IndexedChronicle;
import com.higherfrequencytrading.chronicle.tools.ChronicleTools;
import gnu.trove.map.hash.TObjectLongHashMap;

import java.io.IOException;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;

import static junit.framework.Assert.assertEquals;

/**
* @author peter.lawrey
*/
public class ExampleKeyedExcerptMain {
private static final String TMP = System.getProperty("java.io.tmpdir");

private final Chronicle chronicle;
private final Excerpt excerpt;
private final TObjectLongHashMap<String> keyToExcerpt = new TObjectLongHashMap<String>() {
@Override
public long getNoEntryValue() {
return -1;
}
};

public ExampleKeyedExcerptMain(String basePath) throws IOException {
chronicle = new IndexedChronicle(basePath);
excerpt = chronicle.createExcerpt();
}

public void load() {
while (excerpt.nextIndex()) {
String key = excerpt.readUTF();
keyToExcerpt.put(key, excerpt.index());
}
}

public void putMapFor(String key, Map<String, String> map) {
excerpt.startExcerpt(4096); // a guess
excerpt.writeUTF(key);
excerpt.writeMap(map);
excerpt.finish();
}

public Map<String, String> getMapFor(String key) {

long value = keyToExcerpt.get(key);
if (value < 0) return Collections.emptyMap();
excerpt.index(value);
// skip the key
excerpt.skip(excerpt.readStopBit());
return excerpt.readMap(String.class, String.class);
}

public void close() {
chronicle.close();
}

public static void main(String... ignored) throws IOException {
String basePath = TMP + "/ExampleKeyedExcerptMain";
ChronicleTools.deleteOnExit(basePath);
ExampleKeyedExcerptMain map = new ExampleKeyedExcerptMain(basePath);
map.load();
long start = System.nanoTime();
int keys = 10000000;
for (int i = 0; i < keys; i++) {
Map<String, String> props = new LinkedHashMap<String, String>();
props.put("a", Integer.toString(i)); // an int.
props.put("b", "value-" + i); // String
props.put("c", Double.toString(i / 1000.0)); // a double
map.putMapFor(Integer.toHexString(i), props);
}
map.close();

ExampleKeyedExcerptMain map2 = new ExampleKeyedExcerptMain(basePath);
map2.load();
long start2 = System.nanoTime();
for (int i = 0; i < keys; i++) {
Map<String, Object> props = new LinkedHashMap<String, Object>();
props.put("a", Integer.toString(i)); // an int.
props.put("b", "value-" + i); // String
props.put("c", Double.toString(i / 1000.0)); // a double
Map<String, String> props2 = map2.getMapFor(Integer.toHexString(i));
assertEquals(props, props2);
}
map2.close();
long time = System.nanoTime() - start;
long time2 = System.nanoTime() - start2;
System.out.printf("Took an average of %,d ns to write and read each entry, an average of %,d ns to lookup%n", time / keys, time2 / keys);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,12 @@ public void rewritibleEntries() throws IOException {

private void doRewriteableEntries(boolean useUnsafe, boolean minimiseByteBuffers, boolean synchronousMode) throws IOException {
String basePath = TMP + File.separator + "deleteme.ict";
IndexedChronicle tsc = new IndexedChronicle(basePath, IndexedChronicle.DEFAULT_DATA_BITS_SIZE32,
ByteOrder.nativeOrder(), minimiseByteBuffers, synchronousMode);
tsc.useUnsafe(useUnsafe);
IndexedChronicle tsc = ChronicleBuilder.newIndexedChronicleBuilder(basePath)
.dataBitSizeHint(IndexedChronicle.DEFAULT_DATA_BITS_SIZE32)
.minimiseByteBuffers(minimiseByteBuffers)
.useSynchronousMode(synchronousMode)
.useUnsafe(useUnsafe).build();

deleteOnExit(basePath);

tsc.clear();
Expand Down
2 changes: 1 addition & 1 deletion demo/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -43,4 +43,4 @@
</dependency>
</dependencies>

</project>
</project>
2 changes: 1 addition & 1 deletion testing/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -58,4 +58,4 @@
<artifactId>junit</artifactId>
</dependency>
</dependencies>
</project>
</project>

0 comments on commit 4bf8cba

Please sign in to comment.