Add pipeline framework to make parallel processing simpler (#1077)

Signed-off-by: Adrian Sutton <adrian.sutton@consensys.net>

metrics/src/main/java/tech/pegasys/pantheon/metrics/noop/NoOpMetricsSystem.java

@@ -29,7 +29,7 @@
 
 public class NoOpMetricsSystem implements MetricsSystem {
 
-  private static final Counter NO_OP_COUNTER = new NoOpCounter();
+  public static final Counter NO_OP_COUNTER = new NoOpCounter();
   private static final TimingContext NO_OP_TIMING_CONTEXT = () -> 0;
   private static final OperationTimer NO_OP_TIMER = () -> NO_OP_TIMING_CONTEXT;
   public static final LabelledMetric<OperationTimer> NO_OP_LABELLED_TIMER = label -> NO_OP_TIMER;

services/pipeline/build.gradle

@@ -0,0 +1,41 @@
+/*
+ * Copyright 2018 ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+apply plugin: 'java-library'
+
+jar {
+  baseName 'pantheon-pipeline'
+  manifest {
+    attributes(
+      'Specification-Title': baseName,
+      'Specification-Version': project.version,
+      'Implementation-Title': baseName,
+      'Implementation-Version': calculateVersion()
+      )
+  }
+}
+
+dependencies {
+  api project(':util')
+  implementation project(':metrics')
+
+  implementation 'org.apache.logging.log4j:log4j-api'
+  implementation 'com.google.guava:guava'
+
+  runtime 'org.apache.logging.log4j:log4j-core'
+
+  testImplementation 'junit:junit'
+  testImplementation 'org.assertj:assertj-core'
+  testImplementation 'org.awaitility:awaitility'
+  testImplementation 'org.mockito:mockito-core'
+}

services/pipeline/src/main/java/tech/pegasys/pantheon/services/pipeline/AsyncOperationProcessor.java

@@ -0,0 +1,98 @@
+/*
+ * Copyright 2019 ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package tech.pegasys.pantheon.services.pipeline;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Function;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+class AsyncOperationProcessor<I, O> implements Processor<I, O> {
+  private static final Logger LOG = LogManager.getLogger();
+  private final Function<I, CompletableFuture<O>> processor;
+  private final Collection<CompletableFuture<O>> inProgress;
+  private final int maxConcurrency;
+
+  public AsyncOperationProcessor(
+      final Function<I, CompletableFuture<O>> processor, final int maxConcurrency) {
+    this.processor = processor;
+    this.maxConcurrency = maxConcurrency;
+    this.inProgress = new ArrayList<>(maxConcurrency);
+  }
+
+  @Override
+  public void processNextInput(final ReadPipe<I> inputPipe, final WritePipe<O> outputPipe) {
+    if (inProgress.size() < maxConcurrency) {
+      final I value = inputPipe.get();
+      if (value != null) {
+        final CompletableFuture<O> future = processor.apply(value);
+        // When the future completes, interrupt so if we're waiting for new input we wake up and
+        // schedule the output.
+        final Thread stageThread = Thread.currentThread();
+        future.whenComplete((result, error) -> stageThread.interrupt());
+        inProgress.add(future);
+      }
+
+      outputCompletedTasks(0, outputPipe);
+    } else {
+      outputNextCompletedTask(outputPipe);
+    }
+  }
+
+  @Override
+  public void finalize(final WritePipe<O> outputPipe) {
+    while (!inProgress.isEmpty()) {
+      outputNextCompletedTask(outputPipe);
+    }
+  }
+
+  private void outputNextCompletedTask(final WritePipe<O> outputPipe) {
+    try {
+      waitForAnyFutureToComplete();
+      outputCompletedTasks(1, outputPipe);
+    } catch (final InterruptedException e) {
+      LOG.trace("Interrupted while waiting for processing to complete", e);
+    } catch (final ExecutionException e) {
+      LOG.error("Processing failed and we don't handle exceptions properly yet", e);
+    } catch (final TimeoutException e) {
+      // Ignore and go back around the loop.
+    }
+  }
+
+  @SuppressWarnings("rawtypes")
+  private void waitForAnyFutureToComplete()
+      throws InterruptedException, ExecutionException, TimeoutException {
+    CompletableFuture.anyOf(inProgress.toArray(new CompletableFuture[0])).get(1, TimeUnit.SECONDS);
+  }
+
+  private void outputCompletedTasks(final int minTasksToOutput, final WritePipe<O> outputPipe) {
+    int outputTasks = 0;
+    for (final Iterator<CompletableFuture<O>> i = inProgress.iterator();
+        i.hasNext() && (outputTasks < minTasksToOutput || outputPipe.hasRemainingCapacity()); ) {
+      final CompletableFuture<O> process = i.next();
+      final O result = process.getNow(null);
+      if (result != null) {
+        outputPipe.put(result);
+        i.remove();
+        outputTasks++;
+      }
+    }
+  }
+}

services/pipeline/src/main/java/tech/pegasys/pantheon/services/pipeline/BatchingProcessor.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2019 ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package tech.pegasys.pantheon.services.pipeline;
+
+import java.util.List;
+
+class BatchingProcessor<T> implements Processor<T, List<T>> {
+
+  private final int maximumBatchSize;
+
+  public BatchingProcessor(final int maximumBatchSize) {
+    this.maximumBatchSize = maximumBatchSize;
+  }
+
+  @Override
+  public void processNextInput(final ReadPipe<T> inputPipe, final WritePipe<List<T>> outputPipe) {
+    final List<T> batch = inputPipe.getBatch(maximumBatchSize);
+    if (!batch.isEmpty()) {
+      outputPipe.put(batch);
+    }
+  }
+}

services/pipeline/src/main/java/tech/pegasys/pantheon/services/pipeline/CompleterStage.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2019 ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package tech.pegasys.pantheon.services.pipeline;
+
+import tech.pegasys.pantheon.metrics.Counter;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.function.Consumer;
+
+class CompleterStage<T> implements Runnable {
+  private final ReadPipe<T> input;
+  private final Consumer<T> completer;
+  private final Counter outputCounter;
+  private final CompletableFuture<?> future = new CompletableFuture<>();
+
+  CompleterStage(
+      final ReadPipe<T> input, final Consumer<T> completer, final Counter outputCounter) {
+    this.input = input;
+    this.completer = completer;
+    this.outputCounter = outputCounter;
+  }
+
+  @Override
+  public void run() {
+    while (input.hasMore()) {
+      final T value = input.get();
+      if (value != null) {
+        completer.accept(value);
+        outputCounter.inc();
+      }
+    }
+    future.complete(null);
+  }
+
+  public CompletableFuture<?> getFuture() {
+    return future;
+  }
+}

services/pipeline/src/main/java/tech/pegasys/pantheon/services/pipeline/FlatMapProcessor.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2019 ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package tech.pegasys.pantheon.services.pipeline;
+
+import java.util.Iterator;
+import java.util.function.Function;
+import java.util.stream.Stream;
+
+class FlatMapProcessor<I, O> implements Processor<I, O> {
+
+  private final Function<I, Stream<O>> mapper;
+
+  public FlatMapProcessor(final Function<I, Stream<O>> mapper) {
+    this.mapper = mapper;
+  }
+
+  @Override
+  public void processNextInput(final ReadPipe<I> inputPipe, final WritePipe<O> outputPipe) {
+    final I value = inputPipe.get();
+    if (value != null) {
+      final Iterator<O> outputs = mapper.apply(value).iterator();
+      while (outputs.hasNext()) {
+        outputPipe.put(outputs.next());
+      }
+    }
+  }
+}

services/pipeline/src/main/java/tech/pegasys/pantheon/services/pipeline/IteratorSourceStage.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2019 ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package tech.pegasys.pantheon.services.pipeline;
+
+import java.util.Iterator;
+
+class IteratorSourceStage<T> implements Runnable {
+  private final Iterator<T> source;
+  private final Pipe<T> pipe;
+
+  IteratorSourceStage(final Iterator<T> source, final Pipe<T> pipe) {
+    this.source = source;
+    this.pipe = pipe;
+  }
+
+  @Override
+  public void run() {
+    while (pipe.isOpen() && source.hasNext()) {
+      final T value = source.next();
+      if (value != null) {
+        pipe.put(value);
+      }
+    }
+    pipe.close();
+  }
+}

services/pipeline/src/main/java/tech/pegasys/pantheon/services/pipeline/MapProcessor.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2019 ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package tech.pegasys.pantheon.services.pipeline;
+
+import java.util.function.Function;
+
+class MapProcessor<I, O> implements Processor<I, O> {
+
+  private final Function<I, O> processor;
+
+  public MapProcessor(final Function<I, O> processor) {
+    this.processor = processor;
+  }
+
+  @Override
+  public void processNextInput(final ReadPipe<I> inputPipe, final WritePipe<O> outputPipe) {
+    final I value = inputPipe.get();
+    if (value != null) {
+      outputPipe.put(processor.apply(value));
+    }
+  }
+}