Multi Resource Partitioner

Usecase

• use MultiResourcePartitioner to read files multithreaded, see partition step setup
• write to distinct output file per step

Configuration Details

For general Maven, Spring and Database setup see the project setup wiki page.

• uses a slightly changed MultiResourcePartitioner which puts not only the filename to read in step context, but output filename too

Implementation Details

here is the relevant source

/**
 * Assign the filename of each of the injected resources to an
 * {@link ExecutionContext}.
 * 
 * @see Partitioner#partition(int)
 */
@Override
public Map<String, ExecutionContext> partition(int gridSize) {
    Map<String, ExecutionContext> map = new HashMap<String, ExecutionContext>(gridSize);
    int i = 0;
    for (Resource resource : resources) {
        ExecutionContext context = new ExecutionContext();
        Assert.state(resource.exists(), "Resource does not exist: " + resource);
        try {
            context.putString(keyName, resource.getURL().toExternalForm());
            context.put("outputFileName", createOutputFilename(i, context, resource));
        } catch (IOException e) {
            throw new IllegalArgumentException("File could not be located for: " + resource, e);
        }
        map.put(PARTITION_KEY + i, context);
        i++;
    }
    return map;
}

/**
 * Creates distinct output  file name per partition.
 *
 * @param partitionId
 * @param context
 * @param resource
 * @return 
 */
private String createOutputFilename(int partitionId, ExecutionContext context, Resource resource) {
    String outputFileName = "output-" + String.valueOf(partitionId) + ".txt";
    LOG.info(
            "for inputfile:'"
            + resource.getFilename()
            + "' outputfilename:'"
            + outputFileName
            + "' was created");

    return outputFileName;
}