diff --git a/.gitignore b/.gitignore index acffbd087..b79d58941 100644 --- a/.gitignore +++ b/.gitignore @@ -18,8 +18,8 @@ buildNumber.properties .DS_Store .asciidoctor/ bin/ -/build/ -/.gradle/ +build/ +.gradle/ *.log /.externalToolBuilders/ -.idea +.idea/ diff --git a/README.adoc b/README.adoc index f34eede86..4654c8ec1 100644 --- a/README.adoc +++ b/README.adoc @@ -27,13 +27,13 @@ RIOT is a data import/export tool for Redis. Use RIOT to bulk load/unload data f Download the {repo-url}/releases/latest[latest release] and untar/unzip the archive. -Launch the `riot` script (`riot.bat` for Windows) and follow the usage information provided. +Launch the `riot-file` script (`riot.bat` for Windows) and follow the usage information provided. === Install via Homebrew ``` $ brew install jruaux/tap/riot -$ riot --help +❯ riot --help ``` === Tab Completion diff --git a/build.gradle b/build.gradle index ca44af188..03ad2f0dd 100644 --- a/build.gradle +++ b/build.gradle @@ -1,214 +1,18 @@ plugins { - id 'application' - id 'java' - id 'java-library' - id 'maven-publish' id 'org.springframework.boot' version '2.3.1.RELEASE' id 'io.spring.dependency-management' version '1.0.9.RELEASE' - id 'com.jfrog.bintray' version '1.8.5' - id 'com.github.ben-manes.versions' version '0.28.0' - id 'net.researchgate.release' version '2.8.1' - id 'com.github.breadmoirai.github-release' version '2.2.12' - id 'org.asciidoctor.jvm.convert' version '3.2.0' - id 'org.ajoberstar.git-publish' version '2.1.3' } -repositories { - jcenter() - mavenCentral() - mavenLocal() -} - -ext { - set('springCloudVersion', "Hoxton.SR5") -} - -bootJar { - enabled = false -} - -jar { - enabled = true -} - -dependencies { - implementation 'com.redislabs:picocli-redis:2.0.0' - implementation 'org.slf4j:slf4j-jdk14' - implementation 'org.latencyutils:LatencyUtils:2.0.3' - implementation 'com.redislabs:spring-batch-redis:2.1.1-SNAPSHOT' -// implementation 'org.springframework.boot:spring-boot-starter-data-mongodb' -// implementation 'org.springframework.kafka:spring-kafka' - implementation 'com.redislabs:spring-batch-redisearch:2.3.1' - implementation 'org.springframework.batch:spring-batch-core' - implementation 'org.springframework.boot:spring-boot-autoconfigure' - implementation 'org.springframework:spring-jdbc' - implementation 'org.springframework.cloud:spring-cloud-aws-context' - implementation 'org.springframework.cloud:spring-cloud-aws-autoconfigure' - implementation 'org.ruaux:spring-batch-resource:1.0.0' - implementation 'org.ruaux:spring-batch-faker:1.0.1' - implementation 'org.ruaux:spring-batch-xml:1.0.0' - implementation 'com.zaxxer:HikariCP' - implementation 'me.tongfei:progressbar:0.8.1' - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' - testImplementation 'org.slf4j:slf4j-jdk14' - testImplementation 'org.codehaus.plexus:plexus-utils:3.3.0' - testImplementation 'org.hsqldb:hsqldb' - testImplementation 'org.junit.jupiter:junit-jupiter-engine' - testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine' - testImplementation 'it.ozimov:embedded-redis:0.7.3' - testImplementation 'org.projectlombok:lombok' - testAnnotationProcessor 'org.projectlombok:lombok' -// implementation 'org.xerial:sqlite-jdbc:3.28.0' -// implementation 'com.ibm.db2:jcc:11.5.0.0' -// implementation 'com.microsoft.sqlserver:mssql-jdbc:7.4.1.jre8' -// implementation 'com.oracle.ojdbc:ojdbc8:19.3.0.0' -// implementation 'org.postgresql:postgresql:42.2.8' -} - -configurations { - all*.exclude module : 'spring-boot-starter-logging' -} - -dependencyManagement { - imports { - mavenBom "org.springframework.cloud:spring-cloud-dependencies:${springCloudVersion}" - } -} - -test { - useJUnitPlatform() -} - -bintray { - user = project.hasProperty('bintrayUser') ? project.property('bintrayUser') : '' - key = project.hasProperty('bintrayKey') ? project.property('bintrayKey') : '' - publications = ['mavenJava'] - publish = true - pkg { - repo = 'maven' - name = 'riot' - licenses = ['Apache-2.0'] - vcsUrl = 'https://github.com/Redislabs-Solution-Architects/riot.git' - version { - gpg { - sign = true - } - mavenCentralSync { - sync = true - user = project.hasProperty('ossrhUsername') ? project.property('ossrhUsername') : '' - password = project.hasProperty('ossrhPassword') ? project.property('ossrhPassword') : '' - } +subprojects { + group = 'com.redislabs.riot' + apply plugin: 'java' + apply plugin: 'org.springframework.boot' + apply plugin: 'io.spring.dependency-management' + sourceCompatibility = '1.8' + targetCompatibility = '1.8' + repositories { + jcenter() + mavenCentral() + mavenLocal() } -} -} - -task sourcesJar(type: Jar) { - archiveClassifier = 'sources' - from sourceSets.main.allJava -} - -task javadocJar(type: Jar) { - archiveClassifier = 'javadoc' - from javadoc.destinationDir -} - -artifacts { - archives sourcesJar, javadocJar -} - -publishing { - publications { - mavenJava(MavenPublication) { - from components.java - artifact sourcesJar - artifact javadocJar - groupId 'com.redislabs' - artifactId 'riot' - - pom { - name = 'RIOT' - packaging = 'jar' - description = 'Redis Input/Output Tool' - url = 'https://github.com/Redislabs-Solution-Architects/riot' - - scm { - connection = 'scm:git:git://github.com/Redislabs-Solution-Architects/riot.git' - developerConnection = 'scm:git:git@github.com:Redislabs-Solution-Architects/riot.git' - url = 'https://github.com/Redislabs-Solution-Architects/riot' - } - - licenses { - license { - name = 'The Apache License, Version 2.0' - url = 'http://www.apache.org/licenses/LICENSE-2.0.txt' - } - } - - developers { - developer { - id = 'jruaux' - name = 'Julien Ruaux' - } - } - } - } - } -} - -group = 'com.redislabs' -description = 'RIOT' -sourceCompatibility = JavaVersion.VERSION_1_8 -targetCompatibility = JavaVersion.VERSION_1_8 - -application { - mainClassName = 'com.redislabs.riot.Riot' -} - -jar { - manifest { - attributes("Implementation-Title": "RIOT", "Implementation-Version": archiveVersion) - } -} - -tasks.withType(Tar) { - compression = Compression.GZIP -} - -asciidoctor { - outputOptions { - separateOutputDirs = false - } - attributes 'commandsdir': 'src/test/resources/commands' -} - -task printVersion { - doLast { - println project.version - } -} - -githubRelease { - token = project.hasProperty('githubToken') ? project.property('githubToken') : '' - owner "Redislabs-Solution-Architects" - repo "riot" - releaseAssets distZip, distTar - body changelog() -} - -gitPublish { - repoUri = 'git@github.com:Redislabs-Solution-Architects/riot.git' - referenceRepoUri = 'file:///Users/jruaux/git/riot/' - - branch = 'gh-pages' - - contents { - from 'build/docs/asciidoc' - } -} - -gitPublishPush.dependsOn asciidoctor - -afterReleaseBuild.dependsOn ":githubRelease" -afterReleaseBuild.dependsOn bintrayUpload -afterReleaseBuild.dependsOn gitPublishPush \ No newline at end of file +} \ No newline at end of file diff --git a/core/build.gradle b/core/build.gradle new file mode 100644 index 000000000..e22d0d0f8 --- /dev/null +++ b/core/build.gradle @@ -0,0 +1,36 @@ +plugins { + id 'java-library' +} + +archivesBaseName = 'riot-core' + +bootJar { + enabled = false +} + +jar { + enabled = true +} + +test { + useJUnitPlatform() +} + +dependencies { + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' + api 'com.redislabs:picocli-redis:2.0.2' + api 'com.redislabs:spring-batch-redis:2.2.1' + api 'com.redislabs:spring-batch-redisearch:2.4.1' + implementation 'me.tongfei:progressbar:0.8.1' + implementation 'org.apache.commons:commons-lang3' + implementation 'org.latencyutils:LatencyUtils:2.0.3' + implementation 'org.slf4j:slf4j-api' + api 'org.springframework.batch:spring-batch-core' + implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml' + testImplementation 'org.junit.jupiter:junit-jupiter-engine' +} + +configurations { + all*.exclude module : 'spring-boot-starter-logging' +} \ No newline at end of file diff --git a/core/src/main/java/com/redislabs/riot/AbstractExportCommand.java b/core/src/main/java/com/redislabs/riot/AbstractExportCommand.java new file mode 100644 index 000000000..c7b004130 --- /dev/null +++ b/core/src/main/java/com/redislabs/riot/AbstractExportCommand.java @@ -0,0 +1,25 @@ +package com.redislabs.riot; + +import org.springframework.batch.item.ItemReader; +import org.springframework.batch.item.redis.RedisKeyValueItemReader; +import org.springframework.batch.item.redis.support.KeyValue; +import picocli.CommandLine; + +@CommandLine.Command +public abstract class AbstractExportCommand extends AbstractTransferCommand, O> { + + @CommandLine.Mixin + private RedisExportOptions options = new RedisExportOptions(); + @CommandLine.Option(names = "--key-regex", description = "Regular expression for key-field extraction", paramLabel = "") + private String keyRegex; + + @Override + protected String taskName() { + return "Exporting"; + } + + public ItemReader> reader() { + return configure(RedisKeyValueItemReader.builder().scanCount(options.getScanCount()).scanMatch(options.getScanMatch()).batch(options.getBatchSize()).queueCapacity(options.getQueueCapacity()).threads(options.getThreads())).build(); + } + +} diff --git a/src/main/java/com/redislabs/riot/cli/ImportCommand.java b/core/src/main/java/com/redislabs/riot/AbstractImportCommand.java similarity index 53% rename from src/main/java/com/redislabs/riot/cli/ImportCommand.java rename to core/src/main/java/com/redislabs/riot/AbstractImportCommand.java index 5d0675459..635cdc9f2 100644 --- a/src/main/java/com/redislabs/riot/cli/ImportCommand.java +++ b/core/src/main/java/com/redislabs/riot/AbstractImportCommand.java @@ -1,8 +1,7 @@ -package com.redislabs.riot.cli; +package com.redislabs.riot; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.xml.XmlMapper; -import com.redislabs.lettuce.helper.RedisOptions; import com.redislabs.lettusearch.search.AddOptions; import com.redislabs.riot.convert.IdemConverter; import com.redislabs.riot.convert.KeyMaker; @@ -14,7 +13,7 @@ import lombok.Getter; import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemWriter; -import org.springframework.batch.item.redis.support.RedisCommandItemWriters; +import org.springframework.batch.item.redis.support.CommandItemWriters.*; import org.springframework.batch.item.redisearch.RediSearchItemWriter; import org.springframework.batch.item.redisearch.RediSearchSuggestItemWriter; import org.springframework.batch.item.redisearch.support.DocumentItemProcessor; @@ -30,12 +29,7 @@ import java.util.List; import java.util.Map; -@CommandLine.Command(name = "import", description = "Import data into Redis", subcommands = {FileImportCommand.class, DatabaseImportCommand.class, GenerateCommand.class}, sortOptions = false) -public class ImportCommand extends TransferCommand { - - public enum CommandName { - EVALSHA, EXPIRE, GEOADD, FTADD, FTSEARCH, FTAGGREGATE, FTSUGADD, HMSET, LPUSH, NOOP, RPUSH, SADD, SET, XADD, ZADD - } +public abstract class AbstractImportCommand extends AbstractTransferCommand { @Getter @CommandLine.Option(arity = "1..*", names = "--spel", description = "SpEL expression to produce a field", paramLabel = "") @@ -44,25 +38,12 @@ public enum CommandName { private Map variables = new HashMap<>(); @CommandLine.Option(names = "--date-format", description = "Processor date format (default: ${DEFAULT-VALUE})", paramLabel = "") private String dateFormat = new SimpleDateFormat().toPattern(); - @Getter - @CommandLine.Option(names = "--command", description = "Redis command: ${COMPLETION-CANDIDATES} (default: ${DEFAULT-VALUE})", paramLabel = "") - private CommandName command = CommandName.HMSET; - @CommandLine.Option(names = "--key-separator", description = "Key separator (default: ${DEFAULT-VALUE})", paramLabel = "") - private String separator = KeyMaker.DEFAULT_SEPARATOR; - @CommandLine.Option(names = {"-p", "--keyspace"}, description = "Keyspace prefix", paramLabel = "") - private String keyspace; - @CommandLine.Option(names = {"-k", "--keys"}, arity = "1..*", description = "Key fields", paramLabel = "") - private String[] keyFields = new String[0]; @CommandLine.Option(names = "--remove-fields", description = "Remove fields already used in data structures") private boolean removeFields; - @CommandLine.Option(names = "--member-space", description = "Prefix for member IDs", paramLabel = "") - private String memberSpace; - @CommandLine.Option(names = "--members", arity = "1..*", description = "Member field names for collections", paramLabel = "") - private String[] memberFields = new String[0]; @CommandLine.ArgGroup(exclusive = false, heading = "Redis command options%n") - private final RedisCommandOptions commandOptions = new RedisCommandOptions(); + private final RedisImportOptions redis = new RedisImportOptions(); @CommandLine.ArgGroup(exclusive = false, heading = "RediSearch command options%n") - private final RediSearchCommandOptions redisearch = new RediSearchCommandOptions(); + private final RediSearchImportOptions redisearch = new RediSearchImportOptions(); @Override protected String taskName() { @@ -70,22 +51,22 @@ protected String taskName() { } public SpelProcessor spelProcessor() { - RedisOptions redisOptions = redisOptions(); - return SpelProcessor.builder().connection(redisOptions.connection()).commands(redisOptions.sync()).dateFormat(new SimpleDateFormat(dateFormat)).variables(variables).fields(spel).build(); + return configure(SpelProcessor.builder().dateFormat(new SimpleDateFormat(dateFormat)).variables(variables).fields(spel)).build(); } private KeyMaker> keyMaker() { - return idMaker(keyspace, keyFields); + return idMaker(redis.getKeyspace(), redis.getKeyFields()); } private KeyMaker> memberIdMaker() { - return idMaker(memberSpace, memberFields); + return idMaker(redis.getMemberSpace(), redis.getMemberFields()); } private KeyMaker> idMaker(String prefix, String[] fields) { - return KeyMaker.>builder().separator(separator).prefix(prefix).extractors(fieldExtractors(removeFields, fields)).build(); + return KeyMaker.>builder().separator(redis.getKeySeparator()).prefix(prefix).extractors(fieldExtractors(removeFields, fields)).build(); } + @SuppressWarnings("unchecked") private Converter, String>[] fieldExtractors(boolean remove, String... fields) { List, String>> extractors = new ArrayList<>(); for (String field : fields) { @@ -94,6 +75,7 @@ private Converter, String>[] fieldExtractors(boolean remove, return extractors.toArray(new Converter[0]); } + @SuppressWarnings("rawtypes") public ItemProcessor objectMapProcessor() { List processors = new ArrayList<>(); if (!spel.isEmpty()) { @@ -103,9 +85,10 @@ public ItemProcessor objectMapProcessor() { return processor(processors); } + @SuppressWarnings({"rawtypes", "unchecked"}) public ItemProcessor processor(List processors) { List allProcessors = new ArrayList<>(processors); - switch (command) { + switch (redis.getCommand()) { case FTADD: allProcessors.add(documentItemProcessor()); break; @@ -124,6 +107,7 @@ public ItemProcessor processor(List processors) { return compositeItemProcessor; } + @SuppressWarnings("rawtypes") private ItemProcessor suggestionItemProcessor() { SuggestionItemProcessor.SuggestionItemProcessorBuilder builder = SuggestionItemProcessor.builder().stringConverter(fieldExtractor(redisearch.getField())).scoreConverter(scoreConverter()); if (redisearch.getPayloadField() != null) { @@ -132,6 +116,7 @@ private ItemProcessor suggestionItemProcessor() { return builder.build(); } + @SuppressWarnings("rawtypes") private ItemProcessor documentItemProcessor() { DocumentItemProcessor.DocumentItemProcessorBuilder builder = DocumentItemProcessor.builder().idConverter(keyMaker()).scoreConverter(scoreConverter()); if (redisearch.getPayloadField() != null) { @@ -140,46 +125,46 @@ private ItemProcessor documentItemProcessor() { return builder.build(); } + @SuppressWarnings({"rawtypes", "unchecked"}) public ItemWriter writer() { - RedisOptions redisOptions = redisOptions(); - switch (command) { + switch (redis.getCommand()) { case EVALSHA: - return new RedisCommandItemWriters.Eval<>(redisOptions.connectionPool(), redisOptions.async(), redisOptions.getTimeout(), commandOptions.getEvalSha(), commandOptions.getEvalOutputType(), MapToArrayConverter.builder().fields(keyFields).build(), MapToArrayConverter.builder().fields(commandOptions.getEvalArgs()).build()); + return configure(Eval.>builder().sha(redis.getEvalSha()).outputType(redis.getEvalOutputType()).keysConverter(MapToArrayConverter.builder().fields(redis.getKeyFields()).build()).argsConverter(MapToArrayConverter.builder().fields(redis.getEvalArgs()).build())).build(); case EXPIRE: - return new RedisCommandItemWriters.Expire(redisOptions.connectionPool(), redisOptions.async(), redisOptions.getTimeout(), keyMaker(), longFieldExtractor(commandOptions.getTimeout())); + return configureKeyCommandWriterBuilder(Expire.>builder().timeoutConverter(longFieldExtractor(redis.getTimeout()))).build(); case HMSET: - return new RedisCommandItemWriters.Hmset(redisOptions.connectionPool(), redisOptions.async(), redisOptions.getTimeout(), keyMaker(), new IdemConverter<>()); + return configureKeyCommandWriterBuilder(Hmset.>builder().mapConverter(new IdemConverter<>())).build(); case GEOADD: - return new RedisCommandItemWriters.Geoadd(redisOptions.connectionPool(), redisOptions.async(), redisOptions.getTimeout(), keyMaker(), memberIdMaker(), doubleFieldExtractor(commandOptions.getLongitudeField()), doubleFieldExtractor(commandOptions.getLatitudeField())); + return configureCollectionCommandWriterBuilder(Geoadd.>builder().longitudeConverter(doubleFieldExtractor(redis.getLongitudeField())).latitudeConverter(doubleFieldExtractor(redis.getLatitudeField()))).build(); case LPUSH: - return new RedisCommandItemWriters.Lpush(redisOptions.connectionPool(), redisOptions.async(), redisOptions.getTimeout(), keyMaker(), memberIdMaker()); + return configureCollectionCommandWriterBuilder(Lpush.builder()).build(); case RPUSH: - return new RedisCommandItemWriters.Rpush(redisOptions.connectionPool(), redisOptions.async(), redisOptions.getTimeout(), keyMaker(), memberIdMaker()); + return configureCollectionCommandWriterBuilder(Rpush.builder()).build(); case SADD: - return new RedisCommandItemWriters.Sadd(redisOptions.connectionPool(), redisOptions.async(), redisOptions.getTimeout(), keyMaker(), memberIdMaker()); + return configureCollectionCommandWriterBuilder(Sadd.builder()).build(); case SET: - return new RedisCommandItemWriters.Set(redisOptions.connectionPool(), redisOptions.async(), redisOptions.getTimeout(), keyMaker(), stringValueConverter()); + return configureKeyCommandWriterBuilder(Set.>builder().valueConverter(stringValueConverter())).build(); case NOOP: - return new RedisCommandItemWriters.Noop<>(redisOptions.connectionPool(), redisOptions.async(), redisOptions.getTimeout()); + return configure(Noop.>builder()).build(); case XADD: - return new RedisCommandItemWriters.Xadd(redisOptions.connectionPool(), redisOptions.async(), redisOptions.getTimeout(), keyMaker(), new IdemConverter<>(), fieldExtractor(commandOptions.getIdField()), commandOptions.getMaxlen(), commandOptions.isApproximateTrimming()); + return configureKeyCommandWriterBuilder(Xadd.>builder().bodyConverter(new IdemConverter<>()).idConverter(fieldExtractor(redis.getIdField())).maxlen(redis.getMaxlen()).approximateTrimming(redis.isApproximateTrimming())).build(); case ZADD: - return new RedisCommandItemWriters.Zadd(redisOptions.connectionPool(), redisOptions.async(), redisOptions.getTimeout(), keyMaker(), memberIdMaker(), scoreConverter()); + return configureCollectionCommandWriterBuilder(Zadd.>builder().scoreConverter(scoreConverter())).build(); case FTADD: - return searchWriter(); + AddOptions addOptions = AddOptions.builder().ifCondition(redisearch.getIfCondition()).language(redisearch.getLanguage()).noSave(redisearch.isNosave()).replace(redisearch.isReplace()).replacePartial(redisearch.isReplacePartial()).build(); + return configure(RediSearchItemWriter.builder().index(redisearch.getIndex()).addOptions(addOptions)).build(); case FTSUGADD: - return suggestWriter(); + return configure(RediSearchSuggestItemWriter.builder().increment(redisearch.isIncrement()).key(redisearch.getIndex())).build(); } throw new IllegalArgumentException("Command not supported"); } - private RediSearchItemWriter searchWriter() { - AddOptions addOptions = AddOptions.builder().ifCondition(redisearch.getIfCondition()).language(redisearch.getLanguage()).noSave(redisearch.isNosave()).replace(redisearch.isReplace()).replacePartial(redisearch.isReplacePartial()).build(); - return RediSearchItemWriter.builder().redisOptions(redisOptions()).index(redisearch.getIndex()).addOptions(addOptions).build(); + private >> B configureKeyCommandWriterBuilder(B builder) { + return configure(builder.keyConverter(keyMaker())); } - private RediSearchSuggestItemWriter suggestWriter() { - return RediSearchSuggestItemWriter.builder().redisOptions(redisOptions()).increment(redisearch.isIncrement()).key(redisearch.getIndex()).build(); + private >> B configureCollectionCommandWriterBuilder(B builder) { + return configureKeyCommandWriterBuilder(builder.keyConverter(keyMaker()).memberIdConverter(memberIdMaker())); } private Converter, String> fieldExtractor(String field) { @@ -199,19 +184,19 @@ private Converter, T> fieldExtractor(String field, Class } private Converter, Double> scoreConverter() { - return FieldExtractor.builder(Double.class).field(commandOptions.getField()).defaultValue(commandOptions.getDefaultValue()).remove(removeFields).build(); + return FieldExtractor.builder(Double.class).field(redis.getField()).defaultValue(redis.getDefaultValue()).remove(removeFields).build(); } private Converter, String> stringValueConverter() { - switch (commandOptions.getFormat()) { + switch (redis.getFormat()) { case RAW: - return fieldExtractor(commandOptions.getValueField()); + return fieldExtractor(redis.getValueField()); case XML: - return new ObjectMapperConverter<>(new XmlMapper().writer().withRootName(commandOptions.getRoot())); + return new ObjectMapperConverter<>(new XmlMapper().writer().withRootName(redis.getRoot())); case JSON: - return new ObjectMapperConverter<>(new ObjectMapper().writer().withRootName(commandOptions.getRoot())); + return new ObjectMapperConverter<>(new ObjectMapper().writer().withRootName(redis.getRoot())); } - throw new IllegalArgumentException("Unsupported String format: " + commandOptions.getFormat()); + throw new IllegalArgumentException("Unsupported String format: " + redis.getFormat()); } diff --git a/core/src/main/java/com/redislabs/riot/AbstractTransferCommand.java b/core/src/main/java/com/redislabs/riot/AbstractTransferCommand.java new file mode 100644 index 000000000..caede74e4 --- /dev/null +++ b/core/src/main/java/com/redislabs/riot/AbstractTransferCommand.java @@ -0,0 +1,91 @@ +package com.redislabs.riot; + +import com.redislabs.picocliredis.HelpCommand; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.item.ItemProcessor; +import org.springframework.batch.item.ItemReader; +import org.springframework.batch.item.ItemWriter; +import org.springframework.batch.item.redis.support.ConnectionPoolConfig; +import org.springframework.batch.item.redis.support.RedisConnectionBuilder; +import org.springframework.batch.item.redisearch.support.RediSearchConnectionBuilder; +import picocli.CommandLine; + +@Slf4j +@CommandLine.Command(abbreviateSynopsis = true, sortOptions = false) +public abstract class AbstractTransferCommand extends HelpCommand implements Runnable { + + @CommandLine.ParentCommand + private RiotApp app; + @CommandLine.Option(names = "--threads", description = "Thread count (default: ${DEFAULT-VALUE})", paramLabel = "") + private int threads = 1; + @CommandLine.Option(names = {"-b", "--batch"}, description = "Number of items in each batch (default: ${DEFAULT-VALUE})", paramLabel = "") + private int batchSize = 50; + @CommandLine.Option(names = "--max", description = "Max number of items to read", paramLabel = "") + private Integer maxItemCount; + + protected > B configure(RedisConnectionBuilder builder) { + return app.configure(builder); + } + + protected > B configure(RediSearchConnectionBuilder builder) { + return app.configure(builder); + } + + protected > B configure(RedisConnectionBuilder builder, RedisConnectionOptions redis) { + return app.configure(builder, redis); + } + + protected > B configure(RediSearchConnectionBuilder builder, RedisConnectionOptions redis) { + return app.configure(builder, redis); + } + + public void execute(ItemReader reader, ItemProcessor processor, ItemWriter writer) { + Transfer transfer = transfer(reader, processor, writer); + ProgressBarOptions progressBarOptions = ProgressBarOptions.builder().taskName(taskName()).initialMax(maxItemCount).quiet(app.isQuiet()).build(); + ProgressBarReporter reporter = ProgressBarReporter.builder().transfer(transfer).options(progressBarOptions).build(); + reporter.start(); + transfer.execute(); + reporter.stop(); + } + + protected Transfer transfer(ItemReader reader, ItemProcessor processor, ItemWriter writer) { + return Transfer.builder().reader(reader).processor(processor).writer(writer).batchSize(batchSize).threadCount(threads).maxItemCount(maxItemCount).build(); + } + + protected abstract String taskName(); + + @Override + public void run() { + ItemReader reader; + try { + reader = reader(); + } catch (Exception e) { + log.error("Could not create reader", e); + return; + } + ItemProcessor processor; + try { + processor = processor(); + } catch (Exception e) { + log.error("Could not create processor", e); + return; + } + ItemWriter writer; + try { + writer = writer(); + } catch (Exception e) { + log.error("Could not create writer", e); + return; + } + execute(reader, processor, writer); + } + + protected abstract ItemReader reader() throws Exception; + + protected abstract ItemProcessor processor() throws Exception; + + protected abstract ItemWriter writer() throws Exception; + + +} diff --git a/core/src/main/java/com/redislabs/riot/OneLineLogFormat.java b/core/src/main/java/com/redislabs/riot/OneLineLogFormat.java new file mode 100644 index 000000000..1eef272b1 --- /dev/null +++ b/core/src/main/java/com/redislabs/riot/OneLineLogFormat.java @@ -0,0 +1,49 @@ +package com.redislabs.riot; + +import org.apache.commons.lang3.exception.ExceptionUtils; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.temporal.ChronoField; +import java.util.logging.Formatter; +import java.util.logging.LogRecord; + +public class OneLineLogFormat extends Formatter { + + private final DateTimeFormatter d = new DateTimeFormatterBuilder().appendValue(ChronoField.HOUR_OF_DAY, 2).appendLiteral(':').appendValue(ChronoField.MINUTE_OF_HOUR, 2).optionalStart().appendLiteral(':').appendValue(ChronoField.SECOND_OF_MINUTE, 2).optionalStart().appendFraction(ChronoField.NANO_OF_SECOND, 3, 3, true).toFormatter(); + private final ZoneId offset = ZoneOffset.systemDefault(); + private final boolean verbose; + + public OneLineLogFormat(boolean verbose) { + this.verbose = verbose; + } + + @Override + public String format(LogRecord record) { + String message = formatMessage(record); + ZonedDateTime time = Instant.ofEpochMilli(record.getMillis()).atZone(offset); + if (record.getThrown() == null) { + if (verbose) { + return String.format("%s %s %s\t: %s%n", time.format(d), record.getLevel().getLocalizedName(), record.getLoggerName(), message); + } + return String.format("%s%n", message); + } + if (verbose) { + return String.format("%s %s %s\t: %s%n%s%n", time.format(d), record.getLevel().getLocalizedName(), record.getLoggerName(), message, stackTrace(record)); + } + return String.format("%s: %s%n", message, ExceptionUtils.getRootCause(record.getThrown()).getMessage()); + } + + private String stackTrace(LogRecord record) { + StringWriter sw = new StringWriter(4096); + PrintWriter pw = new PrintWriter(sw); + record.getThrown().printStackTrace(pw); + return sw.toString(); + } +} diff --git a/src/main/java/com/redislabs/riot/ProcessingItemWriter.java b/core/src/main/java/com/redislabs/riot/ProcessingItemWriter.java similarity index 100% rename from src/main/java/com/redislabs/riot/ProcessingItemWriter.java rename to core/src/main/java/com/redislabs/riot/ProcessingItemWriter.java diff --git a/src/main/java/com/redislabs/riot/cli/ProgressBarOptions.java b/core/src/main/java/com/redislabs/riot/ProgressBarOptions.java similarity index 89% rename from src/main/java/com/redislabs/riot/cli/ProgressBarOptions.java rename to core/src/main/java/com/redislabs/riot/ProgressBarOptions.java index 32972134b..94dea3bdf 100644 --- a/src/main/java/com/redislabs/riot/cli/ProgressBarOptions.java +++ b/core/src/main/java/com/redislabs/riot/ProgressBarOptions.java @@ -1,4 +1,4 @@ -package com.redislabs.riot.cli; +package com.redislabs.riot; import lombok.Builder; import lombok.Data; diff --git a/src/main/java/com/redislabs/riot/cli/ProgressBarReporter.java b/core/src/main/java/com/redislabs/riot/ProgressBarReporter.java similarity index 96% rename from src/main/java/com/redislabs/riot/cli/ProgressBarReporter.java rename to core/src/main/java/com/redislabs/riot/ProgressBarReporter.java index 9de92c49a..52337fa2c 100644 --- a/src/main/java/com/redislabs/riot/cli/ProgressBarReporter.java +++ b/core/src/main/java/com/redislabs/riot/ProgressBarReporter.java @@ -1,6 +1,5 @@ -package com.redislabs.riot.cli; +package com.redislabs.riot; -import com.redislabs.riot.Transfer; import lombok.Builder; import lombok.extern.slf4j.Slf4j; import me.tongfei.progressbar.ProgressBar; diff --git a/core/src/main/java/com/redislabs/riot/RediSearchExportOptions.java b/core/src/main/java/com/redislabs/riot/RediSearchExportOptions.java new file mode 100644 index 000000000..3c976722a --- /dev/null +++ b/core/src/main/java/com/redislabs/riot/RediSearchExportOptions.java @@ -0,0 +1,17 @@ +package com.redislabs.riot; + +import lombok.Getter; +import picocli.CommandLine; + +import java.util.List; + +public class RediSearchExportOptions extends RediSearchOptions { + + @Getter + @CommandLine.Option(names = "--query", description = "RediSearch query", paramLabel = "") + private String query; + @Getter + @CommandLine.Option(names = "--ft-options", arity = "1..*", description = "Search/aggregate options", paramLabel = "") + private List options; + +} diff --git a/src/main/java/com/redislabs/riot/cli/RediSearchCommandOptions.java b/core/src/main/java/com/redislabs/riot/RediSearchImportOptions.java similarity index 70% rename from src/main/java/com/redislabs/riot/cli/RediSearchCommandOptions.java rename to core/src/main/java/com/redislabs/riot/RediSearchImportOptions.java index 3a8e019b6..6c76b9f41 100644 --- a/src/main/java/com/redislabs/riot/cli/RediSearchCommandOptions.java +++ b/core/src/main/java/com/redislabs/riot/RediSearchImportOptions.java @@ -1,23 +1,11 @@ -package com.redislabs.riot.cli; +package com.redislabs.riot; import com.redislabs.lettusearch.search.Language; import lombok.Getter; import picocli.CommandLine; -import picocli.CommandLine.Option; -import java.util.List; +public class RediSearchImportOptions extends RediSearchOptions { -public class RediSearchCommandOptions { - - @Getter - @Option(names = {"-i", "--index"}, description = "Name of the RediSearch index", paramLabel = "") - private String index; - @Getter - @Option(names = "--query", description = "RediSearch query", paramLabel = "") - private String query; - @Getter - @Option(names = "--ft-options", arity = "1..*", description = "Search/aggregate options", paramLabel = "") - private List options; @Getter @CommandLine.Option(names = "--payload", description = "Name of the field containing the payload", paramLabel = "") private String payloadField; diff --git a/core/src/main/java/com/redislabs/riot/RediSearchOptions.java b/core/src/main/java/com/redislabs/riot/RediSearchOptions.java new file mode 100644 index 000000000..7e0d6037f --- /dev/null +++ b/core/src/main/java/com/redislabs/riot/RediSearchOptions.java @@ -0,0 +1,11 @@ +package com.redislabs.riot; + +import lombok.Getter; +import picocli.CommandLine; + +public class RediSearchOptions { + + @Getter + @CommandLine.Option(names = {"-i", "--index"}, description = "Name of the RediSearch index", paramLabel = "") + private String index; +} diff --git a/core/src/main/java/com/redislabs/riot/RedisConnectionOptions.java b/core/src/main/java/com/redislabs/riot/RedisConnectionOptions.java new file mode 100644 index 000000000..36bab9038 --- /dev/null +++ b/core/src/main/java/com/redislabs/riot/RedisConnectionOptions.java @@ -0,0 +1,20 @@ +package com.redislabs.riot; + +import io.lettuce.core.RedisURI; +import lombok.Getter; +import org.apache.commons.pool2.impl.GenericObjectPoolConfig; +import picocli.CommandLine; + +@Getter +public class RedisConnectionOptions { + + @CommandLine.Option(names = {"-r", "--redis"}, description = "Redis connection string (default: redis://localhost:6379)", paramLabel = "") + private RedisURI redisURI = RedisURI.create("localhost", RedisURI.DEFAULT_REDIS_PORT); + @CommandLine.Option(names = {"-c", "--cluster"}, description = "Connect to a Redis Cluster") + private boolean cluster; + @CommandLine.Option(names = {"-m", "--metrics"}, description = "Show metrics") + private boolean showMetrics; + @CommandLine.Option(names = "--pool", description = "Max pool connections (default: ${DEFAULT-VALUE})", paramLabel = "") + private int poolMaxTotal = GenericObjectPoolConfig.DEFAULT_MAX_TOTAL; + +} diff --git a/core/src/main/java/com/redislabs/riot/RedisExportOptions.java b/core/src/main/java/com/redislabs/riot/RedisExportOptions.java new file mode 100644 index 000000000..c43580388 --- /dev/null +++ b/core/src/main/java/com/redislabs/riot/RedisExportOptions.java @@ -0,0 +1,24 @@ +package com.redislabs.riot; + +import lombok.Getter; +import org.springframework.batch.item.redis.support.RedisItemReaderBuilder; +import picocli.CommandLine; + +public class RedisExportOptions { + + @Getter + @CommandLine.Option(names = "--count", description = "SCAN COUNT option (default: ${DEFAULT-VALUE})", paramLabel = "") + private long scanCount = RedisItemReaderBuilder.DEFAULT_SCAN_COUNT; + @Getter + @CommandLine.Option(names = "--match", description = "SCAN MATCH pattern (default: ${DEFAULT-VALUE})", paramLabel = "") + private String scanMatch = RedisItemReaderBuilder.DEFAULT_SCAN_MATCH; + @Getter + @CommandLine.Option(names = "--reader-queue", description = "Capacity of the reader queue (default: ${DEFAULT-VALUE})", paramLabel = "", hidden = true) + private int queueCapacity = 10000; + @Getter + @CommandLine.Option(names = "--reader-threads", description = "Number of reader threads (default: ${DEFAULT-VALUE})", paramLabel = "", hidden = true) + private int threads = 1; + @Getter + @CommandLine.Option(names = "--reader-batch", description = "Number of reader values to process at once (default: ${DEFAULT-VALUE})", paramLabel = "") + private int batchSize = 50; +} diff --git a/src/main/java/com/redislabs/riot/cli/RedisCommandOptions.java b/core/src/main/java/com/redislabs/riot/RedisImportOptions.java similarity index 65% rename from src/main/java/com/redislabs/riot/cli/RedisCommandOptions.java rename to core/src/main/java/com/redislabs/riot/RedisImportOptions.java index d950e572c..6bb436c93 100644 --- a/src/main/java/com/redislabs/riot/cli/RedisCommandOptions.java +++ b/core/src/main/java/com/redislabs/riot/RedisImportOptions.java @@ -1,62 +1,62 @@ -package com.redislabs.riot.cli; +package com.redislabs.riot; +import com.redislabs.riot.convert.KeyMaker; import io.lettuce.core.ScriptOutputType; import lombok.Getter; import picocli.CommandLine; -public class RedisCommandOptions { +@Getter +public class RedisImportOptions { public enum StringFormat { RAW, XML, JSON } - @Getter + public enum CommandName { + EVALSHA, EXPIRE, GEOADD, FTADD, FTSEARCH, FTAGGREGATE, FTSUGADD, HMSET, LPUSH, NOOP, RPUSH, SADD, SET, XADD, ZADD + } + + @CommandLine.Option(names = "--command", description = "Redis command: ${COMPLETION-CANDIDATES} (default: ${DEFAULT-VALUE})", paramLabel = "") + private CommandName command = CommandName.HMSET; + @CommandLine.Option(names = "--key-separator", description = "Key separator (default: ${DEFAULT-VALUE})", paramLabel = "") + private String keySeparator = KeyMaker.DEFAULT_SEPARATOR; + @CommandLine.Option(names = {"-p", "--keyspace"}, description = "Keyspace prefix", paramLabel = "") + private String keyspace; + @CommandLine.Option(names = {"-k", "--keys"}, arity = "1..*", description = "Key fields", paramLabel = "") + private String[] keyFields = new String[0]; + @CommandLine.Option(names = "--member-space", description = "Prefix for member IDs", paramLabel = "") + private String memberSpace; + @CommandLine.Option(names = "--members", arity = "1..*", description = "Member field names for collections", paramLabel = "") + private String[] memberFields = new String[0]; @CommandLine.Option(names = "--eval-sha", description = "Digest", paramLabel = "") private String evalSha; - @Getter @CommandLine.Option(names = "--eval-args", arity = "1..*", description = "EVAL arg field names", paramLabel = "") private String[] evalArgs = new String[0]; - @Getter @CommandLine.Option(names = "--eval-output", description = "Output: ${COMPLETION-CANDIDATES} (default: ${DEFAULT-VALUE})", paramLabel = "") private ScriptOutputType evalOutputType = ScriptOutputType.STATUS; - @Getter @CommandLine.Option(names = "--lon", description = "Longitude field", paramLabel = "") private String longitudeField; - @Getter @CommandLine.Option(names = "--lat", description = "Latitude field", paramLabel = "") private String latitudeField; - @Getter @CommandLine.Option(names = "--score", description = "Name of the field to use for scores", paramLabel = "") private String field; - @Getter @CommandLine.Option(names = "--score-default", description = "Score when field not present (default: ${DEFAULT-VALUE})", paramLabel = "") private double defaultValue = 1d; - @Getter @CommandLine.Option(names = "--string-format", description = "Serialization: ${COMPLETION-CANDIDATES} (default: ${DEFAULT-VALUE})", paramLabel = "") private StringFormat format = StringFormat.JSON; - @Getter @CommandLine.Option(names = "--string-raw", description = "String raw value field", paramLabel = "") private String valueField; - @Getter @CommandLine.Option(names = "--ttl-default", description = "EXPIRE default timeout (default: ${DEFAULT-VALUE})", paramLabel = "") private long defaultTimeout = 60; - @Getter @CommandLine.Option(names = "--ttl", description = "EXPIRE timeout field", paramLabel = "") private String timeout; - @Getter @CommandLine.Option(names = "--xadd-id", description = "Stream entry ID field", paramLabel = "") private String idField; - @Getter @CommandLine.Option(names = "--xadd-maxlen", description = "Stream maxlen", paramLabel = "") private Long maxlen; - @Getter @CommandLine.Option(names = "--xadd-trim", description = "Stream efficient trimming (~ flag)") private boolean approximateTrimming; - @Getter @CommandLine.Option(names = "--xml-root", description = "XML root element name", paramLabel = "") private String root; - - - } diff --git a/core/src/main/java/com/redislabs/riot/RiotApp.java b/core/src/main/java/com/redislabs/riot/RiotApp.java new file mode 100644 index 000000000..f2431a03b --- /dev/null +++ b/core/src/main/java/com/redislabs/riot/RiotApp.java @@ -0,0 +1,168 @@ +package com.redislabs.riot; + +import com.redislabs.lettusearch.StatefulRediSearchConnection; +import com.redislabs.picocliredis.RedisApplication; +import io.lettuce.core.SslOptions; +import io.lettuce.core.api.StatefulConnection; +import io.lettuce.core.api.async.BaseRedisAsyncCommands; +import io.lettuce.core.api.sync.BaseRedisCommands; +import io.lettuce.core.cluster.ClusterClientOptions; +import io.lettuce.core.event.DefaultEventPublisherOptions; +import io.lettuce.core.event.metrics.CommandLatencyEvent; +import io.lettuce.core.metrics.DefaultCommandLatencyCollectorOptions; +import io.lettuce.core.resource.ClientResources; +import io.lettuce.core.resource.DefaultClientResources; +import io.netty.util.internal.logging.InternalLoggerFactory; +import io.netty.util.internal.logging.JdkLoggerFactory; +import lombok.Getter; +import org.springframework.batch.item.redis.support.ConnectionPoolConfig; +import org.springframework.batch.item.redis.support.RedisConnectionBuilder; +import org.springframework.batch.item.redisearch.support.RediSearchConnectionBuilder; +import picocli.CommandLine; + +import java.io.File; +import java.time.Duration; +import java.util.logging.ConsoleHandler; +import java.util.logging.Level; +import java.util.logging.LogManager; +import java.util.logging.Logger; + +@CommandLine.Command(sortOptions = false) +public class RiotApp extends RedisApplication { + + @Getter + @CommandLine.Option(names = {"-q", "--quiet"}, description = "Log errors only") + private boolean quiet; + @Getter + @CommandLine.Option(names = {"-d", "--debug"}, description = "Log in debug mode (includes normal stacktrace)") + private boolean debug; + @Getter + @CommandLine.Option(names = {"-i", "--info"}, description = "Set log level to info") + private boolean info; + @CommandLine.Option(names = "--ks", description = "Path to keystore", paramLabel = "", hidden = true) + private File keystore; + @CommandLine.Option(names = "--ks-password", arity = "0..1", interactive = true, description = "Keystore password", paramLabel = "", hidden = true) + private String keystorePassword; + @CommandLine.Option(names = "--ts", description = "Path to truststore", paramLabel = "", hidden = true) + private File truststore; + @CommandLine.Option(names = "--ts-password", arity = "0..1", interactive = true, description = "Truststore password", paramLabel = "", hidden = true) + private String truststorePassword; + @CommandLine.ArgGroup(heading = "Redis connection options%n", exclusive = false) + private RedisConnectionOptions redis = new RedisConnectionOptions(); + + private ClusterClientOptions clientOptions() { + SslOptions.Builder sslOptionsBuilder = SslOptions.builder(); + if (keystore != null) { + if (keystorePassword == null) { + sslOptionsBuilder.keystore(keystore); + } else { + sslOptionsBuilder.keystore(keystore, keystorePassword.toCharArray()); + } + } + if (truststore != null) { + if (truststorePassword == null) { + sslOptionsBuilder.truststore(truststore); + } else { + sslOptionsBuilder.truststore(truststore, truststorePassword); + } + } + return ClusterClientOptions.builder().sslOptions(sslOptionsBuilder.build()).build(); + } + + protected String getLoggerName() { + return "com.redislabs"; + } + + private static final String ROOT_LOGGER = ""; + + @Override + protected void configure() { + InternalLoggerFactory.setDefaultFactory(JdkLoggerFactory.INSTANCE); + LogManager.getLogManager().reset(); + Logger activeLogger = Logger.getLogger(ROOT_LOGGER); + ConsoleHandler handler = new ConsoleHandler(); + handler.setLevel(Level.ALL); + handler.setFormatter(new OneLineLogFormat(isDebug())); + activeLogger.addHandler(handler); + Logger.getLogger(ROOT_LOGGER).setLevel(rootLoggingLevel()); + Logger.getLogger(getLoggerName()).setLevel(packageLoggingLevel()); + } + + private Level packageLoggingLevel() { + if (isQuiet()) { + return Level.OFF; + } + if (isInfo()) { + return Level.FINE; + } + if (isDebug()) { + return Level.FINEST; + } + return Level.INFO; + } + + private Level rootLoggingLevel() { + if (isQuiet()) { + return Level.OFF; + } + if (isInfo()) { + return Level.INFO; + } + if (isDebug()) { + return Level.FINE; + } + return Level.SEVERE; + } + + public > B configure(RedisConnectionBuilder builder) { + return configure(builder, redis); + } + + public > B configure(RediSearchConnectionBuilder builder) { + return configure(builder, redis); + } + + public > B configure(RedisConnectionBuilder builder, RedisConnectionOptions redis) { + return builder.redisURI(redis.getRedisURI()).cluster(redis.isCluster()).clientResources(clientResources(redis)).clientOptions(clientOptions()).poolConfig(ConnectionPoolConfig.builder().maxTotal(redis.getPoolMaxTotal()).build()); + } + + public > B configure(RediSearchConnectionBuilder builder, RedisConnectionOptions redis) { + return builder.redisURI(redis.getRedisURI()).clientResources(clientResources(redis)).clientOptions(clientOptions()).poolConfig(org.springframework.batch.item.redisearch.support.ConnectionPoolConfig.builder().maxTotal(redis.getPoolMaxTotal()).build()); + } + + private ClientResources clientResources(RedisConnectionOptions redis) { + if (redis.isShowMetrics()) { + DefaultClientResources.Builder clientResourcesBuilder = DefaultClientResources.builder(); + clientResourcesBuilder.commandLatencyCollectorOptions(DefaultCommandLatencyCollectorOptions.builder().enable().build()); + clientResourcesBuilder.commandLatencyPublisherOptions(DefaultEventPublisherOptions.builder().eventEmitInterval(Duration.ofSeconds(1)).build()); + ClientResources resources = clientResourcesBuilder.build(); + resources.eventBus().get().filter(redisEvent -> redisEvent instanceof CommandLatencyEvent).cast(CommandLatencyEvent.class).subscribe(e -> System.out.println(e.getLatencies())); + return clientResourcesBuilder.build(); + } + return null; + } + + public StatefulConnection connection() { + RedisConnectionBuilder connectionBuilder = new RedisConnectionBuilder<>(); + configure(connectionBuilder); + return connectionBuilder.connection(); + } + + public BaseRedisCommands sync(StatefulConnection connection) { + RedisConnectionBuilder connectionBuilder = new RedisConnectionBuilder<>(); + configure(connectionBuilder); + return connectionBuilder.sync().apply(connection); + } + + public BaseRedisAsyncCommands async(StatefulConnection connection) { + RedisConnectionBuilder connectionBuilder = new RedisConnectionBuilder<>(); + configure(connectionBuilder); + return connectionBuilder.async().apply(connection); + } + + public StatefulRediSearchConnection rediSearchConnection() { + RediSearchConnectionBuilder connectionBuilder = new RediSearchConnectionBuilder<>(); + configure(connectionBuilder); + return connectionBuilder.connection(); + } +} diff --git a/src/main/java/com/redislabs/riot/Transfer.java b/core/src/main/java/com/redislabs/riot/Transfer.java similarity index 65% rename from src/main/java/com/redislabs/riot/Transfer.java rename to core/src/main/java/com/redislabs/riot/Transfer.java index 49e7f8273..c7f996eaf 100644 --- a/src/main/java/com/redislabs/riot/Transfer.java +++ b/core/src/main/java/com/redislabs/riot/Transfer.java @@ -1,11 +1,13 @@ package com.redislabs.riot; import lombok.Builder; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.springframework.batch.item.*; -import org.springframework.batch.item.redis.support.AbstractRedisItemReader; import org.springframework.batch.item.redis.support.BatchRunnable; +import org.springframework.batch.item.redis.support.RedisItemReader; import org.springframework.batch.item.support.AbstractItemCountingItemStreamItemReader; +import org.springframework.batch.item.support.SynchronizedItemStreamReader; import java.util.ArrayList; import java.util.concurrent.*; @@ -16,45 +18,51 @@ public class Transfer { private final ItemReader reader; private final ItemProcessor processor; private final ItemWriter writer; - private final TransferOptions options; - private final ExecutorService executor; + private final int threadCount; + private final int batchSize; + @Setter + private Long flushPeriod; + private final Integer maxItemCount; private final ArrayList> threads; @Builder - public Transfer(ItemReader reader, ItemProcessor processor, ItemWriter writer, TransferOptions options) { + public Transfer(ItemReader reader, ItemProcessor processor, ItemWriter writer, int threadCount, int batchSize, Long flushPeriod, Integer maxItemCount) { this.reader = reader; this.processor = processor; this.writer = writer; - this.options = options; - this.executor = Executors.newFixedThreadPool(options.getThreadCount()); - this.threads = new ArrayList<>(options.getThreadCount()); + this.threadCount = threadCount; + this.batchSize = batchSize; + this.flushPeriod = flushPeriod; + this.maxItemCount = maxItemCount; + this.threads = new ArrayList<>(threadCount); } public void execute() { + ExecutorService executor = Executors.newFixedThreadPool(threadCount); ExecutionContext executionContext = new ExecutionContext(); if (writer instanceof ItemStream) { log.debug("Opening writer"); ((ItemStream) writer).open(executionContext); } if (reader instanceof ItemStream) { - if (options.getMaxItemCount() != null) { + if (maxItemCount != null) { if (reader instanceof AbstractItemCountingItemStreamItemReader) { - log.debug("Configuring reader with maxItemCount={}", options.getMaxItemCount()); - ((AbstractItemCountingItemStreamItemReader) reader).setMaxItemCount(options.getMaxItemCount()); + log.debug("Configuring reader with maxItemCount={}", maxItemCount); + ((AbstractItemCountingItemStreamItemReader) reader).setMaxItemCount(maxItemCount); } } log.debug("Opening reader"); ((ItemStream) reader).open(executionContext); } - for (int index = 0; index < options.getThreadCount(); index++) { - threads.add(new BatchRunnable<>(reader, new ProcessingItemWriter<>(processor, writer), options.getBatchSize())); + for (int index = 0; index < threadCount; index++) { + threads.add(new BatchRunnable<>(reader(), new ProcessingItemWriter<>(processor, writer), batchSize)); } threads.forEach(executor::submit); executor.shutdown(); ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); ScheduledFuture scheduledFuture = null; - if (options.getFlushPeriod() != null) { - scheduledFuture = scheduler.scheduleAtFixedRate(this::flush, options.getFlushPeriod(), options.getFlushPeriod(), TimeUnit.MILLISECONDS); + if (flushPeriod != null) { + scheduledFuture = scheduler.scheduleAtFixedRate(this::flush, flushPeriod, flushPeriod, TimeUnit.MILLISECONDS); } try { while (!executor.isTerminated()) { @@ -81,9 +89,18 @@ public void execute() { } } + private ItemReader reader() { + if (threadCount > 1 && reader instanceof ItemStreamReader) { + SynchronizedItemStreamReader synchronizedReader = new SynchronizedItemStreamReader<>(); + synchronizedReader.setDelegate((ItemStreamReader) reader); + return synchronizedReader; + } + return reader; + } + private void flush() { - if (reader instanceof AbstractRedisItemReader) { - ((AbstractRedisItemReader) reader).flush(); + if (reader instanceof RedisItemReader) { + ((RedisItemReader) reader).flush(); } for (BatchRunnable thread : threads) { try { diff --git a/src/main/java/com/redislabs/riot/convert/ConversionServiceConverter.java b/core/src/main/java/com/redislabs/riot/convert/ConversionServiceConverter.java similarity index 100% rename from src/main/java/com/redislabs/riot/convert/ConversionServiceConverter.java rename to core/src/main/java/com/redislabs/riot/convert/ConversionServiceConverter.java diff --git a/src/main/java/com/redislabs/riot/convert/IdemConverter.java b/core/src/main/java/com/redislabs/riot/convert/IdemConverter.java similarity index 100% rename from src/main/java/com/redislabs/riot/convert/IdemConverter.java rename to core/src/main/java/com/redislabs/riot/convert/IdemConverter.java diff --git a/src/main/java/com/redislabs/riot/convert/KeyMaker.java b/core/src/main/java/com/redislabs/riot/convert/KeyMaker.java similarity index 100% rename from src/main/java/com/redislabs/riot/convert/KeyMaker.java rename to core/src/main/java/com/redislabs/riot/convert/KeyMaker.java diff --git a/src/main/java/com/redislabs/riot/convert/NullConverter.java b/core/src/main/java/com/redislabs/riot/convert/NullConverter.java similarity index 100% rename from src/main/java/com/redislabs/riot/convert/NullConverter.java rename to core/src/main/java/com/redislabs/riot/convert/NullConverter.java diff --git a/src/main/java/com/redislabs/riot/convert/ObjectMapperConverter.java b/core/src/main/java/com/redislabs/riot/convert/ObjectMapperConverter.java similarity index 100% rename from src/main/java/com/redislabs/riot/convert/ObjectMapperConverter.java rename to core/src/main/java/com/redislabs/riot/convert/ObjectMapperConverter.java diff --git a/src/main/java/com/redislabs/riot/convert/type/StringToDoubleConverter.java b/core/src/main/java/com/redislabs/riot/convert/StringToDoubleConverter.java similarity index 88% rename from src/main/java/com/redislabs/riot/convert/type/StringToDoubleConverter.java rename to core/src/main/java/com/redislabs/riot/convert/StringToDoubleConverter.java index 95d9609c6..fb3c5fe81 100644 --- a/src/main/java/com/redislabs/riot/convert/type/StringToDoubleConverter.java +++ b/core/src/main/java/com/redislabs/riot/convert/StringToDoubleConverter.java @@ -1,4 +1,4 @@ -package com.redislabs.riot.convert.type; +package com.redislabs.riot.convert; import org.springframework.core.convert.converter.Converter; diff --git a/src/main/java/com/redislabs/riot/convert/type/StringToIntegerConverter.java b/core/src/main/java/com/redislabs/riot/convert/StringToIntegerConverter.java similarity index 88% rename from src/main/java/com/redislabs/riot/convert/type/StringToIntegerConverter.java rename to core/src/main/java/com/redislabs/riot/convert/StringToIntegerConverter.java index 9f584f449..55466cb60 100644 --- a/src/main/java/com/redislabs/riot/convert/type/StringToIntegerConverter.java +++ b/core/src/main/java/com/redislabs/riot/convert/StringToIntegerConverter.java @@ -1,4 +1,4 @@ -package com.redislabs.riot.convert.type; +package com.redislabs.riot.convert; import org.springframework.core.convert.converter.Converter; diff --git a/src/main/java/com/redislabs/riot/convert/type/StringToLongConverter.java b/core/src/main/java/com/redislabs/riot/convert/StringToLongConverter.java similarity index 88% rename from src/main/java/com/redislabs/riot/convert/type/StringToLongConverter.java rename to core/src/main/java/com/redislabs/riot/convert/StringToLongConverter.java index 71dda4161..67de0382e 100644 --- a/src/main/java/com/redislabs/riot/convert/type/StringToLongConverter.java +++ b/core/src/main/java/com/redislabs/riot/convert/StringToLongConverter.java @@ -1,4 +1,4 @@ -package com.redislabs.riot.convert.type; +package com.redislabs.riot.convert; import org.springframework.core.convert.converter.Converter; diff --git a/src/main/java/com/redislabs/riot/convert/field/ConstantFieldExtractor.java b/core/src/main/java/com/redislabs/riot/convert/field/ConstantFieldExtractor.java similarity index 100% rename from src/main/java/com/redislabs/riot/convert/field/ConstantFieldExtractor.java rename to core/src/main/java/com/redislabs/riot/convert/field/ConstantFieldExtractor.java diff --git a/src/main/java/com/redislabs/riot/convert/field/DefaultingCompositeConverter.java b/core/src/main/java/com/redislabs/riot/convert/field/DefaultingCompositeConverter.java similarity index 100% rename from src/main/java/com/redislabs/riot/convert/field/DefaultingCompositeConverter.java rename to core/src/main/java/com/redislabs/riot/convert/field/DefaultingCompositeConverter.java diff --git a/src/main/java/com/redislabs/riot/convert/field/DefaultingFieldExtractor.java b/core/src/main/java/com/redislabs/riot/convert/field/DefaultingFieldExtractor.java similarity index 100% rename from src/main/java/com/redislabs/riot/convert/field/DefaultingFieldExtractor.java rename to core/src/main/java/com/redislabs/riot/convert/field/DefaultingFieldExtractor.java diff --git a/src/main/java/com/redislabs/riot/convert/field/FieldExtractor.java b/core/src/main/java/com/redislabs/riot/convert/field/FieldExtractor.java similarity index 90% rename from src/main/java/com/redislabs/riot/convert/field/FieldExtractor.java rename to core/src/main/java/com/redislabs/riot/convert/field/FieldExtractor.java index 74f9a96d3..15f4d6759 100644 --- a/src/main/java/com/redislabs/riot/convert/field/FieldExtractor.java +++ b/core/src/main/java/com/redislabs/riot/convert/field/FieldExtractor.java @@ -1,8 +1,8 @@ package com.redislabs.riot.convert.field; -import com.redislabs.riot.convert.type.StringToDoubleConverter; -import com.redislabs.riot.convert.type.StringToIntegerConverter; -import com.redislabs.riot.convert.type.StringToLongConverter; +import com.redislabs.riot.convert.StringToDoubleConverter; +import com.redislabs.riot.convert.StringToIntegerConverter; +import com.redislabs.riot.convert.StringToLongConverter; import lombok.Setter; import lombok.experimental.Accessors; import org.springframework.core.convert.converter.Converter; @@ -51,7 +51,7 @@ public Converter, T> build() { if (defaultValue == null) { return null; } - return new ConstantFieldExtractor<>(defaultValue); + return new ConstantFieldExtractor, T>(defaultValue); } if (String.class.isAssignableFrom(targetType)) { if (defaultValue == null) { diff --git a/src/main/java/com/redislabs/riot/convert/field/MapToArrayConverter.java b/core/src/main/java/com/redislabs/riot/convert/field/MapToArrayConverter.java similarity index 100% rename from src/main/java/com/redislabs/riot/convert/field/MapToArrayConverter.java rename to core/src/main/java/com/redislabs/riot/convert/field/MapToArrayConverter.java diff --git a/src/main/java/com/redislabs/riot/convert/field/RemovingFieldExtractor.java b/core/src/main/java/com/redislabs/riot/convert/field/RemovingFieldExtractor.java similarity index 100% rename from src/main/java/com/redislabs/riot/convert/field/RemovingFieldExtractor.java rename to core/src/main/java/com/redislabs/riot/convert/field/RemovingFieldExtractor.java diff --git a/src/main/java/com/redislabs/riot/convert/field/SimpleFieldExtractor.java b/core/src/main/java/com/redislabs/riot/convert/field/SimpleFieldExtractor.java similarity index 100% rename from src/main/java/com/redislabs/riot/convert/field/SimpleFieldExtractor.java rename to core/src/main/java/com/redislabs/riot/convert/field/SimpleFieldExtractor.java diff --git a/src/main/java/com/redislabs/riot/convert/map/AbstractMapConverter.java b/core/src/main/java/com/redislabs/riot/convert/map/AbstractMapConverter.java similarity index 100% rename from src/main/java/com/redislabs/riot/convert/map/AbstractMapConverter.java rename to core/src/main/java/com/redislabs/riot/convert/map/AbstractMapConverter.java diff --git a/src/main/java/com/redislabs/riot/convert/map/AbstractMapToCollectionConverter.java b/core/src/main/java/com/redislabs/riot/convert/map/AbstractMapToCollectionConverter.java similarity index 100% rename from src/main/java/com/redislabs/riot/convert/map/AbstractMapToCollectionConverter.java rename to core/src/main/java/com/redislabs/riot/convert/map/AbstractMapToCollectionConverter.java diff --git a/src/main/java/com/redislabs/riot/convert/map/CollectionToStringMapConverter.java b/core/src/main/java/com/redislabs/riot/convert/map/CollectionToStringMapConverter.java similarity index 100% rename from src/main/java/com/redislabs/riot/convert/map/CollectionToStringMapConverter.java rename to core/src/main/java/com/redislabs/riot/convert/map/CollectionToStringMapConverter.java diff --git a/src/main/java/com/redislabs/riot/convert/map/RegexNamedGroupsExtractor.java b/core/src/main/java/com/redislabs/riot/convert/map/RegexNamedGroupsExtractor.java similarity index 100% rename from src/main/java/com/redislabs/riot/convert/map/RegexNamedGroupsExtractor.java rename to core/src/main/java/com/redislabs/riot/convert/map/RegexNamedGroupsExtractor.java diff --git a/src/main/java/com/redislabs/riot/convert/map/StreamToStringMapConverter.java b/core/src/main/java/com/redislabs/riot/convert/map/StreamToStringMapConverter.java similarity index 100% rename from src/main/java/com/redislabs/riot/convert/map/StreamToStringMapConverter.java rename to core/src/main/java/com/redislabs/riot/convert/map/StreamToStringMapConverter.java diff --git a/src/main/java/com/redislabs/riot/convert/map/StringToStringMapConverter.java b/core/src/main/java/com/redislabs/riot/convert/map/StringToStringMapConverter.java similarity index 100% rename from src/main/java/com/redislabs/riot/convert/map/StringToStringMapConverter.java rename to core/src/main/java/com/redislabs/riot/convert/map/StringToStringMapConverter.java diff --git a/src/main/java/com/redislabs/riot/convert/map/ZsetToStringMapConverter.java b/core/src/main/java/com/redislabs/riot/convert/map/ZsetToStringMapConverter.java similarity index 100% rename from src/main/java/com/redislabs/riot/convert/map/ZsetToStringMapConverter.java rename to core/src/main/java/com/redislabs/riot/convert/map/ZsetToStringMapConverter.java diff --git a/src/main/java/com/redislabs/riot/processor/KeyValueItemProcessor.java b/core/src/main/java/com/redislabs/riot/processor/KeyValueItemProcessor.java similarity index 98% rename from src/main/java/com/redislabs/riot/processor/KeyValueItemProcessor.java rename to core/src/main/java/com/redislabs/riot/processor/KeyValueItemProcessor.java index 6eeb5c0e8..c47098e5f 100644 --- a/src/main/java/com/redislabs/riot/processor/KeyValueItemProcessor.java +++ b/core/src/main/java/com/redislabs/riot/processor/KeyValueItemProcessor.java @@ -8,7 +8,7 @@ import lombok.Setter; import lombok.experimental.Accessors; import org.springframework.batch.item.ItemProcessor; -import org.springframework.batch.item.redis.KeyValue; +import org.springframework.batch.item.redis.support.KeyValue; import org.springframework.core.convert.converter.Converter; import java.util.HashMap; diff --git a/src/main/java/com/redislabs/riot/processor/MapAccessor.java b/core/src/main/java/com/redislabs/riot/processor/MapAccessor.java similarity index 100% rename from src/main/java/com/redislabs/riot/processor/MapAccessor.java rename to core/src/main/java/com/redislabs/riot/processor/MapAccessor.java diff --git a/src/main/java/com/redislabs/riot/processor/MapFlattener.java b/core/src/main/java/com/redislabs/riot/processor/MapFlattener.java similarity index 100% rename from src/main/java/com/redislabs/riot/processor/MapFlattener.java rename to core/src/main/java/com/redislabs/riot/processor/MapFlattener.java diff --git a/src/main/java/com/redislabs/riot/processor/MapProcessor.java b/core/src/main/java/com/redislabs/riot/processor/MapProcessor.java similarity index 100% rename from src/main/java/com/redislabs/riot/processor/MapProcessor.java rename to core/src/main/java/com/redislabs/riot/processor/MapProcessor.java diff --git a/src/main/java/com/redislabs/riot/processor/ObjectMapToStringMapProcessor.java b/core/src/main/java/com/redislabs/riot/processor/ObjectMapToStringMapProcessor.java similarity index 100% rename from src/main/java/com/redislabs/riot/processor/ObjectMapToStringMapProcessor.java rename to core/src/main/java/com/redislabs/riot/processor/ObjectMapToStringMapProcessor.java diff --git a/src/main/java/com/redislabs/riot/processor/SpelProcessor.java b/core/src/main/java/com/redislabs/riot/processor/SpelProcessor.java similarity index 76% rename from src/main/java/com/redislabs/riot/processor/SpelProcessor.java rename to core/src/main/java/com/redislabs/riot/processor/SpelProcessor.java index 404c25d45..d637c8ce4 100644 --- a/src/main/java/com/redislabs/riot/processor/SpelProcessor.java +++ b/core/src/main/java/com/redislabs/riot/processor/SpelProcessor.java @@ -2,9 +2,11 @@ import io.lettuce.core.api.StatefulConnection; import io.lettuce.core.api.sync.BaseRedisCommands; -import lombok.Builder; +import lombok.Setter; +import lombok.experimental.Accessors; import lombok.extern.slf4j.Slf4j; import org.springframework.batch.item.ItemProcessor; +import org.springframework.batch.item.redis.support.RedisConnectionBuilder; import org.springframework.expression.Expression; import org.springframework.expression.ExpressionInvocationTargetException; import org.springframework.expression.spel.standard.SpelExpressionParser; @@ -13,7 +15,10 @@ import java.lang.reflect.Method; import java.text.DateFormat; -import java.util.*; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; @@ -25,8 +30,7 @@ public class SpelProcessor implements ItemProcessor, Map expressions = new LinkedHashMap<>(); private final AtomicLong index = new AtomicLong(); - @Builder - private SpelProcessor(StatefulConnection connection, Function, BaseRedisCommands> commands, DateFormat dateFormat, Map variables, Map fields) { + public SpelProcessor(StatefulConnection connection, Function, BaseRedisCommands> commands, DateFormat dateFormat, Map variables, Map fields) { Assert.notNull(connection, "A Redis connection is required."); Assert.notNull(commands, "A connection -> commands function is required."); Assert.notNull(dateFormat, "A DateFormat instance is required."); @@ -78,4 +82,21 @@ public static String geo(String longitude, String latitude) { return longitude + "," + latitude; } + public static SpelProcessorBuilder builder() { + return new SpelProcessorBuilder(); + } + + @Setter + @Accessors(fluent = true) + public static class SpelProcessorBuilder extends RedisConnectionBuilder { + + private DateFormat dateFormat; + private Map variables; + private Map fields; + + public SpelProcessor build() { + return new SpelProcessor(connection(), sync(), dateFormat, variables, fields); + } + } + } diff --git a/src/test/java/com/redislabs/riot/TestKeyMaker.java b/core/src/test/java/com/redislabs/riot/TestKeyMaker.java similarity index 100% rename from src/test/java/com/redislabs/riot/TestKeyMaker.java rename to core/src/test/java/com/redislabs/riot/TestKeyMaker.java diff --git a/src/test/java/com/redislabs/riot/TestMapFlattener.java b/core/src/test/java/com/redislabs/riot/TestMapFlattener.java similarity index 100% rename from src/test/java/com/redislabs/riot/TestMapFlattener.java rename to core/src/test/java/com/redislabs/riot/TestMapFlattener.java diff --git a/db/build.gradle b/db/build.gradle new file mode 100644 index 000000000..2ca74771f --- /dev/null +++ b/db/build.gradle @@ -0,0 +1,43 @@ +plugins { + id 'application' +} + +archivesBaseName = 'riot-db' + +bootJar { + enabled = false +} + +jar { + enabled = true +} + +test { + useJUnitPlatform() +} + +dependencies { + implementation project(':core') + implementation 'org.slf4j:slf4j-jdk14' + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' + implementation 'org.springframework.boot:spring-boot-autoconfigure' + implementation 'org.springframework:spring-jdbc' + implementation 'com.zaxxer:HikariCP' + implementation 'org.xerial:sqlite-jdbc:3.28.0' + implementation 'com.ibm.db2:jcc:11.5.0.0' + implementation 'com.microsoft.sqlserver:mssql-jdbc:7.4.1.jre8' + implementation 'com.oracle.ojdbc:ojdbc8:19.3.0.0' + implementation 'org.postgresql:postgresql:42.2.8' + testImplementation project(':test') + testImplementation 'org.hsqldb:hsqldb' +} + +configurations { + all*.exclude module: 'spring-boot-starter-logging' +} + +application { + mainClassName = 'com.redislabs.riot.db.App' + applicationName = 'riot-db' +} \ No newline at end of file diff --git a/db/src/main/java/com/redislabs/riot/db/App.java b/db/src/main/java/com/redislabs/riot/db/App.java new file mode 100644 index 000000000..ea554cba1 --- /dev/null +++ b/db/src/main/java/com/redislabs/riot/db/App.java @@ -0,0 +1,12 @@ +package com.redislabs.riot.db; + +import com.redislabs.riot.RiotApp; +import picocli.CommandLine; + +@CommandLine.Command(name = "riot-db", subcommands = {DatabaseImportCommand.class, DatabaseExportCommand.class}) +public class App extends RiotApp { + + public static void main(String[] args) { + System.exit(new App().execute(args)); + } +} diff --git a/src/main/java/com/redislabs/riot/cli/DatabaseExportCommand.java b/db/src/main/java/com/redislabs/riot/db/DatabaseExportCommand.java similarity index 78% rename from src/main/java/com/redislabs/riot/cli/DatabaseExportCommand.java rename to db/src/main/java/com/redislabs/riot/db/DatabaseExportCommand.java index aa988125a..02050e3d7 100644 --- a/src/main/java/com/redislabs/riot/cli/DatabaseExportCommand.java +++ b/db/src/main/java/com/redislabs/riot/db/DatabaseExportCommand.java @@ -1,18 +1,21 @@ -package com.redislabs.riot.cli; +package com.redislabs.riot.db; +import com.redislabs.riot.AbstractExportCommand; import com.redislabs.riot.processor.KeyValueItemProcessor; import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.database.JdbcBatchItemWriter; import org.springframework.batch.item.database.builder.JdbcBatchItemWriterBuilder; -import org.springframework.batch.item.redis.KeyValue; +import org.springframework.batch.item.redis.support.KeyValue; import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; import picocli.CommandLine; import java.util.Map; -@CommandLine.Command(name = "db", description = "Export to database") +@CommandLine.Command(name = "export", description = "Export to database") public class DatabaseExportCommand extends AbstractExportCommand> { + @CommandLine.Parameters(arity = "1", description = "SQL INSERT statement", paramLabel = "SQL") + private String sql; @CommandLine.Mixin private DatabaseOptions options = new DatabaseOptions(); @CommandLine.Option(names = "--no-assert-updates", description = "Disable insert verification") @@ -23,7 +26,7 @@ protected JdbcBatchItemWriter> writer() { JdbcBatchItemWriterBuilder> builder = new JdbcBatchItemWriterBuilder<>(); builder.itemSqlParameterSourceProvider(MapSqlParameterSource::new); builder.dataSource(options.getDataSource()); - builder.sql(options.getSql()); + builder.sql(sql); builder.assertUpdates(!noAssertUpdates); JdbcBatchItemWriter> writer = builder.build(); writer.afterPropertiesSet(); diff --git a/src/main/java/com/redislabs/riot/cli/DatabaseImportCommand.java b/db/src/main/java/com/redislabs/riot/db/DatabaseImportCommand.java similarity index 86% rename from src/main/java/com/redislabs/riot/cli/DatabaseImportCommand.java rename to db/src/main/java/com/redislabs/riot/db/DatabaseImportCommand.java index 7383f3d3c..c715c6c4f 100644 --- a/src/main/java/com/redislabs/riot/cli/DatabaseImportCommand.java +++ b/db/src/main/java/com/redislabs/riot/db/DatabaseImportCommand.java @@ -1,5 +1,6 @@ -package com.redislabs.riot.cli; +package com.redislabs.riot.db; +import com.redislabs.riot.AbstractImportCommand; import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.database.JdbcCursorItemReader; import org.springframework.batch.item.database.builder.JdbcCursorItemReaderBuilder; @@ -8,9 +9,11 @@ import java.util.Map; -@CommandLine.Command(name = "db", description = "Import database") +@CommandLine.Command(name = "import", description = "Import database") public class DatabaseImportCommand extends AbstractImportCommand> { + @CommandLine.Parameters(arity = "1", description = "SQL SELECT statement", paramLabel = "SQL") + private String sql; @CommandLine.Mixin private DatabaseOptions options = new DatabaseOptions(); @CommandLine.Option(names = "--fetch", description = "Number of rows to return with each fetch", paramLabel = "") @@ -39,7 +42,7 @@ protected JdbcCursorItemReader> reader() throws Exception { builder.queryTimeout(queryTimeout); } builder.rowMapper(new ColumnMapRowMapper()); - builder.sql(options.getSql()); + builder.sql(sql); builder.useSharedExtendedConnection(useSharedExtendedConnection); builder.verifyCursorPosition(verifyCursorPosition); JdbcCursorItemReader> reader = builder.build(); @@ -48,7 +51,8 @@ protected JdbcCursorItemReader> reader() throws Exception { } @Override + @SuppressWarnings("unchecked") protected ItemProcessor, Object> processor() { - return getParentCommand().objectMapProcessor(); + return objectMapProcessor(); } } diff --git a/src/main/java/com/redislabs/riot/cli/DatabaseOptions.java b/db/src/main/java/com/redislabs/riot/db/DatabaseOptions.java similarity index 86% rename from src/main/java/com/redislabs/riot/cli/DatabaseOptions.java rename to db/src/main/java/com/redislabs/riot/db/DatabaseOptions.java index 989cfa056..3461527ce 100644 --- a/src/main/java/com/redislabs/riot/cli/DatabaseOptions.java +++ b/db/src/main/java/com/redislabs/riot/db/DatabaseOptions.java @@ -1,22 +1,15 @@ -package com.redislabs.riot.cli; - -import javax.sql.DataSource; - -import lombok.Getter; -import org.springframework.boot.autoconfigure.jdbc.DataSourceProperties; +package com.redislabs.riot.db; import com.zaxxer.hikari.HikariDataSource; - import lombok.extern.slf4j.Slf4j; -import picocli.CommandLine; +import org.springframework.boot.autoconfigure.jdbc.DataSourceProperties; import picocli.CommandLine.Option; +import javax.sql.DataSource; + @Slf4j public class DatabaseOptions { - @Getter - @CommandLine.Parameters(arity = "1", description = "SQL statement", paramLabel = "SQL") - private String sql; @Option(names = "--driver", description = "Fully qualified name of the JDBC driver", paramLabel = "") private String driver; @Option(names = "--url", required = true, description = "JDBC URL to connect to the database", paramLabel = "") diff --git a/db/src/test/java/com/redislabs/riot/db/TestRiotDb.java b/db/src/test/java/com/redislabs/riot/db/TestRiotDb.java new file mode 100644 index 000000000..034ecf163 --- /dev/null +++ b/db/src/test/java/com/redislabs/riot/db/TestRiotDb.java @@ -0,0 +1,70 @@ +package com.redislabs.riot.db; + +import com.redislabs.riot.test.BaseTest; +import com.redislabs.riot.test.DataPopulator; +import com.zaxxer.hikari.HikariDataSource; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.springframework.batch.item.redis.support.DataType; +import org.springframework.boot.autoconfigure.jdbc.DataSourceProperties; + +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Collections; +import java.util.List; + +public class TestRiotDb extends BaseTest { + + private final static int COUNT = 1234; + + @Override + protected int execute(String[] args) { + return new App().execute(args); + } + + @Override + protected String applicationName() { + return "riot-db"; + } + + private void populateBeersTable() throws Exception { + Connection connection = dataSource().getConnection(); + connection.createStatement().execute("DROP TABLE IF EXISTS mytable"); + connection.createStatement().execute("CREATE TABLE IF NOT EXISTS mytable (id INT NOT NULL, field1 VARCHAR(500), field2 VARCHAR(500), PRIMARY KEY (id))"); + DataPopulator.builder().connection(super.connection).dataTypes(Collections.singletonList(DataType.HASH)).start(0).end(1234).build().run(); + runFile("/export-db.txt"); + commands().flushall(); + } + + private DataSource dataSource() { + DataSourceProperties properties = new DataSourceProperties(); + properties.setUrl("jdbc:hsqldb:mem:mymemdb"); + properties.setDriverClassName("org.hsqldb.jdbc.JDBCDriver"); + return properties.initializeDataSourceBuilder().type(HikariDataSource.class).build(); + } + + @Test + public void testExportToDatabase() throws Exception { + populateBeersTable(); + Statement statement = dataSource().getConnection().createStatement(); + statement.execute("SELECT * from mytable ORDER BY id ASC"); + ResultSet resultSet = statement.getResultSet(); + int index = 0; + while (resultSet.next()) { + Assertions.assertTrue(resultSet.getInt("id") == index); + index++; + } + Assertions.assertEquals(COUNT, index); + } + + @Test + public void testImportDatabase() throws Exception { + populateBeersTable(); + runFile("/import-db.txt"); + List keys = commands().keys("dbtest:*"); + Assertions.assertEquals(COUNT, keys.size()); + } + +} diff --git a/db/src/test/resources/export-db.txt b/db/src/test/resources/export-db.txt new file mode 100644 index 000000000..9544b2a57 --- /dev/null +++ b/db/src/test/resources/export-db.txt @@ -0,0 +1 @@ +❯ riot-db -r redis://localhost:6379 export "INSERT INTO mytable (id, field1, field2) VALUES (:id, :field1, :field2)" --url "jdbc:hsqldb:mem:mymemdb" --match "hash:*" --key-regex "hash:(?.*)" \ No newline at end of file diff --git a/db/src/test/resources/import-db.txt b/db/src/test/resources/import-db.txt new file mode 100644 index 000000000..58a6d6130 --- /dev/null +++ b/db/src/test/resources/import-db.txt @@ -0,0 +1 @@ +❯ riot-db import "SELECT * FROM mytable" --url "jdbc:hsqldb:mem:mymemdb" --keyspace dbtest --keys id \ No newline at end of file diff --git a/file/build.gradle b/file/build.gradle new file mode 100644 index 000000000..cd41d643f --- /dev/null +++ b/file/build.gradle @@ -0,0 +1,51 @@ +plugins { + id 'application' +} + +archivesBaseName = 'riot-file' + +bootJar { + enabled = false +} + +jar { + enabled = true +} + +test { + useJUnitPlatform() +} + +dependencies { + implementation project(':core') + implementation 'org.slf4j:slf4j-jdk14' + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' + implementation 'org.springframework.cloud:spring-cloud-aws-context' + implementation 'org.springframework.cloud:spring-cloud-aws-autoconfigure' + implementation 'org.ruaux:spring-batch-resource:1.0.0' + implementation 'org.ruaux:spring-batch-xml:1.0.1' + implementation 'org.springframework:spring-oxm' + implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml' + testImplementation project(':test') + testImplementation 'org.ruaux:spring-batch-faker:1.0.1' +} + +configurations { + all*.exclude module: 'spring-boot-starter-logging' +} + +ext { + set('springCloudVersion', "Hoxton.SR5") +} + +dependencyManagement { + imports { + mavenBom "org.springframework.cloud:spring-cloud-dependencies:${springCloudVersion}" + } +} + +application { + mainClassName = 'com.redislabs.riot.file.App' + applicationName = 'riot-file' +} \ No newline at end of file diff --git a/file/src/main/java/com/redislabs/riot/file/App.java b/file/src/main/java/com/redislabs/riot/file/App.java new file mode 100644 index 000000000..f4cf7799c --- /dev/null +++ b/file/src/main/java/com/redislabs/riot/file/App.java @@ -0,0 +1,19 @@ +package com.redislabs.riot.file; + +import com.redislabs.riot.RiotApp; +import org.springframework.batch.item.file.transform.Range; +import picocli.CommandLine; + +@CommandLine.Command(name = "riot-file", subcommands = {FileImportCommand.class, FileExportCommand.class}) +public class App extends RiotApp { + + @Override + protected void registerConverters(CommandLine commandLine) { + commandLine.registerConverter(Range.class, new RangeConverter()); + super.registerConverters(commandLine); + } + + public static void main(String[] args) { + System.exit(new App().execute(args)); + } +} diff --git a/src/main/java/com/redislabs/riot/cli/FileExportCommand.java b/file/src/main/java/com/redislabs/riot/file/FileExportCommand.java similarity index 95% rename from src/main/java/com/redislabs/riot/cli/FileExportCommand.java rename to file/src/main/java/com/redislabs/riot/file/FileExportCommand.java index 943e3f51f..70d7f4e33 100644 --- a/src/main/java/com/redislabs/riot/cli/FileExportCommand.java +++ b/file/src/main/java/com/redislabs/riot/file/FileExportCommand.java @@ -1,14 +1,12 @@ -package com.redislabs.riot.cli; +package com.redislabs.riot.file; -import com.redislabs.riot.cli.file.GZIPOutputStreamResource; -import com.redislabs.riot.cli.file.HeaderCallback; -import com.redislabs.riot.cli.file.MapFieldExtractor; +import com.redislabs.riot.AbstractExportCommand; import com.redislabs.riot.processor.KeyValueItemProcessor; import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.file.FlatFileHeaderCallback; import org.springframework.batch.item.file.FlatFileItemWriter; import org.springframework.batch.item.json.JacksonJsonObjectMarshaller; -import org.springframework.batch.item.redis.KeyValue; +import org.springframework.batch.item.redis.support.KeyValue; import org.springframework.batch.item.resource.*; import org.springframework.batch.item.support.AbstractItemStreamItemWriter; import org.springframework.batch.item.support.PassThroughItemProcessor; @@ -24,7 +22,7 @@ import java.util.Locale; import java.util.Map; -@CommandLine.Command(name = "file", description = "Export file") +@CommandLine.Command(name = "export", description = "Export to file") public class FileExportCommand extends AbstractExportCommand { @CommandLine.Mixin @@ -46,6 +44,18 @@ public class FileExportCommand extends AbstractExportCommand { @CommandLine.Option(names = "--root", description = "XML root element tag name", paramLabel = "") private String root; + @Override + protected ItemProcessor, Object> processor() throws Exception { + switch (options.getFileType()) { + case DELIMITED: + case FIXED: + return (ItemProcessor) KeyValueItemProcessor.builder().build(); + case JSON: + case XML: + return (ItemProcessor) new PassThroughItemProcessor>(); + } + throw new IllegalArgumentException("Unknown file type"); + } private WritableResource resource() throws IOException { if (options.isConsole()) { @@ -76,20 +86,6 @@ protected AbstractItemStreamItemWriter writer() throws IOException { throw new IllegalArgumentException("Unknown file type"); } - @Override - @SuppressWarnings({"rawtypes", "unchecked"}) - protected ItemProcessor, Object> processor() { - switch (options.getFileType()) { - case DELIMITED: - case FIXED: - return (ItemProcessor) KeyValueItemProcessor.builder().build(); - case JSON: - case XML: - return (ItemProcessor) new PassThroughItemProcessor>(); - } - throw new IllegalArgumentException("Unknown file type"); - } - private FlatResourceItemWriterBuilder> flatWriterBuilder() throws IOException { FlatResourceItemWriterBuilder> builder = new FlatResourceItemWriterBuilder<>(); builder.append(append); diff --git a/src/main/java/com/redislabs/riot/cli/FileImportCommand.java b/file/src/main/java/com/redislabs/riot/file/FileImportCommand.java similarity index 93% rename from src/main/java/com/redislabs/riot/cli/FileImportCommand.java rename to file/src/main/java/com/redislabs/riot/file/FileImportCommand.java index c383a91b1..1f94b69a3 100644 --- a/src/main/java/com/redislabs/riot/cli/FileImportCommand.java +++ b/file/src/main/java/com/redislabs/riot/file/FileImportCommand.java @@ -1,10 +1,8 @@ -package com.redislabs.riot.cli; +package com.redislabs.riot.file; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.xml.XmlMapper; -import com.redislabs.riot.cli.file.FileType; -import com.redislabs.riot.cli.file.GZIPInputStreamResource; -import com.redislabs.riot.cli.file.MapFieldSetMapper; +import com.redislabs.riot.AbstractImportCommand; import com.redislabs.riot.processor.MapFlattener; import com.redislabs.riot.processor.MapProcessor; import com.redislabs.riot.processor.ObjectMapToStringMapProcessor; @@ -30,7 +28,7 @@ import java.util.*; @Slf4j -@CommandLine.Command(name = "file", description = "Import file") +@CommandLine.Command(name = "import", description = "Import file") public class FileImportCommand extends AbstractImportCommand> { @CommandLine.Mixin @@ -148,7 +146,7 @@ protected ItemProcessor, Object> processor() { return stringMapProcessor(); case JSON: case XML: - return getParentCommand().processor(Collections.singletonList(MapFlattener.builder().build())); + return processor(Collections.singletonList(MapFlattener.builder().build())); } throw new IllegalArgumentException("Unknown file type: " + fileType); } @@ -160,10 +158,10 @@ private ItemProcessor stringMapProcessor() { if (!regexes.isEmpty()) { processors.add(MapProcessor.builder().regexes(regexes).build()); } - if (!getParentCommand().getSpel().isEmpty()) { - processors.add(getParentCommand().spelProcessor()); + if (!getSpel().isEmpty()) { + processors.add(spelProcessor()); processors.add(ObjectMapToStringMapProcessor.builder().build()); } - return getParentCommand().processor(processors); + return processor(processors); } } diff --git a/src/main/java/com/redislabs/riot/cli/FileOptions.java b/file/src/main/java/com/redislabs/riot/file/FileOptions.java similarity index 91% rename from src/main/java/com/redislabs/riot/cli/FileOptions.java rename to file/src/main/java/com/redislabs/riot/file/FileOptions.java index 2ab1ad9ca..5c45d3f50 100644 --- a/src/main/java/com/redislabs/riot/cli/FileOptions.java +++ b/file/src/main/java/com/redislabs/riot/file/FileOptions.java @@ -1,9 +1,5 @@ -package com.redislabs.riot.cli; +package com.redislabs.riot.file; -import com.redislabs.riot.cli.file.FileExtensions; -import com.redislabs.riot.cli.file.FileType; -import com.redislabs.riot.cli.file.S3ResourceBuilder; -import com.redislabs.riot.cli.file.UncustomizedUrlResource; import lombok.Getter; import org.springframework.batch.item.file.FlatFileItemReader; import org.springframework.batch.item.file.transform.DelimitedLineTokenizer; @@ -20,6 +16,12 @@ public class FileOptions { + public final static String CSV = "csv"; + public final static String TSV = "tsv"; + public final static String FW = "fw"; + public final static String JSON = "json"; + public final static String XML = "xml"; + private final static Pattern EXTENSION_PATTERN = Pattern.compile("(?i)\\.(?\\w+)(?\\.gz)?$"); private static final FileType DEFAULT_FILETYPE = FileType.DELIMITED; @@ -64,9 +66,9 @@ public String getDelimiter() { String extension = getExtension(); if (extension != null) { switch (extension) { - case FileExtensions.TSV: + case FileOptions.TSV: return DelimitedLineTokenizer.DELIMITER_TAB; - case FileExtensions.CSV: + case FileOptions.CSV: return DelimitedLineTokenizer.DELIMITER_COMMA; } } diff --git a/src/main/java/com/redislabs/riot/cli/file/FileType.java b/file/src/main/java/com/redislabs/riot/file/FileType.java similarity index 56% rename from src/main/java/com/redislabs/riot/cli/file/FileType.java rename to file/src/main/java/com/redislabs/riot/file/FileType.java index adbbecf13..4c78b464a 100644 --- a/src/main/java/com/redislabs/riot/cli/file/FileType.java +++ b/file/src/main/java/com/redislabs/riot/file/FileType.java @@ -1,8 +1,8 @@ -package com.redislabs.riot.cli.file; +package com.redislabs.riot.file; public enum FileType { - DELIMITED(FileExtensions.CSV, FileExtensions.TSV), FIXED(FileExtensions.FW), JSON(FileExtensions.JSON), XML(FileExtensions.XML); + DELIMITED(FileOptions.CSV, FileOptions.TSV), FIXED(FileOptions.FW), JSON(FileOptions.JSON), XML(FileOptions.XML); private final String[] extensions; diff --git a/src/main/java/com/redislabs/riot/cli/file/GZIPInputStreamResource.java b/file/src/main/java/com/redislabs/riot/file/GZIPInputStreamResource.java similarity index 93% rename from src/main/java/com/redislabs/riot/cli/file/GZIPInputStreamResource.java rename to file/src/main/java/com/redislabs/riot/file/GZIPInputStreamResource.java index 2f4f5789b..2f15de8db 100644 --- a/src/main/java/com/redislabs/riot/cli/file/GZIPInputStreamResource.java +++ b/file/src/main/java/com/redislabs/riot/file/GZIPInputStreamResource.java @@ -1,4 +1,4 @@ -package com.redislabs.riot.cli.file; +package com.redislabs.riot.file; import java.io.IOException; import java.io.InputStream; diff --git a/src/main/java/com/redislabs/riot/cli/file/GZIPOutputStreamResource.java b/file/src/main/java/com/redislabs/riot/file/GZIPOutputStreamResource.java similarity index 93% rename from src/main/java/com/redislabs/riot/cli/file/GZIPOutputStreamResource.java rename to file/src/main/java/com/redislabs/riot/file/GZIPOutputStreamResource.java index 308076406..ef5d0aa38 100644 --- a/src/main/java/com/redislabs/riot/cli/file/GZIPOutputStreamResource.java +++ b/file/src/main/java/com/redislabs/riot/file/GZIPOutputStreamResource.java @@ -1,4 +1,4 @@ -package com.redislabs.riot.cli.file; +package com.redislabs.riot.file; import org.springframework.batch.item.resource.OutputStreamResource; diff --git a/src/main/java/com/redislabs/riot/cli/file/HeaderCallback.java b/file/src/main/java/com/redislabs/riot/file/HeaderCallback.java similarity index 91% rename from src/main/java/com/redislabs/riot/cli/file/HeaderCallback.java rename to file/src/main/java/com/redislabs/riot/file/HeaderCallback.java index 088ca8c6c..d0f236761 100644 --- a/src/main/java/com/redislabs/riot/cli/file/HeaderCallback.java +++ b/file/src/main/java/com/redislabs/riot/file/HeaderCallback.java @@ -1,4 +1,4 @@ -package com.redislabs.riot.cli.file; +package com.redislabs.riot.file; import org.springframework.batch.item.file.FlatFileHeaderCallback; diff --git a/src/main/java/com/redislabs/riot/cli/file/MapFieldExtractor.java b/file/src/main/java/com/redislabs/riot/file/MapFieldExtractor.java similarity index 92% rename from src/main/java/com/redislabs/riot/cli/file/MapFieldExtractor.java rename to file/src/main/java/com/redislabs/riot/file/MapFieldExtractor.java index 5bb6b37a6..2215065fa 100644 --- a/src/main/java/com/redislabs/riot/cli/file/MapFieldExtractor.java +++ b/file/src/main/java/com/redislabs/riot/file/MapFieldExtractor.java @@ -1,4 +1,4 @@ -package com.redislabs.riot.cli.file; +package com.redislabs.riot.file; import lombok.Builder; import org.springframework.batch.item.file.transform.FieldExtractor; diff --git a/src/main/java/com/redislabs/riot/cli/file/MapFieldSetMapper.java b/file/src/main/java/com/redislabs/riot/file/MapFieldSetMapper.java similarity index 95% rename from src/main/java/com/redislabs/riot/cli/file/MapFieldSetMapper.java rename to file/src/main/java/com/redislabs/riot/file/MapFieldSetMapper.java index 6aa42adcb..ef0112e45 100644 --- a/src/main/java/com/redislabs/riot/cli/file/MapFieldSetMapper.java +++ b/file/src/main/java/com/redislabs/riot/file/MapFieldSetMapper.java @@ -1,4 +1,4 @@ -package com.redislabs.riot.cli.file; +package com.redislabs.riot.file; import org.springframework.batch.item.file.mapping.FieldSetMapper; import org.springframework.batch.item.file.transform.FieldSet; diff --git a/file/src/main/java/com/redislabs/riot/file/ProcessingItemWriter.java b/file/src/main/java/com/redislabs/riot/file/ProcessingItemWriter.java new file mode 100644 index 000000000..be7e02716 --- /dev/null +++ b/file/src/main/java/com/redislabs/riot/file/ProcessingItemWriter.java @@ -0,0 +1,30 @@ +package com.redislabs.riot.file; + +import org.springframework.batch.item.ItemProcessor; +import org.springframework.batch.item.ItemWriter; +import org.springframework.batch.item.support.AbstractItemStreamItemWriter; +import org.springframework.util.ClassUtils; + +import java.util.ArrayList; +import java.util.List; + +public class ProcessingItemWriter extends AbstractItemStreamItemWriter { + + private final ItemProcessor processor; + private final ItemWriter writer; + + public ProcessingItemWriter(ItemProcessor processor, ItemWriter writer) { + setName(ClassUtils.getShortName(getClass())); + this.processor = processor; + this.writer = writer; + } + + @Override + public void write(List items) throws Exception { + List targetItems = new ArrayList<>(items.size()); + for (I item : items) { + targetItems.add(processor.process(item)); + } + writer.write(targetItems); + } +} diff --git a/src/main/java/com/redislabs/riot/cli/file/RangeConverter.java b/file/src/main/java/com/redislabs/riot/file/RangeConverter.java similarity index 90% rename from src/main/java/com/redislabs/riot/cli/file/RangeConverter.java rename to file/src/main/java/com/redislabs/riot/file/RangeConverter.java index 7b4d0161b..0cb111f7b 100644 --- a/src/main/java/com/redislabs/riot/cli/file/RangeConverter.java +++ b/file/src/main/java/com/redislabs/riot/file/RangeConverter.java @@ -1,4 +1,4 @@ -package com.redislabs.riot.cli.file; +package com.redislabs.riot.file; import org.springframework.batch.item.file.transform.Range; diff --git a/src/main/java/com/redislabs/riot/cli/file/S3ResourceBuilder.java b/file/src/main/java/com/redislabs/riot/file/S3ResourceBuilder.java similarity index 95% rename from src/main/java/com/redislabs/riot/cli/file/S3ResourceBuilder.java rename to file/src/main/java/com/redislabs/riot/file/S3ResourceBuilder.java index 6464548bc..0ab6dc6bc 100644 --- a/src/main/java/com/redislabs/riot/cli/file/S3ResourceBuilder.java +++ b/file/src/main/java/com/redislabs/riot/file/S3ResourceBuilder.java @@ -1,4 +1,4 @@ -package com.redislabs.riot.cli.file; +package com.redislabs.riot.file; import java.net.URI; diff --git a/src/main/java/com/redislabs/riot/cli/file/SimpleAWSCredentialsProvider.java b/file/src/main/java/com/redislabs/riot/file/SimpleAWSCredentialsProvider.java similarity index 93% rename from src/main/java/com/redislabs/riot/cli/file/SimpleAWSCredentialsProvider.java rename to file/src/main/java/com/redislabs/riot/file/SimpleAWSCredentialsProvider.java index cd4788758..083187ae5 100644 --- a/src/main/java/com/redislabs/riot/cli/file/SimpleAWSCredentialsProvider.java +++ b/file/src/main/java/com/redislabs/riot/file/SimpleAWSCredentialsProvider.java @@ -1,4 +1,4 @@ -package com.redislabs.riot.cli.file; +package com.redislabs.riot.file; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; diff --git a/src/main/java/com/redislabs/riot/cli/file/UncustomizedUrlResource.java b/file/src/main/java/com/redislabs/riot/file/UncustomizedUrlResource.java similarity index 96% rename from src/main/java/com/redislabs/riot/cli/file/UncustomizedUrlResource.java rename to file/src/main/java/com/redislabs/riot/file/UncustomizedUrlResource.java index e9ac50f51..f9fa09c31 100644 --- a/src/main/java/com/redislabs/riot/cli/file/UncustomizedUrlResource.java +++ b/file/src/main/java/com/redislabs/riot/file/UncustomizedUrlResource.java @@ -1,4 +1,4 @@ -package com.redislabs.riot.cli.file; +package com.redislabs.riot.file; import java.io.IOException; import java.net.HttpURLConnection; diff --git a/src/test/java/com/redislabs/riot/TestFile.java b/file/src/test/java/com/redislabs/riot/file/TestRiotFile.java similarity index 83% rename from src/test/java/com/redislabs/riot/TestFile.java rename to file/src/test/java/com/redislabs/riot/file/TestRiotFile.java index 065eadd47..330bb0270 100644 --- a/src/test/java/com/redislabs/riot/TestFile.java +++ b/file/src/test/java/com/redislabs/riot/file/TestRiotFile.java @@ -1,4 +1,4 @@ -package com.redislabs.riot; +package com.redislabs.riot.file; import com.fasterxml.jackson.databind.ObjectMapper; import com.github.javafaker.Faker; @@ -10,7 +10,7 @@ import com.redislabs.lettusearch.index.field.TextField; import com.redislabs.lettusearch.search.Document; import com.redislabs.lettusearch.search.SearchResults; -import com.redislabs.riot.cli.file.MapFieldSetMapper; +import com.redislabs.riot.test.BaseTest; import io.lettuce.core.GeoArgs.Unit; import io.lettuce.core.ScoredValue; import org.junit.jupiter.api.Assertions; @@ -23,7 +23,7 @@ import org.springframework.batch.item.json.JacksonJsonObjectReader; import org.springframework.batch.item.json.JsonItemReader; import org.springframework.batch.item.json.builder.JsonItemReaderBuilder; -import org.springframework.batch.item.redis.KeyValue; +import org.springframework.batch.item.redis.support.KeyValue; import org.springframework.batch.item.support.AbstractItemCountingItemStreamItemReader; import org.springframework.core.io.FileSystemResource; import org.springframework.core.io.InputStreamResource; @@ -34,14 +34,26 @@ import java.util.*; import java.util.zip.GZIPInputStream; -public class TestFile extends BaseTest { +public class TestRiotFile extends BaseTest { + + private final static int COUNT = 2410; + + @Override + protected int execute(String[] args) throws Exception { + return new App().execute(args); + } + + @Override + protected String applicationName() { + return "riot-file"; + } @Test - public void testExportCsv() throws Exception { + public void exportCsv() throws Exception { File file = new File("/tmp/beers.csv"); file.delete(); - runFile("import-json-hash"); - runFile("export-csv"); + runFile("/import-json-hash.txt"); + runFile("/export-csv.txt"); String[] header = Files.readAllLines(file.toPath()).get(0).split("\\|"); FlatFileItemReaderBuilder> builder = new FlatFileItemReaderBuilder<>(); builder.name("flat-file-reader"); @@ -72,11 +84,11 @@ private List readAll(AbstractItemCountingItemStreamItemReader reader) @SuppressWarnings("rawtypes") @Test - public void testExportJson() throws Exception { + public void exportJson() throws Exception { File file = new File("/tmp/beers.json"); file.delete(); - runFile("import-json-hash"); - runFile("export-json"); + runFile("/import-json-hash.txt"); + runFile("/export-json.txt"); JsonItemReaderBuilder builder = new JsonItemReaderBuilder<>(); builder.name("json-file-reader"); builder.resource(new FileSystemResource(file)); @@ -89,7 +101,7 @@ public void testExportJson() throws Exception { } @Test - public void testExportDataStructuresToJson() throws Exception { + public void exportDataStructuresToJson() throws Exception { File file = new File("/tmp/datastructures.json"); file.delete(); Faker faker = new Faker(); @@ -114,7 +126,7 @@ public void testExportDataStructuresToJson() throws Exception { commands.xadd("stream:" + index, hash); commands.geoadd("geo:" + index, faker.number().randomDouble(5, -90, 90), faker.number().randomDouble(5, -80, 80), faker.rickAndMorty().character()); } - runCommand("export file /tmp/datastructures.json"); + runCommand("export /tmp/datastructures.json"); JsonItemReaderBuilder builder = new JsonItemReaderBuilder<>(); builder.name("datastructures-json-file-reader"); builder.resource(new FileSystemResource(file)); @@ -128,11 +140,11 @@ public void testExportDataStructuresToJson() throws Exception { @SuppressWarnings("rawtypes") @Test - public void testExportJsonGz() throws Exception { + public void exportJsonGz() throws Exception { File file = new File("/tmp/beers.json.gz"); file.delete(); - runFile("import-json-hash"); - runFile("export-json-gz"); + runFile("/import-json-hash.txt"); + runFile("/export-json-gz.txt"); JsonItemReaderBuilder builder = new JsonItemReaderBuilder<>(); builder.name("json-file-reader"); FileSystemResource resource = new FileSystemResource(file); @@ -146,14 +158,14 @@ public void testExportJsonGz() throws Exception { } @Test - public void testImportCsvHash() throws Exception { - runFile("import-csv-hash"); + public void importCsvHash() throws Exception { + runFile("/import-csv-hash.txt"); List keys = commands().keys("beer:*"); - Assertions.assertEquals(BEER_COUNT, keys.size()); + Assertions.assertEquals(COUNT, keys.size()); } @Test - public void testImportCsvSearch() throws Exception { + public void importCsvSearch() throws Exception { String FIELD_ABV = "abv"; String FIELD_NAME = "name"; String FIELD_STYLE = "style"; @@ -162,25 +174,25 @@ public void testImportCsvSearch() throws Exception { commands().flushall(); Schema schema = Schema.builder().field(TextField.builder().name(FIELD_NAME).sortable(true).build()).field(TextField.builder().name(FIELD_STYLE).matcher(PhoneticMatcher.English).sortable(true).build()).field(NumericField.builder().name(FIELD_ABV).sortable(true).build()).field(NumericField.builder().name(FIELD_OUNCES).sortable(true).build()).build(); commands().create(INDEX, schema, null); - runFile("import-csv-search"); + runFile("/import-csv-search.txt"); SearchResults results = commands().search(INDEX, "*"); - Assertions.assertEquals(BEER_COUNT, results.getCount()); + Assertions.assertEquals(COUNT, results.getCount()); } @Test - public void testImportCsvProcessorSearchGeo() throws Exception { + public void importCsvProcessorSearchGeo() throws Exception { String INDEX = "airports"; commands().flushall(); Schema schema = Schema.builder().field(TextField.builder().name("Name").sortable(true).build()).field(GeoField.builder().name("Location").sortable(true).build()).build(); commands().create(INDEX, schema, null); - runFile("import-csv-processor-search-geo"); + runFile("/import-csv-processor-search-geo.txt"); SearchResults results = commands().search(INDEX, "@Location:[-77 38 50 mi]"); Assertions.assertEquals(3, results.getCount()); } @Test - public void testImportCsvGeo() throws Exception { - runFile("import-csv-geo"); + public void importCsvGeo() throws Exception { + runFile("/import-csv-geo.txt"); Set results = commands().georadius("airportgeo", -122.4194, 37.7749, 20, Unit.mi); Assertions.assertTrue(results.contains("3469")); Assertions.assertTrue(results.contains("10360")); @@ -188,9 +200,9 @@ public void testImportCsvGeo() throws Exception { } @Test - public void testImportElasticJson() throws Exception { + public void importElasticJson() throws Exception { String url = getClass().getClassLoader().getResource("es_test-index.json").getFile(); - runFile("import-elastic-json", url); + runFile("/import-elastic-json.txt", url); Assertions.assertEquals(2, commands().keys("estest:*").size()); Map doc1 = commands().hgetall("estest:doc1"); Assertions.assertEquals("ruan", doc1.get("_source.name")); @@ -198,8 +210,8 @@ public void testImportElasticJson() throws Exception { } @Test - public void testImportJsonHash() throws Exception { - runFile("import-json-hash"); + public void importJsonHash() throws Exception { + runFile("/import-json-hash.txt"); List keys = commands().keys("beer:*"); Assertions.assertEquals(4432, keys.size()); Map beer1 = commands().hgetall("beer:1"); @@ -207,8 +219,8 @@ public void testImportJsonHash() throws Exception { } @Test - public void testImportCsvProcessorHashDateFormat() throws Exception { - runFile("import-csv-processor-hash-dateformat"); + public void importCsvProcessorHashDateFormat() throws Exception { + runFile("/import-csv-processor-hash-dateformat.txt"); List keys = commands().keys("event:*"); Assertions.assertEquals(568, keys.size()); Map event = commands().hgetall("event:248206"); @@ -219,11 +231,11 @@ public void testImportCsvProcessorHashDateFormat() throws Exception { } @Test - public void testImportCsvProcessorSearch() throws Exception { + public void importCsvProcessorSearch() throws Exception { String INDEX = "laevents"; Schema schema = Schema.builder().field(TextField.builder().name("Title").build()).field(NumericField.builder().name("lon").build()).field(NumericField.builder().name("kat").build()).field(GeoField.builder().name("location").sortable(true).build()).build(); commands().create(INDEX, schema, null); - runFile("import-csv-processor-search"); + runFile("/import-csv-processor-search.txt"); SearchResults results = commands().search(INDEX, "@location:[-118.446014 33.998415 10 mi]"); Assertions.assertTrue(results.getCount() > 0); for (Document result : results) { @@ -231,4 +243,5 @@ public void testImportCsvProcessorSearch() throws Exception { Assertions.assertTrue(lat > 33 && lat < 35); } } + } diff --git a/src/test/resources/es_test-index.json b/file/src/test/resources/es_test-index.json similarity index 100% rename from src/test/resources/es_test-index.json rename to file/src/test/resources/es_test-index.json diff --git a/file/src/test/resources/export-csv.txt b/file/src/test/resources/export-csv.txt new file mode 100644 index 000000000..e8f10dc90 --- /dev/null +++ b/file/src/test/resources/export-csv.txt @@ -0,0 +1 @@ +❯ riot-file export /tmp/beers.csv --match beer:* --key-regex "beer:(?.*)" --fields id name brewery_id abv --header --delimiter '|' \ No newline at end of file diff --git a/file/src/test/resources/export-json-gz.txt b/file/src/test/resources/export-json-gz.txt new file mode 100644 index 000000000..aef484328 --- /dev/null +++ b/file/src/test/resources/export-json-gz.txt @@ -0,0 +1 @@ +❯ riot-file export /tmp/beers.json.gz --match beer:* --fields id name brewery_id abv \ No newline at end of file diff --git a/file/src/test/resources/export-json.txt b/file/src/test/resources/export-json.txt new file mode 100644 index 000000000..9d39a411f --- /dev/null +++ b/file/src/test/resources/export-json.txt @@ -0,0 +1 @@ +❯ riot-file export /tmp/beers.json --match beer:* --key-regex "beer:(?.*)" --fields id name brewery_id abv \ No newline at end of file diff --git a/file/src/test/resources/import-csv-geo.txt b/file/src/test/resources/import-csv-geo.txt new file mode 100644 index 000000000..0babd9e07 --- /dev/null +++ b/file/src/test/resources/import-csv-geo.txt @@ -0,0 +1 @@ +❯ riot-file import https://raw.githubusercontent.com/jpatokal/openflights/master/data/airports.dat --command geoadd --keyspace airportgeo --members AirportID --lon Longitude --lat Latitude --fields AirportID Name City Country IATA ICAO Latitude Longitude Altitude Timezone DST Tz Type Source \ No newline at end of file diff --git a/file/src/test/resources/import-csv-hash.txt b/file/src/test/resources/import-csv-hash.txt new file mode 100644 index 000000000..893f42b17 --- /dev/null +++ b/file/src/test/resources/import-csv-hash.txt @@ -0,0 +1 @@ +❯ riot-file import https://raw.githubusercontent.com/nickhould/craft-beers-dataset/master/data/processed/beers.csv --keyspace beer --keys id --header \ No newline at end of file diff --git a/file/src/test/resources/import-csv-processor-hash-dateformat.txt b/file/src/test/resources/import-csv-processor-hash-dateformat.txt new file mode 100644 index 000000000..c0e2417f9 --- /dev/null +++ b/file/src/test/resources/import-csv-processor-hash-dateformat.txt @@ -0,0 +1 @@ +❯ riot-file import https://data.lacity.org/api/views/rx9t-fp7k/rows.csv?accessType=DOWNLOAD --date-format "MM/dd/yyyy HH:mm:ss a" --spel "EventStartDate=remove('Event Start Date')" "EpochStart=#date.parse(EventStartDate).getTime()" "index=#index" --keyspace event --keys Id --header \ No newline at end of file diff --git a/file/src/test/resources/import-csv-processor-search-geo.txt b/file/src/test/resources/import-csv-processor-search-geo.txt new file mode 100644 index 000000000..c69a058a0 --- /dev/null +++ b/file/src/test/resources/import-csv-processor-search-geo.txt @@ -0,0 +1 @@ +❯ riot-file import https://raw.githubusercontent.com/jpatokal/openflights/master/data/airports.dat --command ftadd --index airports -k AirportID --spel "Location=#geo(Longitude,Latitude)" --fields AirportID Name City Country IATA ICAO Latitude Longitude Altitude Timezone DST Tz Type Source \ No newline at end of file diff --git a/file/src/test/resources/import-csv-processor-search.txt b/file/src/test/resources/import-csv-processor-search.txt new file mode 100644 index 000000000..688fefd50 --- /dev/null +++ b/file/src/test/resources/import-csv-processor-search.txt @@ -0,0 +1 @@ +❯ riot-file import https://data.lacity.org/api/views/rx9t-fp7k/rows.csv?accessType=DOWNLOAD --spel location=#geo(lon,lat) --command ftadd --index laevents --keys Id --header --regex 'Event Location'="\((?.+),\s+(?.+)\)" \ No newline at end of file diff --git a/file/src/test/resources/import-csv-search.txt b/file/src/test/resources/import-csv-search.txt new file mode 100644 index 000000000..89bb69cc4 --- /dev/null +++ b/file/src/test/resources/import-csv-search.txt @@ -0,0 +1 @@ +❯ riot-file import https://raw.githubusercontent.com/nickhould/craft-beers-dataset/master/data/processed/beers.csv --command ftadd --index beers --keys id --header \ No newline at end of file diff --git a/file/src/test/resources/import-elastic-json.txt b/file/src/test/resources/import-elastic-json.txt new file mode 100644 index 000000000..1bcf685b4 --- /dev/null +++ b/file/src/test/resources/import-elastic-json.txt @@ -0,0 +1 @@ +❯ riot-file import %s --keyspace estest --keys _id \ No newline at end of file diff --git a/file/src/test/resources/import-json-hash.txt b/file/src/test/resources/import-json-hash.txt new file mode 100644 index 000000000..090ddf9d2 --- /dev/null +++ b/file/src/test/resources/import-json-hash.txt @@ -0,0 +1 @@ +❯ riot-file import https://raw.githubusercontent.com/rethinkdb/beerthink/master/data/beers.json --keyspace beer --keys id \ No newline at end of file diff --git a/file/src/test/resources/import-xml-hash.txt b/file/src/test/resources/import-xml-hash.txt new file mode 100644 index 000000000..023d7a447 --- /dev/null +++ b/file/src/test/resources/import-xml-hash.txt @@ -0,0 +1 @@ +❯ riot-file import https://raw.githubusercontent.com/Redislabs-Solution-Architects/riot/master/src/test/resources/releases.xml --keyspace release --keys id \ No newline at end of file diff --git a/src/test/resources/timestamp.json b/file/src/test/resources/timestamp.json similarity index 100% rename from src/test/resources/timestamp.json rename to file/src/test/resources/timestamp.json diff --git a/gen/build.gradle b/gen/build.gradle new file mode 100644 index 000000000..05c14c6c9 --- /dev/null +++ b/gen/build.gradle @@ -0,0 +1,35 @@ +plugins { + id 'application' +} + +archivesBaseName = 'riot-gen' + +bootJar { + enabled = false +} + +jar { + enabled = true +} + +test { + useJUnitPlatform() +} + +dependencies { + implementation project(':core') + implementation 'org.slf4j:slf4j-jdk14' + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' + implementation 'org.ruaux:spring-batch-faker:1.0.1' + testImplementation project(':test') +} + +configurations { + all*.exclude module: 'spring-boot-starter-logging' +} + +application { + mainClassName = 'com.redislabs.riot.gen.App' + applicationName = 'riot-gen' +} \ No newline at end of file diff --git a/gen/src/main/java/com/redislabs/riot/gen/App.java b/gen/src/main/java/com/redislabs/riot/gen/App.java new file mode 100644 index 000000000..df4001b6f --- /dev/null +++ b/gen/src/main/java/com/redislabs/riot/gen/App.java @@ -0,0 +1,13 @@ +package com.redislabs.riot.gen; + +import com.redislabs.riot.RiotApp; +import picocli.CommandLine; + +@CommandLine.Command(name = "riot-gen", subcommands = {GenerateCommand.class, FakerHelpCommand.class}) +public class App extends RiotApp { + + public static void main(String[] args) { + System.exit(new App().execute(args)); + } + +} diff --git a/src/main/java/com/redislabs/riot/cli/FakerHelpCommand.java b/gen/src/main/java/com/redislabs/riot/gen/FakerHelpCommand.java similarity index 98% rename from src/main/java/com/redislabs/riot/cli/FakerHelpCommand.java rename to gen/src/main/java/com/redislabs/riot/gen/FakerHelpCommand.java index 658560b7f..c0171239b 100644 --- a/src/main/java/com/redislabs/riot/cli/FakerHelpCommand.java +++ b/gen/src/main/java/com/redislabs/riot/gen/FakerHelpCommand.java @@ -1,4 +1,4 @@ -package com.redislabs.riot.cli; +package com.redislabs.riot.gen; import com.github.javafaker.Faker; import picocli.CommandLine; diff --git a/src/main/java/com/redislabs/riot/cli/GenerateCommand.java b/gen/src/main/java/com/redislabs/riot/gen/GenerateCommand.java similarity index 81% rename from src/main/java/com/redislabs/riot/cli/GenerateCommand.java rename to gen/src/main/java/com/redislabs/riot/gen/GenerateCommand.java index e023674bf..4c898b2e3 100644 --- a/src/main/java/com/redislabs/riot/cli/GenerateCommand.java +++ b/gen/src/main/java/com/redislabs/riot/gen/GenerateCommand.java @@ -1,24 +1,25 @@ -package com.redislabs.riot.cli; +package com.redislabs.riot.gen; -import com.redislabs.lettuce.helper.RedisOptions; -import com.redislabs.lettusearch.RediSearchClient; import com.redislabs.lettusearch.RediSearchCommands; import com.redislabs.lettusearch.RediSearchUtils; +import com.redislabs.lettusearch.StatefulRediSearchConnection; import com.redislabs.lettusearch.index.IndexInfo; import com.redislabs.lettusearch.index.field.Field; import com.redislabs.lettusearch.index.field.GeoField; import com.redislabs.lettusearch.index.field.TagField; import com.redislabs.lettusearch.index.field.TextField; +import com.redislabs.riot.AbstractImportCommand; import lombok.extern.slf4j.Slf4j; import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemReader; import org.springframework.batch.item.faker.FakerItemReader; +import org.springframework.batch.item.redisearch.support.RediSearchConnectionBuilder; import picocli.CommandLine; import java.util.*; @Slf4j -@CommandLine.Command(name = "gen", description = "Import generated data", subcommands = FakerHelpCommand.class) +@CommandLine.Command(name = "import", description = "Import generated data") public class GenerateCommand extends AbstractImportCommand> { @CommandLine.Parameters(description = "SpEL expressions", paramLabel = "SPEL") @@ -64,9 +65,10 @@ private Map fakerFields() { if (fakerIndex == null) { return fields; } - RedisOptions redisOptions = redisOptions(); - RediSearchClient rediSearchClient = redisOptions.getClientResources() == null ? RediSearchClient.create(redisOptions.getRedisURI()) : RediSearchClient.create(redisOptions.getClientResources(), redisOptions.getRedisURI()); - RediSearchCommands commands = rediSearchClient.connect().sync(); + RediSearchConnectionBuilder connectionBuilder = new RediSearchConnectionBuilder<>(); + configure(connectionBuilder); + StatefulRediSearchConnection connection = connectionBuilder.connection(); + RediSearchCommands commands = connection.sync(); IndexInfo info = RediSearchUtils.getInfo(commands.ftInfo(fakerIndex)); for (Field field : info.getFields()) { fields.put(field.getName(), expression(field)); @@ -77,6 +79,6 @@ private Map fakerFields() { @Override protected ItemProcessor, Object> processor() { - return getParentCommand().objectMapProcessor(); + return objectMapProcessor(); } } diff --git a/src/test/java/com/redislabs/riot/TestGenerator.java b/gen/src/test/java/com/redislabs/riot/gen/TestGen.java similarity index 84% rename from src/test/java/com/redislabs/riot/TestGenerator.java rename to gen/src/test/java/com/redislabs/riot/gen/TestGen.java index f7b8cb819..2b1038123 100644 --- a/src/test/java/com/redislabs/riot/TestGenerator.java +++ b/gen/src/test/java/com/redislabs/riot/gen/TestGen.java @@ -1,4 +1,4 @@ -package com.redislabs.riot; +package com.redislabs.riot.gen; import com.redislabs.lettusearch.index.Schema; import com.redislabs.lettusearch.index.field.NumericField; @@ -6,6 +6,7 @@ import com.redislabs.lettusearch.index.field.TagField; import com.redislabs.lettusearch.index.field.TextField; import com.redislabs.lettusearch.search.SearchResults; +import com.redislabs.riot.test.BaseTest; import io.lettuce.core.Range; import io.lettuce.core.StreamMessage; import org.junit.jupiter.api.Assertions; @@ -15,11 +16,21 @@ import java.util.Map; import java.util.Set; -public class TestGenerator extends BaseTest { +public class TestGen extends BaseTest { + + @Override + protected int execute(String[] args) throws Exception { + return new App().execute(args); + } + + @Override + protected String applicationName() { + return "riot-gen"; + } @Test public void genFakerHash() throws Exception { - runFile("gen-faker-hash"); + runFile("/gen-faker-hash.txt"); List keys = commands().keys("person:*"); Assertions.assertEquals(100, keys.size()); Map person = commands().hgetall(keys.get(0)); @@ -29,7 +40,7 @@ public void genFakerHash() throws Exception { } public void genFakerScriptProcessorHash() throws Exception { - runFile("gen-faker-script-processor-hash"); + runFile("/gen-faker-script-processor-hash.txt"); List keys = commands().keys("person:*"); Assertions.assertEquals(100, keys.size()); Map person = commands().hgetall(keys.get(0)); @@ -41,7 +52,7 @@ public void genFakerScriptProcessorHash() throws Exception { @Test public void genFakerSet() throws Exception { - runFile("gen-faker-set"); + runFile("/gen-faker-set.txt"); Set names = commands().smembers("got:characters"); Assertions.assertTrue(names.size() > 10); Assertions.assertTrue(names.contains("Lysa Meadows")); @@ -49,7 +60,7 @@ public void genFakerSet() throws Exception { @Test public void genFakerZset() throws Exception { - runFile("gen-faker-zset"); + runFile("/gen-faker-zset.txt"); List keys = commands().keys("leases:*"); Assertions.assertTrue(keys.size() > 100); String key = keys.get(0); @@ -58,7 +69,7 @@ public void genFakerZset() throws Exception { @Test public void genFakerStream() throws Exception { - runFile("gen-faker-stream"); + runFile("/gen-faker-stream.txt"); List> messages = commands().xrange("teststream:1", Range.unbounded()); Assertions.assertTrue(messages.size() > 0); } @@ -78,7 +89,7 @@ public void genFakerIndexIntrospection() throws Exception { .field(NumericField.builder().name(FIELD_ABV).sortable(true).build()) .field(NumericField.builder().name(FIELD_OUNCES).sortable(true).build()).build(); commands().create(INDEX, schema, null); - runFile("gen-faker-index-introspection"); + runFile("/gen-faker-index-introspection.txt"); SearchResults results = commands().search(INDEX, "*"); Assertions.assertEquals(100, results.getCount()); } diff --git a/gen/src/test/resources/gen-faker-hash.txt b/gen/src/test/resources/gen-faker-hash.txt new file mode 100644 index 000000000..dee037540 --- /dev/null +++ b/gen/src/test/resources/gen-faker-hash.txt @@ -0,0 +1 @@ +❯ riot-gen import id=#index firstName=name.firstName lastName=name.lastName address=address.fullAddress --max 100 --keyspace person --keys id \ No newline at end of file diff --git a/gen/src/test/resources/gen-faker-index-introspection.txt b/gen/src/test/resources/gen-faker-index-introspection.txt new file mode 100644 index 000000000..35b03f799 --- /dev/null +++ b/gen/src/test/resources/gen-faker-index-introspection.txt @@ -0,0 +1 @@ +❯ riot-gen import --faker-index beerIntrospection --max 100 --command ftadd --index beerIntrospection --keys id \ No newline at end of file diff --git a/gen/src/test/resources/gen-faker-script-processor-hash.txt b/gen/src/test/resources/gen-faker-script-processor-hash.txt new file mode 100644 index 000000000..4ec495a60 --- /dev/null +++ b/gen/src/test/resources/gen-faker-script-processor-hash.txt @@ -0,0 +1 @@ +❯ riot-gen import id=#index firstName=name.firstName lastName=name.lastName address=address.fullAddress --script "function process(item) { item.address = item.address.toUpperCase(); return item; } process(item);" --max 100 --keyspace person --keys id \ No newline at end of file diff --git a/gen/src/test/resources/gen-faker-set.txt b/gen/src/test/resources/gen-faker-set.txt new file mode 100644 index 000000000..4e44d981d --- /dev/null +++ b/gen/src/test/resources/gen-faker-set.txt @@ -0,0 +1 @@ +❯ riot-gen import name=gameOfThrones.character --max 10000 --command sadd --members name --keyspace got:characters \ No newline at end of file diff --git a/gen/src/test/resources/gen-faker-stream.txt b/gen/src/test/resources/gen-faker-stream.txt new file mode 100644 index 000000000..ce08310f1 --- /dev/null +++ b/gen/src/test/resources/gen-faker-stream.txt @@ -0,0 +1 @@ +❯ riot-gen import id=#index category=number.randomDigit --batch 50 --max 1000 --command xadd --keyspace teststream --keys category \ No newline at end of file diff --git a/gen/src/test/resources/gen-faker-zset.txt b/gen/src/test/resources/gen-faker-zset.txt new file mode 100644 index 000000000..44f89758a --- /dev/null +++ b/gen/src/test/resources/gen-faker-zset.txt @@ -0,0 +1 @@ +❯ riot-gen import ip=number.digits(4) lease=number.digits(2) time=number.digits(5) --batch 50 --max 10000 --command zadd --keyspace leases --keys ip --members lease --score=time \ No newline at end of file diff --git a/gen/src/test/resources/gen-faker.txt b/gen/src/test/resources/gen-faker.txt new file mode 100644 index 000000000..03e61ced2 --- /dev/null +++ b/gen/src/test/resources/gen-faker.txt @@ -0,0 +1 @@ +❯ riot-gen -r redis://localhost:6379 import index=#index firstName=name.firstName lastName=name.lastName address=address.fullAddress --max 1000 --keyspace test --keys index \ No newline at end of file diff --git a/gradle.properties b/gradle.properties index 9041a2693..e997a9af9 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1 +1 @@ -version=1.11.2 \ No newline at end of file +version=2.0.0 \ No newline at end of file diff --git a/kafka/build.gradle b/kafka/build.gradle new file mode 100644 index 000000000..95554d170 --- /dev/null +++ b/kafka/build.gradle @@ -0,0 +1,27 @@ +plugins { + id 'application' + id 'java' + id 'java-library' + id 'org.springframework.boot' version '2.3.1.RELEASE' + id 'io.spring.dependency-management' version '1.0.9.RELEASE' +} + +bootJar { + enabled = false +} + +jar { + enabled = true +} + +test { + useJUnitPlatform() +} + +dependencies { + implementation 'org.springframework.kafka:spring-kafka' +} + +application { + mainClassName = 'com.redislabs.riot-file.kafka.App' +} \ No newline at end of file diff --git a/mongo/build.gradle b/mongo/build.gradle new file mode 100644 index 000000000..5e447516e --- /dev/null +++ b/mongo/build.gradle @@ -0,0 +1,27 @@ +plugins { + id 'application' + id 'java' + id 'java-library' + id 'org.springframework.boot' version '2.3.1.RELEASE' + id 'io.spring.dependency-management' version '1.0.9.RELEASE' +} + +bootJar { + enabled = false +} + +jar { + enabled = true +} + +test { + useJUnitPlatform() +} + +dependencies { + implementation 'org.springframework.boot:spring-boot-starter-data-mongodb' +} + +application { + mainClassName = 'com.redislabs.riot-file.mongodb.App' +} \ No newline at end of file diff --git a/redis/build.gradle b/redis/build.gradle new file mode 100644 index 000000000..7f03c3344 --- /dev/null +++ b/redis/build.gradle @@ -0,0 +1,35 @@ +plugins { + id 'application' +} + +archivesBaseName = 'riot-redis' + +bootJar { + enabled = false +} + +jar { + enabled = true +} + +test { + useJUnitPlatform() +} + +dependencies { + implementation project(':core') + implementation 'org.slf4j:slf4j-jdk14' + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' + implementation 'org.latencyutils:LatencyUtils:2.0.3' + testImplementation project(':test') +} + +configurations { + all*.exclude module: 'spring-boot-starter-logging' +} + +application { + mainClassName = 'com.redislabs.riot.redis.App' + applicationName = 'riot-redis' +} \ No newline at end of file diff --git a/redis/src/main/java/com/redislabs/riot/redis/AbstractRedisCommand.java b/redis/src/main/java/com/redislabs/riot/redis/AbstractRedisCommand.java new file mode 100644 index 000000000..099a44d8f --- /dev/null +++ b/redis/src/main/java/com/redislabs/riot/redis/AbstractRedisCommand.java @@ -0,0 +1,28 @@ +package com.redislabs.riot.redis; + +import com.redislabs.picocliredis.HelpCommand; +import com.redislabs.riot.RiotApp; +import io.lettuce.core.api.StatefulConnection; +import io.lettuce.core.api.sync.BaseRedisCommands; +import lombok.extern.slf4j.Slf4j; +import picocli.CommandLine; +import picocli.CommandLine.Command; + +@Slf4j +@Command +public abstract class AbstractRedisCommand extends HelpCommand { + + @CommandLine.ParentCommand + private RiotApp app; + + @Override + @SuppressWarnings("unchecked") + public void run() { + StatefulConnection connection = app.connection(); + BaseRedisCommands commands = app.sync(connection); + execute(commands); + } + + protected abstract void execute(BaseRedisCommands commands); + +} diff --git a/redis/src/main/java/com/redislabs/riot/redis/App.java b/redis/src/main/java/com/redislabs/riot/redis/App.java new file mode 100644 index 000000000..78f085bdb --- /dev/null +++ b/redis/src/main/java/com/redislabs/riot/redis/App.java @@ -0,0 +1,12 @@ +package com.redislabs.riot.redis; + +import com.redislabs.riot.RiotApp; +import picocli.CommandLine; + +@CommandLine.Command(name = "riot-redis", subcommands = {ReplicateCommand.class, InfoCommand.class, LatencyCommand.class, PingCommand.class}) +public class App extends RiotApp { + + public static void main(String[] args) { + System.exit(new App().execute(args)); + } +} diff --git a/redis/src/main/java/com/redislabs/riot/redis/InfoCommand.java b/redis/src/main/java/com/redislabs/riot/redis/InfoCommand.java new file mode 100644 index 000000000..218a4959a --- /dev/null +++ b/redis/src/main/java/com/redislabs/riot/redis/InfoCommand.java @@ -0,0 +1,16 @@ +package com.redislabs.riot.redis; + +import io.lettuce.core.api.sync.BaseRedisCommands; +import io.lettuce.core.api.sync.RedisServerCommands; +import lombok.extern.slf4j.Slf4j; +import picocli.CommandLine; + +@Slf4j +@CommandLine.Command(name = "info") +public class InfoCommand extends AbstractRedisCommand { + + @Override + protected void execute(BaseRedisCommands commands) { + log.info(((RedisServerCommands) commands).info()); + } +} diff --git a/redis/src/main/java/com/redislabs/riot/redis/LatencyCommand.java b/redis/src/main/java/com/redislabs/riot/redis/LatencyCommand.java new file mode 100644 index 000000000..eb898f54c --- /dev/null +++ b/redis/src/main/java/com/redislabs/riot/redis/LatencyCommand.java @@ -0,0 +1,53 @@ +package com.redislabs.riot.redis; + +import io.lettuce.core.api.sync.BaseRedisCommands; +import io.lettuce.core.metrics.CommandMetrics; +import io.lettuce.core.metrics.DefaultCommandLatencyCollectorOptions; +import org.HdrHistogram.Histogram; +import org.LatencyUtils.LatencyStats; +import picocli.CommandLine; + +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.TimeUnit; + +@CommandLine.Command(name="latency") +public class LatencyCommand extends AbstractRedisCommand { + + @CommandLine.Option(names = "--iterations", description = "Number of latency tests (default: ${DEFAULT-VALUE})", paramLabel = "") + private int iterations = 1000; + @CommandLine.Option(names = "--sleep", description = "Sleep duration between calls (default: ${DEFAULT-VALUE})", paramLabel = "") + private long sleep = 1; + @CommandLine.Option(names = "--unit", description = "Latency unit (default: ${DEFAULT-VALUE})", paramLabel = "") + private TimeUnit unit = TimeUnit.MILLISECONDS; + @CommandLine.Option(names = "--show-distribution", description = "Show latency distribution") + private boolean showDistribution = false; + + + @Override + protected void execute(BaseRedisCommands commands) { + LatencyStats stats = new LatencyStats(); + for (int index = 0; index < iterations; index++) { + long startTime = System.nanoTime(); + commands.ping(); + stats.recordLatency(System.nanoTime() - startTime); + try { + Thread.sleep(sleep); + } catch (InterruptedException e) { + return; + } + } + Histogram histogram = stats.getIntervalHistogram(); + if (showDistribution) { + histogram.outputPercentileDistribution(System.out, 1000000.0); + } else { + DefaultCommandLatencyCollectorOptions options = DefaultCommandLatencyCollectorOptions.create(); + Map percentiles = new TreeMap<>(); + for (double targetPercentile : options.targetPercentiles()) { + percentiles.put(targetPercentile, unit.convert(histogram.getValueAtPercentile(targetPercentile), TimeUnit.NANOSECONDS)); + } + CommandMetrics.CommandLatency latency = new CommandMetrics.CommandLatency(unit.convert(histogram.getMinValue(), TimeUnit.NANOSECONDS), unit.convert(histogram.getMaxValue(), TimeUnit.NANOSECONDS), percentiles); + System.out.println(latency.toString()); + } + } +} diff --git a/redis/src/main/java/com/redislabs/riot/redis/PingCommand.java b/redis/src/main/java/com/redislabs/riot/redis/PingCommand.java new file mode 100644 index 000000000..6a2947f7d --- /dev/null +++ b/redis/src/main/java/com/redislabs/riot/redis/PingCommand.java @@ -0,0 +1,16 @@ +package com.redislabs.riot.redis; + +import io.lettuce.core.api.sync.BaseRedisCommands; +import lombok.extern.slf4j.Slf4j; +import picocli.CommandLine; + +@Slf4j +@CommandLine.Command(name = "ping") +public class PingCommand extends AbstractRedisCommand { + + @Override + protected void execute(BaseRedisCommands commands) { + log.info("Received ping reply: {}", commands.ping()); + } + +} diff --git a/redis/src/main/java/com/redislabs/riot/redis/ReplicateCommand.java b/redis/src/main/java/com/redislabs/riot/redis/ReplicateCommand.java new file mode 100644 index 000000000..3ace238f4 --- /dev/null +++ b/redis/src/main/java/com/redislabs/riot/redis/ReplicateCommand.java @@ -0,0 +1,58 @@ +package com.redislabs.riot.redis; + +import com.redislabs.riot.AbstractTransferCommand; +import com.redislabs.riot.RedisConnectionOptions; +import com.redislabs.riot.RedisExportOptions; +import com.redislabs.riot.Transfer; +import org.springframework.batch.item.ItemProcessor; +import org.springframework.batch.item.ItemReader; +import org.springframework.batch.item.ItemWriter; +import org.springframework.batch.item.redis.RedisKeyDumpItemReader; +import org.springframework.batch.item.redis.RedisKeyDumpItemWriter; +import org.springframework.batch.item.redis.support.KeyDump; +import org.springframework.batch.item.support.PassThroughItemProcessor; +import picocli.CommandLine; +import picocli.CommandLine.Command; + +@Command(name = "replicate", description = "Replicate a Redis database to another Redis database") +public class ReplicateCommand extends AbstractTransferCommand, KeyDump> { + + @CommandLine.ArgGroup(exclusive = false, heading = "Target Redis connection options%n") + private RedisConnectionOptions targetRedis = new RedisConnectionOptions(); + @CommandLine.Mixin + private RedisExportOptions options = new RedisExportOptions(); + @CommandLine.Option(names = "--flush-interval", description = "Duration between notification flushes (default: ${DEFAULT-VALUE})", paramLabel = "") + private long flushPeriod = 50; + @CommandLine.Option(names = "--live", description = "Live replication") + private boolean live; + + @Override + protected ItemReader> reader() { + return configure(RedisKeyDumpItemReader.builder().scanCount(options.getScanCount()).scanMatch(options.getScanMatch()).batch(options.getBatchSize()).threads(options.getThreads()).queueCapacity(options.getQueueCapacity()).live(live)).build(); + } + + @Override + protected ItemProcessor, KeyDump> processor() { + return new PassThroughItemProcessor<>(); + } + + @Override + protected ItemWriter> writer() { + return configure(RedisKeyDumpItemWriter.builder().replace(true), targetRedis).build(); + } + + @Override + protected Transfer, KeyDump> transfer(ItemReader> reader, ItemProcessor, KeyDump> processor, ItemWriter> writer) { + Transfer, KeyDump> transfer = super.transfer(reader, processor, writer); + if (live) { + transfer.setFlushPeriod(flushPeriod); + } + return transfer; + } + + @Override + protected String taskName() { + return "Replicating"; + } + +} diff --git a/src/test/java/com/redislabs/riot/TestReplicate.java b/redis/src/test/java/com/redis/riot/redis/TestReplicate.java similarity index 63% rename from src/test/java/com/redislabs/riot/TestReplicate.java rename to redis/src/test/java/com/redis/riot/redis/TestReplicate.java index d95079633..c93c97bf1 100644 --- a/src/test/java/com/redislabs/riot/TestReplicate.java +++ b/redis/src/test/java/com/redis/riot/redis/TestReplicate.java @@ -1,31 +1,45 @@ -package com.redislabs.riot; +package com.redis.riot.redis; +import com.redislabs.riot.redis.App; +import com.redislabs.riot.test.BaseTest; +import com.redislabs.riot.test.DataPopulator; import io.lettuce.core.RedisClient; import io.lettuce.core.RedisURI; import io.lettuce.core.api.sync.RedisCommands; -import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import redis.embedded.RedisServer; -@Slf4j public class TestReplicate extends BaseTest { + private final static Logger log = LoggerFactory.getLogger(TestReplicate.class); + private final static String TARGET_HOST = "localhost"; private final static int TARGET_PORT = 6380; + @Override + protected int execute(String[] args) { + return new App().execute(args); + } + + @Override + protected String applicationName() { + return "riot-redis"; + } + @Test - public void testReplicate() throws Exception { + public void testReplicate() { RedisServer target = serverBuilder(TARGET_PORT).build(); try { target.start(); - runFile("gen-faker"); + DataPopulator.builder().connection(connection).build().run(); Long sourceSize = commands().dbsize(); Assertions.assertTrue(sourceSize > 0); - runFile("replicate"); + runFile("/replicate.txt"); RedisClient targetClient = RedisClient.create(RedisURI.create(TARGET_HOST, TARGET_PORT)); - Long targetSize = targetClient.connect().sync().dbsize(); - Assertions.assertEquals(sourceSize, targetSize); + Assertions.assertEquals(sourceSize, targetClient.connect().sync().dbsize()); } finally { target.stop(); } @@ -36,17 +50,17 @@ public void testReplicateLive() throws Exception { RedisServer target = serverBuilder(TARGET_PORT).build(); try { target.start(); - runFile("gen-faker"); - Thread replicateThread = new Thread(() -> runFile("replicate-live")); + DataPopulator.builder().connection(connection).build().run(); + Thread replicateThread = new Thread(() -> runFile("/replicate-live.txt")); replicateThread.start(); Thread.sleep(500); RedisCommands commands = commands(); int count = 39; for (int index = 0; index < count; index++) { - commands.set("string:" + index, "value" + index); + commands.set("livestring:" + index, "value" + index); Thread.sleep(1); } - Thread.sleep(1000); + Thread.sleep(300); log.info("Interrupting"); replicateThread.interrupt(); RedisClient targetClient = RedisClient.create(RedisURI.create(TARGET_HOST, TARGET_PORT)); diff --git a/redis/src/test/resources/replicate-live.txt b/redis/src/test/resources/replicate-live.txt new file mode 100644 index 000000000..12fa1c424 --- /dev/null +++ b/redis/src/test/resources/replicate-live.txt @@ -0,0 +1 @@ +❯ riot-redis -r redis://localhost:6379 replicate -r redis://localhost:6380 --live \ No newline at end of file diff --git a/redis/src/test/resources/replicate.txt b/redis/src/test/resources/replicate.txt new file mode 100644 index 000000000..f864907a7 --- /dev/null +++ b/redis/src/test/resources/replicate.txt @@ -0,0 +1 @@ +❯ riot-redis -r redis://localhost:6379 replicate -r redis://localhost:6380 --batch 1 \ No newline at end of file diff --git a/riot b/riot-db similarity index 56% rename from riot rename to riot-db index 04f5e85c3..dfc537b59 100755 --- a/riot +++ b/riot-db @@ -2,4 +2,4 @@ ./gradlew -q --console plain installDist -./build/install/riot/bin/riot "$@" \ No newline at end of file +./db/build/install/riot-db/bin/riot-db "$@" \ No newline at end of file diff --git a/riot-file b/riot-file new file mode 100755 index 000000000..025affaed --- /dev/null +++ b/riot-file @@ -0,0 +1,5 @@ +#!/bin/sh -e + +./gradlew -q --console plain installDist + +./file/build/install/riot-file/bin/riot-file "$@" \ No newline at end of file diff --git a/riot-gen b/riot-gen new file mode 100755 index 000000000..91846c3b6 --- /dev/null +++ b/riot-gen @@ -0,0 +1,5 @@ +#!/bin/sh -e + +./gradlew -q --console plain installDist + +./gen/build/install/riot-gen/bin/riot-gen "$@" \ No newline at end of file diff --git a/riot-redis b/riot-redis new file mode 100755 index 000000000..37fad081c --- /dev/null +++ b/riot-redis @@ -0,0 +1,5 @@ +#!/bin/sh -e + +./gradlew -q --console plain installDist + +./redis/build/install/riot-redis/bin/riot-redis "$@" \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index 5ba065a63..574328d2e 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,5 +1,2 @@ -/* - * This file was generated by the Gradle 'init' task. - */ - rootProject.name = 'riot' +include 'core', 'db', 'file', 'gen', 'redis', 'test' \ No newline at end of file diff --git a/src/docs/asciidoc/index.adoc b/src/docs/asciidoc/index.adoc index b7d838efe..675e46c1b 100644 --- a/src/docs/asciidoc/index.adoc +++ b/src/docs/asciidoc/index.adoc @@ -294,13 +294,13 @@ riot --metrics … ==== Strings [source,shell] ---- -$ riot -s redis-12001.redislabs.com:12001 --max-total 96 gen --batch 5000 --threads 96 --max 100000000 --command set --keyspace hash --keys index --value index +❯ riot -s redis-12001.redislabs.com:12001 --max-total 96 gen --batch 5000 --threads 96 --max 100000000 --command set --keyspace hash --keys index --value index ---- image::images/rs-strings.png[] ==== Streams [source,shell] ---- -$ riot -s redis-12001.internal.jrx.demo.redislabs.com:12001 --pool-max-total 96 gen --batch 5000 --threads 96 --max 100000000 --command xadd --keyspace stream --keys partition +❯ riot -s redis-12001.internal.jrx.demo.redislabs.com:12001 --pool-max-total 96 gen --batch 5000 --threads 96 --max 100000000 --command xadd --keyspace stream --keys partition ---- image::images/rs-streams.png[] \ No newline at end of file diff --git a/src/main/java/com/redislabs/riot/Riot.java b/src/main/java/com/redislabs/riot/Riot.java deleted file mode 100644 index ba01f8394..000000000 --- a/src/main/java/com/redislabs/riot/Riot.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.redislabs.riot; - -import com.redislabs.lettuce.helper.PoolOptions; -import com.redislabs.lettuce.helper.RedisOptions; -import com.redislabs.picocliredis.Application; -import com.redislabs.picocliredis.RedisCommandLineOptions; -import com.redislabs.riot.cli.ExportCommand; -import com.redislabs.riot.cli.ImportCommand; -import com.redislabs.riot.cli.ReplicateCommand; -import com.redislabs.riot.cli.TestCommand; -import com.redislabs.riot.cli.file.RangeConverter; -import org.springframework.batch.item.file.transform.Range; -import picocli.CommandLine; -import picocli.CommandLine.Command; - -@Command(name = "riot", subcommands = {ImportCommand.class, ExportCommand.class, ReplicateCommand.class, TestCommand.class}) -public class Riot extends Application { - - @CommandLine.Mixin - private RedisCommandLineOptions redisCommandLineOptions = RedisCommandLineOptions.builder().build(); - - public static void main(String[] args) { - System.exit(new Riot().execute(args)); - } - - @Override - protected void registerConverters(CommandLine commandLine) { - commandLine.registerConverter(Range.class, new RangeConverter()); - super.registerConverters(commandLine); - } - - public RedisOptions redisOptions() { - return redisOptions(redisCommandLineOptions); - } - - public static RedisOptions redisOptions(RedisCommandLineOptions options) { - RedisOptions.RedisOptionsBuilder builder = RedisOptions.builder().redisURI(options.getRedisURI()).clientOptions(options.clientOptions()).cluster(options.isCluster()); - if (options.isShowMetrics()) { - builder.clientResources(options.clientResources()); - } - return builder.build(); - } - - -} diff --git a/src/main/java/com/redislabs/riot/TransferOptions.java b/src/main/java/com/redislabs/riot/TransferOptions.java deleted file mode 100644 index 1cc8b24f5..000000000 --- a/src/main/java/com/redislabs/riot/TransferOptions.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.redislabs.riot; - -import lombok.Builder; -import lombok.Data; - -@Data -@Builder -public class TransferOptions { - - public final static int DEFAULT_THREAD_COUNT = 1; - public final static int DEFAULT_BATCH_SIZE = 50; - - @Builder.Default - private int threadCount = DEFAULT_THREAD_COUNT; - @Builder.Default - private int batchSize = DEFAULT_BATCH_SIZE; - private Long flushPeriod; - private Integer maxItemCount; -} diff --git a/src/main/java/com/redislabs/riot/cli/AbstractExportCommand.java b/src/main/java/com/redislabs/riot/cli/AbstractExportCommand.java deleted file mode 100644 index e03ae3014..000000000 --- a/src/main/java/com/redislabs/riot/cli/AbstractExportCommand.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.redislabs.riot.cli; - -import org.springframework.batch.item.ItemReader; -import org.springframework.batch.item.redis.KeyValue; - -public abstract class AbstractExportCommand extends AbstractTransferCommand, O> { - - protected ExportCommand getParentCommand() { - return (ExportCommand) parentCommand; - } - - @Override - protected ItemReader> reader() { - return getParentCommand().reader(); - } -} diff --git a/src/main/java/com/redislabs/riot/cli/AbstractImportCommand.java b/src/main/java/com/redislabs/riot/cli/AbstractImportCommand.java deleted file mode 100644 index 8a6699371..000000000 --- a/src/main/java/com/redislabs/riot/cli/AbstractImportCommand.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.redislabs.riot.cli; - -import org.springframework.batch.item.ItemWriter; - -public abstract class AbstractImportCommand extends AbstractTransferCommand { - - @Override - protected ItemWriter writer() { - return getParentCommand().writer(); - } - - protected ImportCommand getParentCommand() { - return (ImportCommand) parentCommand; - } -} diff --git a/src/main/java/com/redislabs/riot/cli/AbstractTransferCommand.java b/src/main/java/com/redislabs/riot/cli/AbstractTransferCommand.java deleted file mode 100644 index 1b8981493..000000000 --- a/src/main/java/com/redislabs/riot/cli/AbstractTransferCommand.java +++ /dev/null @@ -1,60 +0,0 @@ -package com.redislabs.riot.cli; - -import com.redislabs.lettuce.helper.RedisOptions; -import com.redislabs.picocliredis.HelpCommand; -import lombok.extern.slf4j.Slf4j; -import org.springframework.batch.item.ItemProcessor; -import org.springframework.batch.item.ItemReader; -import org.springframework.batch.item.ItemWriter; -import picocli.CommandLine; -import picocli.CommandLine.Command; - -@Slf4j -@Command -public abstract class AbstractTransferCommand extends HelpCommand implements Runnable { - - @CommandLine.ParentCommand - protected TransferCommand parentCommand; - - @Override - public void run() { - ItemReader reader; - try { - reader = reader(); - } catch (Exception e) { - log.error("Could not create reader", e); - return; - } - ItemProcessor processor; - try { - processor = processor(); - } catch (Exception e) { - log.error("Could not create processor", e); - return; - } - ItemWriter writer; - try { - writer = writer(); - } catch (Exception e) { - log.error("Could not create writer", e); - return; - } - execute(reader, processor, writer); - } - - protected void execute(ItemReader reader, ItemProcessor processor, ItemWriter writer) { - parentCommand.execute(reader, processor, writer); - } - - protected abstract ItemReader reader() throws Exception; - - protected abstract ItemProcessor processor() throws Exception; - - protected abstract ItemWriter writer() throws Exception; - - protected RedisOptions redisOptions() { - return parentCommand.redisOptions(); - } - - -} diff --git a/src/main/java/com/redislabs/riot/cli/ExportCommand.java b/src/main/java/com/redislabs/riot/cli/ExportCommand.java deleted file mode 100644 index 47ef1a3c2..000000000 --- a/src/main/java/com/redislabs/riot/cli/ExportCommand.java +++ /dev/null @@ -1,28 +0,0 @@ -package com.redislabs.riot.cli; - -import org.springframework.batch.item.ItemReader; -import org.springframework.batch.item.redis.KeyValue; -import org.springframework.batch.item.redis.RedisKeyValueItemReader; -import org.springframework.batch.item.redis.support.QueueOptions; -import org.springframework.batch.item.redis.support.ReaderOptions; -import picocli.CommandLine; - -@CommandLine.Command(name = "export", description = "Export data from Redis", subcommands = {FileExportCommand.class, DatabaseExportCommand.class}) -public class ExportCommand extends TransferCommand { - - @CommandLine.Mixin - private ExportOptions exportOptions = new ExportOptions(); - @CommandLine.Option(names = "--key-regex", description = "Regular expression for key-field extraction", paramLabel = "") - private String keyRegex; - - @Override - protected String taskName() { - return "Exporting"; - } - - public ItemReader> reader() { - ReaderOptions readerOptions = ReaderOptions.builder().scanCount(exportOptions.getScanCount()).scanMatch(exportOptions.getScanMatch()).batchSize(exportOptions.getBatchSize()).valueQueueOptions(QueueOptions.builder().capacity(exportOptions.getQueueCapacity()).build()).threadCount(exportOptions.getThreads()).build(); - return RedisKeyValueItemReader.builder().redisOptions(redisOptions()).readerOptions(readerOptions).build(); - } - -} diff --git a/src/main/java/com/redislabs/riot/cli/ExportOptions.java b/src/main/java/com/redislabs/riot/cli/ExportOptions.java deleted file mode 100644 index 8822a552a..000000000 --- a/src/main/java/com/redislabs/riot/cli/ExportOptions.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.redislabs.riot.cli; - -import lombok.Getter; -import org.springframework.batch.item.redis.support.ReaderOptions; -import picocli.CommandLine.Option; - -public class ExportOptions { - - @Getter - @Option(names = "--count", description = "SCAN COUNT option (default: ${DEFAULT-VALUE})", paramLabel = "") - private long scanCount = ReaderOptions.DEFAULT_SCAN_COUNT; - @Getter - @Option(names = "--match", description = "SCAN MATCH pattern (default: ${DEFAULT-VALUE})", paramLabel = "") - private String scanMatch = ReaderOptions.DEFAULT_SCAN_MATCH; - @Getter - @Option(names = "--reader-queue", description = "Capacity of the reader queue (default: ${DEFAULT-VALUE})", paramLabel = "", hidden = true) - private int queueCapacity = 10000; - @Getter - @Option(names = "--reader-threads", description = "Number of reader threads (default: ${DEFAULT-VALUE})", paramLabel = "", hidden = true) - private int threads = 1; - @Getter - @Option(names = "--reader-batch", description = "Number of reader values to process at once (default: ${DEFAULT-VALUE})", paramLabel = "") - private int batchSize = 50; - -} \ No newline at end of file diff --git a/src/main/java/com/redislabs/riot/cli/ReplicateCommand.java b/src/main/java/com/redislabs/riot/cli/ReplicateCommand.java deleted file mode 100644 index 9563bdd7e..000000000 --- a/src/main/java/com/redislabs/riot/cli/ReplicateCommand.java +++ /dev/null @@ -1,74 +0,0 @@ -package com.redislabs.riot.cli; - -import com.redislabs.lettuce.helper.RedisOptions; -import com.redislabs.picocliredis.RedisCommandLineOptions; -import com.redislabs.riot.Riot; -import com.redislabs.riot.TransferOptions; -import org.springframework.batch.item.ItemWriter; -import org.springframework.batch.item.redis.KeyDump; -import org.springframework.batch.item.redis.RedisKeyDumpItemReader; -import org.springframework.batch.item.redis.RedisKeyDumpItemWriter; -import org.springframework.batch.item.redis.support.QueueOptions; -import org.springframework.batch.item.redis.support.ReaderOptions; -import org.springframework.batch.item.support.PassThroughItemProcessor; -import picocli.CommandLine; -import picocli.CommandLine.Command; - -@Command(name = "replicate", description = "Replicate a Redis database to another Redis database") -public class ReplicateCommand extends TransferCommand { - - @CommandLine.ParentCommand - private Riot riot; - @CommandLine.Mixin - private RedisCommandLineOptions target = new RedisCommandLineOptions(); - @CommandLine.Option(names = "--target-pool", description = "Max number of connections in target pool", paramLabel = "") - private Integer targetPoolMaxTotal; - @CommandLine.Mixin - private ExportOptions options = new ExportOptions(); - @CommandLine.Option(names = "--flush-interval", description = "Duration between notification flushes (default: ${DEFAULT-VALUE})", paramLabel = "") - private long flushPeriod = 50; - @CommandLine.Option(names = "--live", description = "Live replication") - private boolean live; - @CommandLine.Option(names = "--notification-queue", description = "Keyspace notification queue capacity (default: ${DEFAULT-VALUE})", paramLabel = "", hidden = true) - private int notificationQueueCapacity = 10000; - - private RedisOptions targetRedisOptions() { - RedisOptions redisOptions = Riot.redisOptions(target); - redisOptions.getPoolOptions().setMaxTotal(targetPoolMaxTotal == null ? getPoolMaxTotal() : targetPoolMaxTotal); - return redisOptions; - } - - @Override - protected boolean isQuiet() { - return riot.isQuiet(); - } - - @Override - protected TransferOptions transferOptions() { - TransferOptions transferOptions = super.transferOptions(); - if (live) { - transferOptions.setFlushPeriod(flushPeriod); - } - return transferOptions; - } - - private RedisKeyDumpItemReader reader() { - ReaderOptions readerOptions = ReaderOptions.builder().scanCount(options.getScanCount()).scanMatch(options.getScanMatch()).batchSize(options.getBatchSize()).threadCount(options.getThreads()).valueQueueOptions(QueueOptions.builder().capacity(options.getQueueCapacity()).build()).live(live).keyspaceNotificationQueueOptions(QueueOptions.builder().capacity(notificationQueueCapacity).build()).build(); - return RedisKeyDumpItemReader.builder().redisOptions(redisOptions()).readerOptions(readerOptions).build(); - } - - @Override - protected String taskName() { - return "Replicating"; - } - - @Override - public void run() { - execute(reader(), new PassThroughItemProcessor<>(), writer()); - } - - private ItemWriter> writer() { - return RedisKeyDumpItemWriter.builder().redisOptions(targetRedisOptions()).replace(true).build(); - } - -} diff --git a/src/main/java/com/redislabs/riot/cli/RiotCommand.java b/src/main/java/com/redislabs/riot/cli/RiotCommand.java deleted file mode 100644 index 0c365214c..000000000 --- a/src/main/java/com/redislabs/riot/cli/RiotCommand.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.redislabs.riot.cli; - -import com.redislabs.lettuce.helper.RedisOptions; -import com.redislabs.picocliredis.HelpCommand; -import com.redislabs.riot.Riot; -import picocli.CommandLine; -import picocli.CommandLine.Command; -import picocli.CommandLine.ParentCommand; - -@Command -public class RiotCommand extends HelpCommand implements Runnable { - - @ParentCommand - private Riot riot; - - protected boolean isQuiet() { - return riot.isQuiet(); - } - - @Override - public void run() { - CommandLine.usage(this, System.out); - } - - public RedisOptions redisOptions() { - return riot.redisOptions(); - } - - -} diff --git a/src/main/java/com/redislabs/riot/cli/TestCommand.java b/src/main/java/com/redislabs/riot/cli/TestCommand.java deleted file mode 100644 index 469d28fa2..000000000 --- a/src/main/java/com/redislabs/riot/cli/TestCommand.java +++ /dev/null @@ -1,81 +0,0 @@ -package com.redislabs.riot.cli; - -import io.lettuce.core.api.sync.BaseRedisCommands; -import io.lettuce.core.api.sync.RedisServerCommands; -import io.lettuce.core.metrics.CommandMetrics; -import io.lettuce.core.metrics.DefaultCommandLatencyCollectorOptions; -import lombok.extern.slf4j.Slf4j; -import org.HdrHistogram.Histogram; -import org.LatencyUtils.LatencyStats; -import picocli.CommandLine.Command; -import picocli.CommandLine.Option; - -import java.util.Map; -import java.util.TreeMap; -import java.util.concurrent.TimeUnit; - -@Slf4j -@Command(name = "test", description = "Execute a test") -public class TestCommand extends RiotCommand implements Runnable { - - public enum RedisTestType { - INFO, PING, LATENCY - } - - @Option(names = {"-t", "--test"}, description = "Test to execute: ${COMPLETION-CANDIDATES} (default: ${DEFAULT-VALUE})", paramLabel = "") - private RedisTestType test = RedisTestType.PING; - @Option(names = "--latency-iterations", description = "Number of iterations for latency test (default: ${DEFAULT-VALUE})", paramLabel = "") - private int latencyIterations = 1000; - @Option(names = "--latency-sleep", description = "Sleep duration in milliseconds between calls (default: ${DEFAULT-VALUE})", paramLabel = "") - private long latencySleep = 1; - @Option(names = "--latency-unit", description = "Latency unit (default: ${DEFAULT-VALUE})", paramLabel = "") - private TimeUnit latencyTimeUnit = TimeUnit.MILLISECONDS; - @Option(names = "--latency-distribution", description = "Show latency distribution") - private boolean latencyDistribution = false; - - @Override - @SuppressWarnings("unchecked") - public void run() { - BaseRedisCommands commands = redisOptions().commands(); - switch (test) { - case PING: - log.info("Received ping reply: {}", commands.ping()); - break; - case INFO: - log.info(((RedisServerCommands) commands).info()); - break; - case LATENCY: - latency(commands); - break; - default: - throw new IllegalArgumentException("Unknown test command: " + test); - } - } - - private void latency(BaseRedisCommands commands) { - LatencyStats stats = new LatencyStats(); - for (int index = 0; index < latencyIterations; index++) { - long startTime = System.nanoTime(); - commands.ping(); - stats.recordLatency(System.nanoTime() - startTime); - try { - Thread.sleep(latencySleep); - } catch (InterruptedException e) { - return; - } - } - Histogram histogram = stats.getIntervalHistogram(); - if (latencyDistribution) { - histogram.outputPercentileDistribution(System.out, 1000000.0); - } else { - DefaultCommandLatencyCollectorOptions options = DefaultCommandLatencyCollectorOptions.create(); - Map percentiles = new TreeMap<>(); - for (double targetPercentile : options.targetPercentiles()) { - percentiles.put(targetPercentile, latencyTimeUnit.convert(histogram.getValueAtPercentile(targetPercentile), TimeUnit.NANOSECONDS)); - } - CommandMetrics.CommandLatency latency = new CommandMetrics.CommandLatency(latencyTimeUnit.convert(histogram.getMinValue(), TimeUnit.NANOSECONDS), latencyTimeUnit.convert(histogram.getMaxValue(), TimeUnit.NANOSECONDS), percentiles); - System.out.println(latency.toString()); - } - } - -} diff --git a/src/main/java/com/redislabs/riot/cli/TransferCommand.java b/src/main/java/com/redislabs/riot/cli/TransferCommand.java deleted file mode 100644 index b082cd0ad..000000000 --- a/src/main/java/com/redislabs/riot/cli/TransferCommand.java +++ /dev/null @@ -1,52 +0,0 @@ -package com.redislabs.riot.cli; - -import com.redislabs.lettuce.helper.RedisOptions; -import com.redislabs.riot.Transfer; -import com.redislabs.riot.TransferOptions; -import org.springframework.batch.item.ItemProcessor; -import org.springframework.batch.item.ItemReader; -import org.springframework.batch.item.ItemWriter; -import picocli.CommandLine; - -public abstract class TransferCommand extends RiotCommand { - - @CommandLine.Option(names = {"-t", "--threads"}, description = "Thread count (default: ${DEFAULT-VALUE})", paramLabel = "") - private int threads = 1; - @CommandLine.Option(names = "--pool", description = "Max number of connections in pool (default: #threads)", paramLabel = "") - private Integer poolMaxTotal; - @CommandLine.Option(names = {"-b", "--batch"}, description = "Number of items in each batch (default: ${DEFAULT-VALUE})", paramLabel = "") - private int batchSize = 50; - @CommandLine.Option(names = "--max", description = "Max number of items to read", paramLabel = "") - private Integer maxItemCount; - - @Override - public RedisOptions redisOptions() { - RedisOptions redisOptions = super.redisOptions(); - redisOptions.getPoolOptions().setMaxTotal(getPoolMaxTotal()); - return redisOptions; - } - - protected int getPoolMaxTotal() { - if (this.poolMaxTotal == null) { - return this.threads; - } - return this.poolMaxTotal; - - } - - public void execute(ItemReader reader, ItemProcessor processor, ItemWriter writer) { - Transfer transfer = Transfer.builder().reader(reader).processor(processor).writer(writer).options(transferOptions()).build(); - ProgressBarOptions progressBarOptions = ProgressBarOptions.builder().taskName(taskName()).initialMax(maxItemCount).quiet(isQuiet()).build(); - ProgressBarReporter reporter = ProgressBarReporter.builder().transfer(transfer).options(progressBarOptions).build(); - reporter.start(); - transfer.execute(); - reporter.stop(); - } - - protected abstract String taskName(); - - protected TransferOptions transferOptions() { - return TransferOptions.builder().batchSize(batchSize).threadCount(threads).maxItemCount(maxItemCount).build(); - } - -} diff --git a/src/main/java/com/redislabs/riot/cli/file/FileExtensions.java b/src/main/java/com/redislabs/riot/cli/file/FileExtensions.java deleted file mode 100644 index 7ded40f5a..000000000 --- a/src/main/java/com/redislabs/riot/cli/file/FileExtensions.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.redislabs.riot.cli.file; - -public interface FileExtensions { - - String CSV = "csv"; - String TSV = "tsv"; - String FW = "fw"; - String JSON = "json"; - String XML = "xml"; -} diff --git a/src/test/java/com/redislabs/riot/TestDatabase.java b/src/test/java/com/redislabs/riot/TestDatabase.java deleted file mode 100644 index f15d3aa8f..000000000 --- a/src/test/java/com/redislabs/riot/TestDatabase.java +++ /dev/null @@ -1,57 +0,0 @@ -package com.redislabs.riot; - -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.Statement; -import java.util.List; - -import javax.sql.DataSource; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.springframework.boot.autoconfigure.jdbc.DataSourceProperties; - -import com.zaxxer.hikari.HikariDataSource; - -public class TestDatabase extends BaseTest { - - @Test - public void testExportToDatabase() throws Exception { - populateBeersTable(); - Statement statement = dataSource().getConnection().createStatement(); - statement.execute("SELECT * from beers"); - ResultSet resultSet = statement.getResultSet(); - int count = 0; - while (resultSet.next()) { - Assertions.assertTrue(resultSet.getInt("id") > 0); - count++; - } - Assertions.assertEquals(BEER_COUNT, count); - } - - private void populateBeersTable() throws Exception { - Connection connection = dataSource().getConnection(); - connection.createStatement().execute("DROP TABLE IF EXISTS beers"); - connection.createStatement() - .execute("CREATE TABLE IF NOT EXISTS beers (id INT NOT NULL, name VARCHAR(500), PRIMARY KEY (id))"); - runFile("import-csv-hash"); - runFile("export-db"); - commands().flushall(); - } - - @Test - public void testImportDatabase() throws Exception { - populateBeersTable(); - runFile("import-db"); - List keys = commands().keys("dbbeer:*"); - Assertions.assertEquals(BEER_COUNT, keys.size()); - } - - private DataSource dataSource() { - DataSourceProperties properties = new DataSourceProperties(); - properties.setUrl("jdbc:hsqldb:mem:mymemdb"); - properties.setDriverClassName("org.hsqldb.jdbc.JDBCDriver"); - return properties.initializeDataSourceBuilder().type(HikariDataSource.class).build(); - } - -} diff --git a/src/test/resources/commands/export-csv.txt b/src/test/resources/commands/export-csv.txt deleted file mode 100644 index 917e57cbe..000000000 --- a/src/test/resources/commands/export-csv.txt +++ /dev/null @@ -1 +0,0 @@ -$ riot export --match beer:* --key-regex "beer:(?.*)" file /tmp/beers.csv --fields id name brewery_id abv --header --delimiter '|' \ No newline at end of file diff --git a/src/test/resources/commands/export-db.txt b/src/test/resources/commands/export-db.txt deleted file mode 100644 index d19a4b001..000000000 --- a/src/test/resources/commands/export-db.txt +++ /dev/null @@ -1 +0,0 @@ -$ riot -r redis://localhost:6379 export --match "beer:*" --key-regex "beer:(?.*)" db "INSERT INTO beers (id, name) VALUES (:id, :name)" --url jdbc:hsqldb:mem:mymemdb \ No newline at end of file diff --git a/src/test/resources/commands/export-json-gz.txt b/src/test/resources/commands/export-json-gz.txt deleted file mode 100644 index bd97e90eb..000000000 --- a/src/test/resources/commands/export-json-gz.txt +++ /dev/null @@ -1 +0,0 @@ -$ riot export --match beer:* file /tmp/beers.json.gz --fields id name brewery_id abv \ No newline at end of file diff --git a/src/test/resources/commands/export-json.txt b/src/test/resources/commands/export-json.txt deleted file mode 100644 index b03585744..000000000 --- a/src/test/resources/commands/export-json.txt +++ /dev/null @@ -1 +0,0 @@ -$ riot export --match beer:* --key-regex "beer:(?.*)" file /tmp/beers.json --fields id name brewery_id abv \ No newline at end of file diff --git a/src/test/resources/commands/gen-faker-hash.txt b/src/test/resources/commands/gen-faker-hash.txt deleted file mode 100644 index 7dc8f2177..000000000 --- a/src/test/resources/commands/gen-faker-hash.txt +++ /dev/null @@ -1 +0,0 @@ -$ riot import --max 100 --keyspace person --keys id gen id=#index firstName=name.firstName lastName=name.lastName address=address.fullAddress \ No newline at end of file diff --git a/src/test/resources/commands/gen-faker-index-introspection.txt b/src/test/resources/commands/gen-faker-index-introspection.txt deleted file mode 100644 index 238b8cab5..000000000 --- a/src/test/resources/commands/gen-faker-index-introspection.txt +++ /dev/null @@ -1 +0,0 @@ -$ riot import --max 100 --command ftadd --index beerIntrospection --keys id gen --faker-index beerIntrospection \ No newline at end of file diff --git a/src/test/resources/commands/gen-faker-script-processor-hash.txt b/src/test/resources/commands/gen-faker-script-processor-hash.txt deleted file mode 100644 index 8e2da4c62..000000000 --- a/src/test/resources/commands/gen-faker-script-processor-hash.txt +++ /dev/null @@ -1 +0,0 @@ -$ riot import --script "function process(item) { item.address = item.address.toUpperCase(); return item; } process(item);" --max 100 --keyspace person --keys id gen id=#index firstName=name.firstName lastName=name.lastName address=address.fullAddress \ No newline at end of file diff --git a/src/test/resources/commands/gen-faker-set.txt b/src/test/resources/commands/gen-faker-set.txt deleted file mode 100644 index 2ff4cbc3f..000000000 --- a/src/test/resources/commands/gen-faker-set.txt +++ /dev/null @@ -1 +0,0 @@ -$ riot import --max 10000 --command sadd --members name --keyspace got:characters gen name=gameOfThrones.character \ No newline at end of file diff --git a/src/test/resources/commands/gen-faker-stream.txt b/src/test/resources/commands/gen-faker-stream.txt deleted file mode 100644 index 4f8d906e3..000000000 --- a/src/test/resources/commands/gen-faker-stream.txt +++ /dev/null @@ -1 +0,0 @@ -$ riot import --batch 50 --max 1000 --command xadd --keyspace teststream --keys category gen id=#index category=number.randomDigit \ No newline at end of file diff --git a/src/test/resources/commands/gen-faker-zset.txt b/src/test/resources/commands/gen-faker-zset.txt deleted file mode 100644 index 640c8f635..000000000 --- a/src/test/resources/commands/gen-faker-zset.txt +++ /dev/null @@ -1 +0,0 @@ -$ riot import --batch 50 --max 10000 --command zadd --keyspace leases --keys ip --members lease --score=time gen ip=number.digits(4) lease=number.digits(2) time=number.digits(5) \ No newline at end of file diff --git a/src/test/resources/commands/gen-faker.txt b/src/test/resources/commands/gen-faker.txt deleted file mode 100644 index c50a82ace..000000000 --- a/src/test/resources/commands/gen-faker.txt +++ /dev/null @@ -1 +0,0 @@ -$ riot -r redis://localhost:6379 import --max 1000 --keyspace test --keys index gen index=#index firstName=name.firstName lastName=name.lastName address=address.fullAddress \ No newline at end of file diff --git a/src/test/resources/commands/gen-simple.txt b/src/test/resources/commands/gen-simple.txt deleted file mode 100644 index 2aade4059..000000000 --- a/src/test/resources/commands/gen-simple.txt +++ /dev/null @@ -1 +0,0 @@ -$ riot gen --metadata -d field1=100 field2=1000 --batch 100 --threads 3 --max 10000 --keyspace test --keys index \ No newline at end of file diff --git a/src/test/resources/commands/import-csv-geo.txt b/src/test/resources/commands/import-csv-geo.txt deleted file mode 100644 index 88fdc2d2c..000000000 --- a/src/test/resources/commands/import-csv-geo.txt +++ /dev/null @@ -1 +0,0 @@ -$ riot import --command geoadd --keyspace airportgeo --members AirportID --lon Longitude --lat Latitude file https://raw.githubusercontent.com/jpatokal/openflights/master/data/airports.dat --fields AirportID Name City Country IATA ICAO Latitude Longitude Altitude Timezone DST Tz Type Source \ No newline at end of file diff --git a/src/test/resources/commands/import-csv-hash.txt b/src/test/resources/commands/import-csv-hash.txt deleted file mode 100644 index 5e592c162..000000000 --- a/src/test/resources/commands/import-csv-hash.txt +++ /dev/null @@ -1 +0,0 @@ -$ riot import --keyspace beer --keys id file https://raw.githubusercontent.com/nickhould/craft-beers-dataset/master/data/processed/beers.csv --header \ No newline at end of file diff --git a/src/test/resources/commands/import-csv-processor-hash-dateformat.txt b/src/test/resources/commands/import-csv-processor-hash-dateformat.txt deleted file mode 100644 index 5fc2dc474..000000000 --- a/src/test/resources/commands/import-csv-processor-hash-dateformat.txt +++ /dev/null @@ -1 +0,0 @@ -$ riot import --date-format "MM/dd/yyyy HH:mm:ss a" --spel "EventStartDate=remove('Event Start Date')" "EpochStart=#date.parse(EventStartDate).getTime()" "index=#index" --keyspace event --keys Id file "https://data.lacity.org/api/views/rx9t-fp7k/rows.csv?accessType=DOWNLOAD" --header \ No newline at end of file diff --git a/src/test/resources/commands/import-csv-processor-search-geo.txt b/src/test/resources/commands/import-csv-processor-search-geo.txt deleted file mode 100644 index 6c55c15f2..000000000 --- a/src/test/resources/commands/import-csv-processor-search-geo.txt +++ /dev/null @@ -1 +0,0 @@ -$ riot import --command ftadd --index airports -k AirportID --spel "Location=#geo(Longitude,Latitude)" file https://raw.githubusercontent.com/jpatokal/openflights/master/data/airports.dat --fields AirportID Name City Country IATA ICAO Latitude Longitude Altitude Timezone DST Tz Type Source \ No newline at end of file diff --git a/src/test/resources/commands/import-csv-processor-search.txt b/src/test/resources/commands/import-csv-processor-search.txt deleted file mode 100644 index 382b597d7..000000000 --- a/src/test/resources/commands/import-csv-processor-search.txt +++ /dev/null @@ -1 +0,0 @@ -$ riot import --spel location=#geo(lon,lat) --command ftadd --index laevents --keys Id file "https://data.lacity.org/api/views/rx9t-fp7k/rows.csv?accessType=DOWNLOAD" --header --regex 'Event Location'="\((?.+),\s+(?.+)\)" \ No newline at end of file diff --git a/src/test/resources/commands/import-csv-search.txt b/src/test/resources/commands/import-csv-search.txt deleted file mode 100644 index 8652ee637..000000000 --- a/src/test/resources/commands/import-csv-search.txt +++ /dev/null @@ -1 +0,0 @@ -$ riot import --command ftadd --index beers --keys id file https://raw.githubusercontent.com/nickhould/craft-beers-dataset/master/data/processed/beers.csv --header \ No newline at end of file diff --git a/src/test/resources/commands/import-db.txt b/src/test/resources/commands/import-db.txt deleted file mode 100644 index 116cdda58..000000000 --- a/src/test/resources/commands/import-db.txt +++ /dev/null @@ -1 +0,0 @@ -$ riot -r redis://localhost:6379 import --keyspace dbbeer --keys id db "SELECT * FROM beers" --url jdbc:hsqldb:mem:mymemdb \ No newline at end of file diff --git a/src/test/resources/commands/import-elastic-json.txt b/src/test/resources/commands/import-elastic-json.txt deleted file mode 100644 index 3fa5ce2ab..000000000 --- a/src/test/resources/commands/import-elastic-json.txt +++ /dev/null @@ -1 +0,0 @@ -$ riot import --keyspace estest --keys _id file %s \ No newline at end of file diff --git a/src/test/resources/commands/import-json-hash.txt b/src/test/resources/commands/import-json-hash.txt deleted file mode 100644 index 7b683e7de..000000000 --- a/src/test/resources/commands/import-json-hash.txt +++ /dev/null @@ -1 +0,0 @@ -$ riot import --keyspace beer --keys id file https://raw.githubusercontent.com/rethinkdb/beerthink/master/data/beers.json \ No newline at end of file diff --git a/src/test/resources/commands/import-xml-hash.txt b/src/test/resources/commands/import-xml-hash.txt deleted file mode 100644 index 15044b446..000000000 --- a/src/test/resources/commands/import-xml-hash.txt +++ /dev/null @@ -1 +0,0 @@ -$ riot import --keyspace release --keys id file https://raw.githubusercontent.com/Redislabs-Solution-Architects/riot/master/src/test/resources/releases.xml \ No newline at end of file diff --git a/src/test/resources/commands/replicate-live.txt b/src/test/resources/commands/replicate-live.txt deleted file mode 100644 index 22fcc80a0..000000000 --- a/src/test/resources/commands/replicate-live.txt +++ /dev/null @@ -1 +0,0 @@ -$ riot -r redis://localhost:6379 replicate -r redis://localhost:6380 --live \ No newline at end of file diff --git a/src/test/resources/commands/replicate.txt b/src/test/resources/commands/replicate.txt deleted file mode 100644 index 6c263bbea..000000000 --- a/src/test/resources/commands/replicate.txt +++ /dev/null @@ -1 +0,0 @@ -$ riot -r redis://localhost:6379 replicate -r redis://localhost:6380 --batch 1 \ No newline at end of file diff --git a/test/build.gradle b/test/build.gradle new file mode 100644 index 000000000..601d1acf4 --- /dev/null +++ b/test/build.gradle @@ -0,0 +1,29 @@ +plugins { + id 'java-library' +} + +bootJar { + enabled = false +} + +jar { + enabled = true +} + +dependencies { + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' + implementation project(':core') + implementation 'org.slf4j:slf4j-jdk14' + implementation 'org.codehaus.plexus:plexus-utils:3.3.0' + implementation 'org.junit.jupiter:junit-jupiter-api' + api 'org.junit.jupiter:junit-jupiter-engine' + api('it.ozimov:embedded-redis:0.7.3') { + exclude group: 'org.slf4j', module: 'slf4j-simple' + } + api 'com.redislabs:spring-batch-redisearch:2.4.1' +} + +configurations { + all*.exclude module: 'spring-boot-starter-logging' +} \ No newline at end of file diff --git a/src/test/java/com/redislabs/riot/BaseTest.java b/test/src/main/java/com/redislabs/riot/test/BaseTest.java similarity index 71% rename from src/test/java/com/redislabs/riot/BaseTest.java rename to test/src/main/java/com/redislabs/riot/test/BaseTest.java index 28c67dd78..9dfc21c26 100644 --- a/src/test/java/com/redislabs/riot/BaseTest.java +++ b/test/src/main/java/com/redislabs/riot/test/BaseTest.java @@ -1,9 +1,8 @@ -package com.redislabs.riot; +package com.redislabs.riot.test; import com.redislabs.lettusearch.RediSearchClient; import com.redislabs.lettusearch.RediSearchCommands; import com.redislabs.lettusearch.StatefulRediSearchConnection; -import com.redislabs.picocliredis.Application; import io.lettuce.core.RedisURI; import org.apache.commons.io.IOUtils; import org.codehaus.plexus.util.cli.CommandLineUtils; @@ -18,30 +17,27 @@ import java.io.InputStream; import java.nio.charset.Charset; -public class BaseTest { +public abstract class BaseTest { - private final static String COMMAND_PREAMBLE = "$ riot "; + private final static String COMMAND_PREAMBLE = "❯"; private final static int REDIS_PORT = 6379; private final static String REDIS_HOST = "localhost"; private static RedisServer server; private static RediSearchClient client; - private static StatefulRediSearchConnection connection; - protected static final int BEER_COUNT = 2410; + protected static StatefulRediSearchConnection connection; @BeforeAll public static void setup() { - server = serverBuilder(REDIS_PORT).setting("notify-keyspace-events AK") - .setting("loadmodule /Users/jruaux/git/RediSearch/build/redisearch.so").build(); + server = serverBuilder(REDIS_PORT).setting("notify-keyspace-events AK").setting("loadmodule /Users/jruaux/git/RediSearch/build/redisearch.so").build(); server.start(); client = RediSearchClient.create(RedisURI.create(REDIS_HOST, REDIS_PORT)); connection = client.connect(); } protected static RedisServerBuilder serverBuilder(int port) { - RedisExecProvider provider = RedisExecProvider.defaultProvider().override(OS.MAC_OS_X, - "/usr/local/bin/redis-server"); + RedisExecProvider provider = RedisExecProvider.defaultProvider().override(OS.MAC_OS_X, "/usr/local/bin/redis-server"); return RedisServer.builder().redisExecProvider(provider).port(port); } @@ -68,7 +64,7 @@ public static void teardown() { } protected int runFile(String filename, Object... args) { - try (InputStream inputStream = BaseTest.class.getResourceAsStream("/commands/" + filename + ".txt")) { + try (InputStream inputStream = getClass().getResourceAsStream(filename)) { return runCommand(removePreamble(IOUtils.toString(inputStream, Charset.defaultCharset())), args); } catch (Exception e) { e.printStackTrace(); @@ -77,14 +73,24 @@ protected int runFile(String filename, Object... args) { } protected int runCommand(String command, Object... args) throws Exception { - return new Riot().execute(CommandLineUtils.translateCommandline(String.format(command, args))); + return execute(CommandLineUtils.translateCommandline(String.format(command, args))); } + protected abstract int execute(String[] args) throws Exception; + + protected String commandPrefix() { + return COMMAND_PREAMBLE + " " + applicationName(); + } + + protected abstract String applicationName(); + private String removePreamble(String command) { - if (command.startsWith(COMMAND_PREAMBLE)) { - return command.substring(COMMAND_PREAMBLE.length()); + if (command.startsWith(commandPrefix())) { + return command.substring(commandPrefix().length()); } return command; } + + } diff --git a/test/src/main/java/com/redislabs/riot/test/DataPopulator.java b/test/src/main/java/com/redislabs/riot/test/DataPopulator.java new file mode 100644 index 000000000..081ba5ec0 --- /dev/null +++ b/test/src/main/java/com/redislabs/riot/test/DataPopulator.java @@ -0,0 +1,71 @@ +package com.redislabs.riot.test; + +import io.lettuce.core.api.StatefulRedisConnection; +import io.lettuce.core.api.sync.RedisCommands; +import lombok.Builder; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.item.redis.support.DataType; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Slf4j +@Builder +public class DataPopulator implements Runnable { + + @NonNull + private StatefulRedisConnection connection; + @Builder.Default + private int start = 0; + @Builder.Default + private int end = 1000; + private Long sleep; + private long expire; + @Builder.Default + private int collectionModulo = 10; + @Builder.Default + private int zsetScoreModulo = 3; + @NonNull + @Builder.Default + private List dataTypes = Arrays.asList(DataType.values()); + + @Override + public void run() { + RedisCommands commands = connection.sync(); + for (int index = start; index < end; index++) { + if (dataTypes.contains(DataType.STRING)) { + String stringKey = "string:" + index; + commands.set(stringKey, "value:" + index); + if (expire > 0) { + commands.expireat(stringKey, expire); + } + } + Map hash = new HashMap<>(); + hash.put("field1", "value" + index); + hash.put("field2", "value" + index); + if (dataTypes.contains(DataType.HASH)) { + commands.hmset("hash:" + index, hash); + } + if (dataTypes.contains(DataType.SET)) { + commands.sadd("set:" + (index % collectionModulo), "member:" + index); + } + if (dataTypes.contains(DataType.ZSET)) { + commands.zadd("zset:" + (index % collectionModulo), index % zsetScoreModulo, "member:" + index); + } + if (dataTypes.contains(DataType.STREAM)) { + commands.xadd("stream:" + (index % collectionModulo), hash); + } + if (sleep == null) { + continue; + } + try { + Thread.sleep(sleep); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } +}