Skip to content

Commit

Permalink
made generator metadata optional
Browse files Browse the repository at this point in the history
  • Loading branch information
jruaux committed Mar 24, 2020
1 parent 6d0dfcc commit e184669
Show file tree
Hide file tree
Showing 4 changed files with 51 additions and 45 deletions.
33 changes: 17 additions & 16 deletions src/docs/asciidoc/index.adoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
:idprefix:
:idseparator: -
:toc: macro
:sectanchors:
:icons: font
:project-repo: Redislabs-Solution-Architects/riot
:repo-url: https://github.com/{project-repo}
Expand Down Expand Up @@ -122,7 +123,7 @@ include::{commandsdir}/import-csv-processor-hash-dateformat.txt[]

Use an external or inline script to provide the processor logic.

.Javascript example
==== Javascript example
[source,javascript]
----
function process(item) {
Expand All @@ -142,85 +143,85 @@ include::{commandsdir}/gen-faker-script-processor-hash.txt[]

=== Files

.Import CSV
==== Import CSV
[source,shell]
----
include::{commandsdir}/import-csv-hash.txt[]
----

.Import JSON
==== Import JSON
[source,shell]
----
include::{commandsdir}/import-json-hash.txt[]
----

.Import CSV into RediSearch
==== Import CSV into RediSearch
[source,shell]
----
include::{commandsdir}/import-csv-search.txt[]
----

.Export CSV
==== Export CSV
[source,shell]
----
include::{commandsdir}/export-csv.txt[]
----

.Export JSON
==== Export JSON
[source,shell]
----
include::{commandsdir}/export-json.txt[]
----

.Export compressed JSON
==== Export compressed JSON
[source,shell]
----
include::{commandsdir}/export-json_gz.txt[]
----

=== Databases

.Import from a database
==== Import from a database
[source,shell]
----
include::{commandsdir}/import-db.txt[]
----

.Export to a database
==== Export to a database
[source,shell]
----
include::{commandsdir}/export-db.txt[]
----

=== Generators

.Generate hashes using simple generator
==== Generate hashes using simple generator
[source,shell]
----
include::{commandsdir}/gen-simple.txt[]
----

.Generate hashes using Faker
==== Generate hashes using Faker
[source,shell]
----
include::{commandsdir}/gen-faker-hash.txt[]
----

.Generate Redis sets using Faker
==== Generate Redis sets using Faker
[source,shell]
----
include::{commandsdir}/gen-faker-set.txt[]
----

.Generate RediSearch data using introspection
==== Generate RediSearch data using introspection
[source,shell]
----
include::{commandsdir}/gen-faker-index-introspection.txt[]
----

=== Replication

.Live replication from `localhost:16379` to `localhost:16380`
==== Live replication from `localhost:16379` to `localhost:16380`
[source,shell]
----
include::{commandsdir}/replicate-live.txt[]
Expand All @@ -242,14 +243,14 @@ riot --metrics …
----

=== Using Redis Enterprise
.Strings
==== Strings
[source,shell]
----
$ riot -s redis-12001.redislabs.com:12001 --max-total 96 gen --batch 5000 --threads 96 --max 100000000 --command set --keyspace hash --keys index --value index
----
image::images/rs-strings.png[]

.Streams
==== Streams
[source,shell]
----
$ riot -s redis-12001.internal.jrx.demo.redislabs.com:12001 --max-total 96 gen --batch 5000 --threads 96 --max 100000000 --command xadd --keyspace stream --keys partition
Expand Down
5 changes: 4 additions & 1 deletion src/main/java/com/redislabs/riot/cli/GeneratorCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ public class GeneratorCommand extends ImportCommand {
private Map<String, Integer> simpleFields = new LinkedHashMap<>();
@Option(names = "--locale", description = "Faker locale (default: ${DEFAULT-VALUE})", paramLabel = "<tag>")
private Locale locale = Locale.ENGLISH;
@Option(names = "--metadata", description = "Include metadata (index, partition)")
private boolean includeMetadata;

private String expression(Field field) {
if (field instanceof TextField) {
Expand All @@ -60,7 +62,8 @@ private List<String> fakerArgs(Map<String, String> fakerFields) {

@Override
protected GeneratorReader reader() throws Exception {
return new GeneratorReader().locale(locale).fakerFields(fakerFields()).simpleFields(simpleFields);
return GeneratorReader.builder().locale(locale).includeMetadata(includeMetadata).fakerFields(fakerFields())
.simpleFields(simpleFields).build();
}

private Map<String, String> fakerFields() {
Expand Down
56 changes: 29 additions & 27 deletions src/main/java/com/redislabs/riot/generator/GeneratorReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,27 +15,26 @@
import org.springframework.expression.EvaluationContext;
import org.springframework.expression.Expression;
import org.springframework.expression.spel.standard.SpelExpressionParser;
import org.springframework.expression.spel.support.SimpleEvaluationContext.Builder;
import org.springframework.expression.spel.support.SimpleEvaluationContext;
import org.springframework.lang.Nullable;
import org.springframework.util.ClassUtils;

import com.redislabs.riot.transfer.FlowThread;

import lombok.Builder;
import lombok.Setter;
import lombok.experimental.Accessors;

@Accessors(fluent = true)
public class GeneratorReader extends AbstractItemStreamItemReader<Map<String, Object>> {

public final static String FIELD_INDEX = "index";
public final static String FIELD_PARTITION = "partition";
public final static String FIELD_PARTITIONS = "partitions";

@Setter
private Locale locale;
private int maxItemCount = Integer.MAX_VALUE;
private Map<String, Expression> fakerFields = new HashMap<>();
private Map<String, String> simpleFields = new LinkedHashMap<>();
private @Setter int maxItemCount = Integer.MAX_VALUE;
private @Setter Locale locale;
private @Setter boolean includeMetadata;
private @Setter Map<String, Expression> fakerFields = new HashMap<>();
private @Setter Map<String, String> simpleFields = new LinkedHashMap<>();

private ThreadLocal<Integer> partition = new ThreadLocal<>();
private ThreadLocal<Integer> partitions = new ThreadLocal<>();
Expand All @@ -48,6 +47,21 @@ public GeneratorReader() {
setName(ClassUtils.getShortName(getClass()));
}

@Builder
private GeneratorReader(Integer maxItemCount, Locale locale, Boolean includeMetadata,
Map<String, String> fakerFields, Map<String, Integer> simpleFields) {
if (maxItemCount != null) {
this.maxItemCount = maxItemCount;
}
this.locale = locale;
if (includeMetadata != null) {
this.includeMetadata = includeMetadata;
}
SpelExpressionParser parser = new SpelExpressionParser();
fakerFields.forEach((k, v) -> this.fakerFields.put(k, parser.parseExpression(v)));
simpleFields.forEach((k, v) -> this.simpleFields.put(k, StringUtils.leftPad("", v, "x")));
}

public int partition() {
return partition.get();
}
Expand All @@ -56,20 +70,6 @@ public int partitions() {
return partitions.get();
}

public GeneratorReader fakerFields(Map<String, String> fields) {
SpelExpressionParser parser = new SpelExpressionParser();
fields.forEach((k, v) -> fakerFields.put(k, parser.parseExpression(v)));
return this;
}

public GeneratorReader simpleFields(Map<String, Integer> fields) {
for (Entry<String, Integer> field : fields.entrySet()) {
String string = StringUtils.leftPad("", field.getValue(), "x");
simpleFields.put(field.getKey(), string);
}
return this;
}

@Override
public void open(ExecutionContext executionContext) throws ItemStreamException {
this.partition.set(executionContext.getInt(FlowThread.CONTEXT_PARTITION));
Expand All @@ -83,8 +83,8 @@ public void open(ExecutionContext executionContext) throws ItemStreamException {
this.currentItemCount.set(0);
GeneratorFaker faker = new GeneratorFaker(locale, this);
ReflectivePropertyAccessor accessor = new ReflectivePropertyAccessor();
Builder builder = new Builder(accessor).withInstanceMethods().withRootObject(faker);
this.context.set(builder.build());
this.context
.set(new SimpleEvaluationContext.Builder(accessor).withInstanceMethods().withRootObject(faker).build());
}

public int index() {
Expand All @@ -101,9 +101,11 @@ public void close() {
public Map<String, Object> read() throws Exception, UnexpectedInputException, ParseException {
currentItemCount.set(currentItemCount.get() + 1);
Map<String, Object> map = new HashMap<>();
map.put(FIELD_INDEX, index());
map.put(FIELD_PARTITION, partition.get());
map.put(FIELD_PARTITIONS, partitions.get());
if (includeMetadata) {
map.put(FIELD_INDEX, index());
map.put(FIELD_PARTITION, partition.get());
map.put(FIELD_PARTITIONS, partitions.get());
}
for (Entry<String, Expression> entry : fakerFields.entrySet()) {
map.put(entry.getKey(), entry.getValue().getValue(context.get()));
}
Expand Down
2 changes: 1 addition & 1 deletion src/test/resources/commands/gen-simple.txt
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
$ riot gen -d field1=100 field2=1000 --batch 100 --threads 3 --max 10000 --keyspace test --keys index
$ riot gen --metadata -d field1=100 field2=1000 --batch 100 --threads 3 --max 10000 --keyspace test --keys index

0 comments on commit e184669

Please sign in to comment.