Skip to content

Commit

Permalink
Merge pull request #3 from open-ch/feature/quote_cq_names
Browse files Browse the repository at this point in the history 
quote cq names
  • Loading branch information
Konrad Bucheli authored Dec 13, 2018
2 parents 1e8e9fc + 339356d commit 18901af
Show file tree
Hide file tree
Showing 4 changed files with 95 additions and 81 deletions.
124 changes: 64 additions & 60 deletions influxdb-schema-updater
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,10 +146,10 @@ sub extract_database_updates {
my ($db_schemas_in_influxdb, $db_schemas_in_config, $dryrun, $force) = @_;

my ($old_dbs, $eq_dbs, $new_dbs) = get_Ldifference_intersection_Rdifference([keys %{$db_schemas_in_influxdb}], [keys %{$db_schemas_in_config}]);

my %rp_updates;
for my $db (@$eq_dbs, @$new_dbs) {
my ($old, $updated, $new) = extract_retention_policy_updates($db, $db_schemas_in_influxdb->{$db}, $db_schemas_in_config->{$db}->{rps}, $dryrun, $force);
my ($old, $updated, $new) = extract_retention_policy_updates($db, $db_schemas_in_influxdb->{$db}, $db_schemas_in_config->{$db}, $dryrun, $force);
$rp_updates{old_rps}->{$db} = $old;
$rp_updates{updated_rps}->{$db} = $updated;
$rp_updates{new_rps}->{$db} = $new;
Expand All @@ -169,7 +169,7 @@ sub extract_database_updates {
object => 'db',
db => $db,
name => $db,
query => "DROP DATABASE $db;",
query => qq{DROP DATABASE "$db";},
skip => $dryrun || !$force,
};
}
Expand All @@ -180,7 +180,7 @@ sub extract_database_updates {
object => 'db',
db => $db,
name => $db,
query => $db_schemas_in_config->{$db}->{create_query},
query => qq{CREATE DATABASE "$db";},
skip => $dryrun,
};
}
Expand Down Expand Up @@ -209,7 +209,7 @@ sub extract_retention_policy_updates {
object => 'rp',
db => $db,
name => $rp,
query => "DROP RETENTION POLICY \"$rp\" ON $db;",
query => qq{DROP RETENTION POLICY "$rp" ON "$db";},
skip => $dryrun || !$force,
};
}
Expand All @@ -221,7 +221,7 @@ sub extract_retention_policy_updates {
object => 'rp',
db => $db,
name => $rp,
query => "ALTER RETENTION POLICY \"$rp\" ON $db DURATION $rps_in_config->{$rp}->{duration} REPLICATION 1 SHARD DURATION $rps_in_config->{$rp}->{shard_duration}" . ($rps_in_config->{$rp}->{default} ? ' DEFAULT;' : ';'),
query => qq{ALTER RETENTION POLICY "$rp" ON "$db" DURATION $rps_in_config->{$rp}->{duration} REPLICATION 1 SHARD DURATION $rps_in_config->{$rp}->{shard_duration}} . ($rps_in_config->{$rp}->{default} ? ' DEFAULT;' : ';'),
skip => $dryrun,
};
}
Expand All @@ -233,7 +233,7 @@ sub extract_retention_policy_updates {
object => 'rp',
db => $db,
name => $rp,
query => "CREATE RETENTION POLICY \"$rp\" ON $db DURATION $rps_in_config->{$rp}->{duration} REPLICATION $rps_in_config->{$rp}->{rp_replication} SHARD DURATION $rps_in_config->{$rp}->{shard_duration}" . ($rps_in_config->{$rp}->{default} ? ' DEFAULT;' : ';'),
query => qq{CREATE RETENTION POLICY "$rp" ON "$db" DURATION $rps_in_config->{$rp}->{duration} REPLICATION $rps_in_config->{$rp}->{rp_replication} SHARD DURATION $rps_in_config->{$rp}->{shard_duration}} . ($rps_in_config->{$rp}->{default} ? ' DEFAULT;' : ';'),
skip => $dryrun,
};
}
Expand Down Expand Up @@ -276,7 +276,7 @@ sub extract_continuous_query_updates {
object => 'cq',
db => $db,
name => $cq,
query => "DROP CONTINUOUS QUERY $cq ON $db;",
query => qq{DROP CONTINUOUS QUERY "$cq" ON "$db";},
skip => $dryrun || !$force,
};
}
Expand All @@ -287,7 +287,7 @@ sub extract_continuous_query_updates {
object => 'cq',
db => $db,
name => $cq,
query => "DROP CONTINUOUS QUERY $cq ON $db; $all_cqs_in_config->{$db}->{$cq};",
query => qq{DROP CONTINUOUS QUERY "$cq" ON "$db"; $all_cqs_in_config->{$db}->{$cq};},
skip => $dryrun,
};
}
Expand Down Expand Up @@ -346,7 +346,7 @@ sub load_db_schemas_in_influxdb {

my %db_schemas_in_influxdb;
for my $db (@dbs_in_influxdb) {
my $rp_query_res = query_influxql($influxdb_client, "SHOW RETENTION POLICIES ON $db");
my $rp_query_res = query_influxql($influxdb_client, qq{SHOW RETENTION POLICIES ON "$db"});
$db_schemas_in_influxdb{$db} = {
map { $_->[0] => {
duration => $_->[1],
Expand All @@ -369,13 +369,11 @@ sub load_db_schemas_in_influxdb {
# $schema_dir string: the directory name where the config files are
#
# Returns:
# a reference to a hash holding the parsed statements. This hash is structured as below.
# $db_schemas: a reference to a hash holding the parsed statements. This hash is structured as below.
#
# {
# <db> => {
# create_query => '...',
# rps => {
# <rp> => {
# <rp> => {
# duration => ...,
# shard_duration => ...,
# default => ...,
Expand All @@ -389,21 +387,39 @@ sub load_db_schemas_in_influxdb {
sub load_db_schemas_in_config {
my ($schema_dir) = @_;

# this will be populated with all statements parsed from the files
my %parsed_statements;
my %db_schemas;
my $db_files = get_schema_files_for_dir("$schema_dir/db");

for my $db_file (@$db_files) {
my $statements_in_file = read_text("$schema_dir/db/$db_file");
my $parsed_statements_from_file = parse_statements($statements_in_file);

# add the statements from this file to the initial hash
%parsed_statements = (%parsed_statements, %$parsed_statements_from_file);
my ($databases, $rps) = parse_statements($statements_in_file);

# loop all the databases and add them to the hash
for my $db (@$databases) {
# make sure every database is only created once
if (exists $db_schemas{$db}) {
die "duplicate database $db in file $db_file detected\n";
}
$db_schemas{$db} = {};
}
# loop all the retention policies and assign them to the correct database
for my $rp (@$rps) {
my $db = $rp->{database};
if (exists $db_schemas{$db}) {
# make sure every retention policy is only created once
if (exists $db_schemas{$db}->{$rp->{rp_name}}) {
die "duplicate rp $rp on db $db in file $db_file detected\n";
}
$db_schemas{$db}->{$rp->{rp_name}} = $rp;
}
else {
die "database $db specified in rp $rp from file $db_file does not exist\n";
}
}
}
return \%parsed_statements;
return \%db_schemas;
}


#
# Iterates over the lines of a config file, parses the valid statements and adds them to the given hash.
# Valid statements are:
Expand All @@ -412,19 +428,20 @@ sub load_db_schemas_in_config {
#
# Arguments:
# $string_to_parse string: the contents of the config file (loaded as string)
# $parsed_statements reference: a reference to a hash. The function will populate it with the parsed statements.
#
# Returns:
# a reference to a hash holding the parsed statements from the file.
# $databases: reference to a list which contains all the database names
# $rps: reference to a list which contains hashes of all the retention policies
#
sub parse_statements {
my ($string_to_parse) = @_;
# we want to iterate line-by-line
my @splitted_lines = split "\n", $string_to_parse;
my %parsed_statements;
my @databases;
my @rps;

# captures a 'create database' statement with an optional retention policy definition
my $db_regex = qr/^\s*(create \s+ database \s+ (\w+)) # a create db statement, optionally starting with a space. Any valid word as db name. Captured group.
my $db_regex = qr/^\s*(create \s+ database \s+ "?([\w.]+)"?) # a create db statement, optionally starting with a space. Any valid word as db name. Captured group.
(
\s+ with \s+ duration # optional statement to define a retention policy in the db creation statement
\s+ ((?:\d+[smhdw])|(?:inf)) # captured policy duration as one or more numbers followed by a letter, or 'inf'
Expand Down Expand Up @@ -453,7 +470,7 @@ sub parse_statements {
for my $line (@splitted_lines) {
# ignore empty and commented lines
next if $line =~ /^\s*(#.*)?$/;

if ($line =~ /$integrated_regex/) {
# capture groups from the regex
my $create_db_statement = $1;
Expand All @@ -472,23 +489,30 @@ sub parse_statements {

# the line is a 'create database' statement...
if ($create_db_statement) {
check_if_db_statement_exists_and_die(\%parsed_statements, $db_name);
$parsed_statements{$db_name}{create_query} = $create_db_statement . ';';
push @databases, $db_name;
# ...and has a retention policy defined inline
if ($inline_rp_statement) {
# flagged as the default policy (assume that the inline policy is the default)
$parsed_statements{$db_name}{rps}{$inline_rp_name} = {duration => $inline_rp_duration,
shard_duration => $inline_rp_shard_duration,
rp_replication => $inline_rp_replication,
default => 1};
# flagged as the default policy (assume that the inline policy is the default)ƒ
push @rps, {
database => $db_name,
rp_name => $inline_rp_name,
duration => $inline_rp_duration,
shard_duration => $inline_rp_shard_duration,
rp_replication => $inline_rp_replication,
default => 1,
};
}
}
# the line is a 'create retention policy' statement
elsif ($rp_name) {
$parsed_statements{$rp_db_name}{rps}{$rp_name} = {duration => $rp_duration,
shard_duration => $rp_shard_duration,
rp_replication => $rp_replication,
default => $default_rp}
push @rps, {
database => $rp_db_name,
rp_name => $rp_name,
duration => $rp_duration,
shard_duration => $rp_shard_duration,
rp_replication => $rp_replication,
default => $default_rp,
}
}
else {
die "error, should never reach this"
Expand All @@ -498,29 +522,9 @@ sub parse_statements {
die "could not parse input: $line";
}
}
return \%parsed_statements;
return return \@databases, \@rps;
}


#
# Checks if the db creation statement is already added in the parsed statements hash. If yes then it throws an exception.
#
# Arguments:
# $parsed_statements reference: a reference to the hash containing the db statements
# $db_name string:
# Returns:
# -
#
sub check_if_db_statement_exists_and_die {
my ($parsed_statements, $db_name) = @_;

# there should be no 'create database' statement yet for this database
if ($parsed_statements->{$db_name}{create_query}) {
die "Create statement already defined for database $db_name. Fix the config file";
}
}


# {
# <db> => {
# <cq_name> => <cq_create_query>,
Expand Down
2 changes: 2 additions & 0 deletions t/data/test_name_with_dot/cq/db1.ifql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
CREATE CONTINUOUS QUERY "cq.test" ON "db.test" RESAMPLE EVERY 5m FOR 10m BEGIN SELECT LAST(a) AS b, c INTO "rp.test"."m.test" FROM "rp.test"."m.test" GROUP BY time(5m) END;

1 change: 1 addition & 0 deletions t/data/test_name_with_dot/db/db1.ifql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
CREATE DATABASE "db.test" WITH DURATION 260w REPLICATION 1 SHARD DURATION 12w NAME "rp.test";
Loading

0 comments on commit 18901af

Please sign in to comment.