Merge pull request #319 from hiroyuki-sato/topic/add-load-tests

Add {before|after}_load tests in the postgresql plugin

embulk-output-postgresql/src/test/java/org/embulk/output/postgresql/AfterLoadTest.java

@@ -0,0 +1,150 @@
+package org.embulk.output.postgresql;
+
+import static org.embulk.output.postgresql.PostgreSQLTests.execute;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+import java.util.Arrays;
+
+import org.embulk.config.ConfigSource;
+import org.embulk.formatter.csv.CsvFormatterPlugin;
+import org.embulk.input.file.LocalFileInputPlugin;
+import org.embulk.output.PostgreSQLOutputPlugin;
+import org.embulk.output.file.LocalFileOutputPlugin;
+import org.embulk.parser.csv.CsvParserPlugin;
+import org.embulk.spi.FileInputPlugin;
+import org.embulk.spi.FileOutputPlugin;
+import org.embulk.spi.FormatterPlugin;
+import org.embulk.spi.OutputPlugin;
+import org.embulk.spi.ParserPlugin;
+import org.embulk.test.EmbulkTests;
+import org.embulk.test.TestingEmbulk;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+public class AfterLoadTest
+{
+    private static final String BASIC_RESOURCE_PATH = "/org/embulk/output/postgresql/test/expect/after_load/";
+
+    private static ConfigSource loadYamlResource(TestingEmbulk embulk, String fileName)
+    {
+        return embulk.loadYamlResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    private static String readResource(String fileName)
+    {
+        return EmbulkTests.readResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    @Rule
+    public TestingEmbulk embulk = TestingEmbulk.builder()
+            .registerPlugin(FileInputPlugin.class, "file", LocalFileInputPlugin.class)
+            .registerPlugin(ParserPlugin.class, "csv", CsvParserPlugin.class)
+            .registerPlugin(FormatterPlugin.class, "csv", CsvFormatterPlugin.class)
+            .registerPlugin(FileOutputPlugin.class, "file", LocalFileOutputPlugin.class)
+            .registerPlugin(OutputPlugin.class, "postgresql", PostgreSQLOutputPlugin.class)
+            .build();
+
+    private ConfigSource baseConfig;
+
+    @Before
+    public void setup()
+    {
+        baseConfig = PostgreSQLTests.baseConfig();
+        execute(readResource("setup.sql")); // setup rows
+    }
+
+    @Test
+    public void testInsertAfterLoad() throws Exception
+    {
+        execute("insert into test1 values('B001', 0, 'z')");
+        execute("insert into test1 values('B002', 9, 'z')");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_insert_after_load.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test_insert_after_load_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testInsertDirectAfterLoad() throws Exception
+    {
+        execute("insert into test1 values('B001', 0, 'z')");
+        execute("insert into test1 values('B002', 9, 'z')");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_insert_direct_after_load.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test_insert_after_load_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testTruncateInsertAfterLoad() throws Exception
+    {
+        execute("insert into test1 values('B001', 0, 'z')");
+        execute("insert into test1 values('B002', 9, 'z')");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_replace_after_load.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test_replace_after_load_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testReplaceAfterLoad() throws Exception
+    {
+        execute("insert into test1 values('B001', 0, 'z')");
+        execute("insert into test1 values('B002', 9, 'z')");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_truncate_insert_after_load.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test_truncate_insert_after_load_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testMergeAfterLoad() throws Exception
+    {
+        execute("insert into test1 values('A002', 1, 'y')");
+        execute("insert into test1 values('A003', 1, 'y')");
+        execute("insert into test1 values('B001', 0, 'z')");
+        execute("insert into test1 values('B002', 9, 'z')");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_merge_after_load.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test_merge_after_load_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testMergeDirectAfterLoad() throws Exception
+    {
+        execute("insert into test1 values('A002', 1, 'y')");
+        execute("insert into test1 values('A003', 1, 'y')");
+        execute("insert into test1 values('B001', 0, 'z')");
+        execute("insert into test1 values('B002', 9, 'z')");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_merge_direct_after_load.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test_merge_after_load_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    private Path toPath(String fileName) throws URISyntaxException
+    {
+        URL url = EmbulkTests.class.getResource(BASIC_RESOURCE_PATH + fileName);
+        return FileSystems.getDefault().getPath(new File(url.toURI()).getAbsolutePath());
+    }
+
+    private String selectRecords(TestingEmbulk embulk, String tableName) throws IOException
+    {
+        return PostgreSQLTests.selectRecords(embulk, tableName, Arrays.asList("id", "int_item", "varchar_item"));
+    }
+}

embulk-output-postgresql/src/test/java/org/embulk/output/postgresql/BeforeLoadTest.java

@@ -0,0 +1,138 @@
+package org.embulk.output.postgresql;
+
+import static org.embulk.output.postgresql.PostgreSQLTests.execute;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+import java.util.Arrays;
+
+import org.embulk.config.ConfigSource;
+import org.embulk.formatter.csv.CsvFormatterPlugin;
+import org.embulk.input.file.LocalFileInputPlugin;
+import org.embulk.output.PostgreSQLOutputPlugin;
+import org.embulk.output.file.LocalFileOutputPlugin;
+import org.embulk.parser.csv.CsvParserPlugin;
+import org.embulk.spi.FileInputPlugin;
+import org.embulk.spi.FileOutputPlugin;
+import org.embulk.spi.FormatterPlugin;
+import org.embulk.spi.OutputPlugin;
+import org.embulk.spi.ParserPlugin;
+import org.embulk.test.EmbulkTests;
+import org.embulk.test.TestingEmbulk;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+public class BeforeLoadTest
+{
+    private static final String BASIC_RESOURCE_PATH = "/org/embulk/output/postgresql/test/expect/before_load/";
+
+    private static ConfigSource loadYamlResource(TestingEmbulk embulk, String fileName)
+    {
+        return embulk.loadYamlResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    private static String readResource(String fileName)
+    {
+        return EmbulkTests.readResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    @Rule
+    public TestingEmbulk embulk = TestingEmbulk.builder()
+            .registerPlugin(FileInputPlugin.class, "file", LocalFileInputPlugin.class)
+            .registerPlugin(ParserPlugin.class, "csv", CsvParserPlugin.class)
+            .registerPlugin(FormatterPlugin.class, "csv", CsvFormatterPlugin.class)
+            .registerPlugin(FileOutputPlugin.class, "file", LocalFileOutputPlugin.class)
+            .registerPlugin(OutputPlugin.class, "postgresql", PostgreSQLOutputPlugin.class)
+            .build();
+
+    private ConfigSource baseConfig;
+
+    @Before
+    public void setup()
+    {
+        baseConfig = PostgreSQLTests.baseConfig();
+        execute(readResource("setup.sql")); // setup rows
+    }
+
+    @Test
+    public void testInsertBeforeLoad() throws Exception
+    {
+        execute("insert into test1 values('B001', 0, 'z')");
+        execute("insert into test1 values('B002', 9, 'z')");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_insert_before_load.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test_insert_before_load_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testInsertDirectBeforeLoad() throws Exception
+    {
+        execute("insert into test1 values('B001', 0, 'z')");
+        execute("insert into test1 values('B002', 9, 'z')");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_insert_direct_before_load.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test_insert_before_load_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testTruncateInsertBeforeLoad() throws Exception
+    {
+        execute("insert into test1 values('B001', 0, 'z')");
+        execute("insert into test1 values('B002', 9, 'z')");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_truncate_insert_before_load.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test_truncate_insert_before_load_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testMergeBeforeLoad() throws Exception
+    {
+        execute("insert into test1 values('A002', 1, 'y')");
+        execute("insert into test1 values('A003', 1, 'y')");
+        execute("insert into test1 values('B001', 0, 'z')");
+        execute("insert into test1 values('B002', 9, 'z')");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_merge_before_load.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test_merge_before_load_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testMergeDirectBeforeLoad() throws Exception
+    {
+        execute("insert into test1 values('A002', 1, 'y')");
+        execute("insert into test1 values('A003', 1, 'y')");
+        execute("insert into test1 values('B001', 0, 'z')");
+        execute("insert into test1 values('B002', 9, 'z')");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_merge_direct_before_load.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test_merge_before_load_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    private Path toPath(String fileName) throws URISyntaxException
+    {
+        URL url = EmbulkTests.class.getResource(BASIC_RESOURCE_PATH + fileName);
+        return FileSystems.getDefault().getPath(new File(url.toURI()).getAbsolutePath());
+    }
+
+    private String selectRecords(TestingEmbulk embulk, String tableName) throws IOException
+    {
+        return PostgreSQLTests.selectRecords(embulk, tableName, Arrays.asList("id", "int_item", "varchar_item"));
+    }
+}

embulk-output-postgresql/src/test/java/org/embulk/output/postgresql/PostgreSQLTests.java

@@ -8,10 +8,13 @@
 import java.io.OutputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.List;
+import java.util.stream.Collectors;
 
 import org.embulk.config.ConfigSource;
 
 import static java.util.Locale.ENGLISH;
+import static java.util.stream.Collectors.joining;
 import static org.embulk.test.EmbulkTests.readSortedFile;
 
 public class PostgreSQLTests
@@ -60,4 +63,14 @@ public static String selectRecords(TestingEmbulk embulk, String tableName) throw
         execute("\\copy " + tableName + " to '" + temp.toString().replace("\\", "\\\\") + "' delimiter ','");
         return readSortedFile(temp);
     }
+
+    public static String selectRecords(TestingEmbulk embulk, String tableName, List<String> columnList) throws IOException
+    {
+        Path temp = embulk.createTempFile("txt");
+        Files.delete(temp);
+        final String cols = columnList.stream().collect(Collectors.joining(","));
+        execute(String.format("\\COPY (SELECT %s FROM %s) TO '%s' With CSV DELIMITER ',';", cols,  tableName, temp.toString().replace("\\", "\\\\")));
+        return readSortedFile(temp);
+    }
+
 }

embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/after_load/setup.sql

@@ -0,0 +1,8 @@
+drop table if exists test1;
+
+create table test1 (
+    id              char(4),
+    int_item        int,
+    varchar_item    varchar(8),
+    primary key (id)
+);

embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/after_load/test1.csv

@@ -0,0 +1,4 @@
+id:string,int_item:long,varchar_item:string
+A001,9,a
+A002,0,b
+A003,9,c

embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/after_load/test_expected.diff

@@ -0,0 +1,2 @@
+in: {}
+out: {}

embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/after_load/test_insert_after_load.yml

@@ -0,0 +1,3 @@
+table: test1
+mode: insert
+after_load: "update test1 set varchar_item = 'x' where int_item = 9"

embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/after_load/test_insert_after_load_expected.csv

@@ -0,0 +1,5 @@
+A001,9,x
+A002,0,b
+A003,9,x
+B001,0,z
+B002,9,x

embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/after_load/test_insert_direct_after_load.yml

@@ -0,0 +1,3 @@
+table: test1
+mode: insert_direct
+after_load: "update test1 set varchar_item = 'x' where int_item = 9"

embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/after_load/test_merge_after_load.yml

@@ -0,0 +1,3 @@
+table: test1
+mode: merge
+after_load: "update test1 set varchar_item = 'x' where int_item = 9"

embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/after_load/test_merge_after_load_expected.csv

@@ -0,0 +1,5 @@
+A001,9,x
+A002,0,b
+A003,9,x
+B001,0,z
+B002,9,x

embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/after_load/test_merge_direct_after_load.yml

@@ -0,0 +1,3 @@
+table: test1
+mode: merge_direct
+after_load: "update test1 set varchar_item = 'x' where int_item = 9"

embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/after_load/test_replace_after_load.yml

@@ -0,0 +1,3 @@
+table: test1
+mode: replace
+after_load: "update test1 set varchar_item = 'x' where int_item = 9"

embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/after_load/test_replace_after_load_expected.csv

@@ -0,0 +1,3 @@
+A001,9,x
+A002,0,b
+A003,9,x

embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/after_load/test_truncate_insert_after_load.yml

@@ -0,0 +1,3 @@
+table: test1
+mode: truncate_insert
+after_load: "update test1 set varchar_item = 'x' where int_item = 9"

embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/after_load/test_truncate_insert_after_load_expected.csv

@@ -0,0 +1,3 @@
+A001,9,x
+A002,0,b
+A003,9,x

embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/before_load/setup.sql

@@ -0,0 +1,8 @@
+drop table if exists test1;
+
+create table test1 (
+    id              char(4),
+    int_item        int,
+    varchar_item    varchar(8),
+    primary key (id)
+);

embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/before_load/test1.csv

@@ -0,0 +1,4 @@
+id:string,int_item:long,varchar_item:string
+A001,9,a
+A002,0,b
+A003,9,c

embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/before_load/test_expected.diff

@@ -0,0 +1,2 @@
+in: {}
+out: {}