Skip to content

Commit

Permalink
Updated the OpenSearch sink to use the org.opensearch.dataprepper pac…
Browse files Browse the repository at this point in the history
…kage name. Contributes to opensearch-project#344. (opensearch-project#1780)

Signed-off-by: David Venable <[email protected]>
  • Loading branch information
dlvenable authored Sep 19, 2022
1 parent 8a341f6 commit 48d8bc3
Show file tree
Hide file tree
Showing 58 changed files with 103 additions and 103 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ jobs:
sleep 90
- name: Run Open Distro tests
run: |
./gradlew :data-prepper-plugins:opensearch:integrationTest --tests "com.amazon.dataprepper.plugins.sink.opensearch.OpenSearchIT" -Dtests.opensearch.host=localhost:9200 -Dtests.opensearch.user=admin -Dtests.opensearch.password=admin
./gradlew :data-prepper-plugins:opensearch:integrationTest --tests "org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchIT" -Dtests.opensearch.host=localhost:9200 -Dtests.opensearch.user=admin -Dtests.opensearch.password=admin
./gradlew :data-prepper-plugins:opensearch:integrationTest -Dtests.opensearch.host=localhost:9200 -Dtests.opensearch.user=admin -Dtests.opensearch.password=admin -Dtests.opensearch.bundle=true
- name: Upload Unit Test Results
if: always()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ jobs:
sleep 90
- name: Run OpenSearch tests
run: |
./gradlew :data-prepper-plugins:opensearch:integrationTest --tests "com.amazon.dataprepper.plugins.sink.opensearch.OpenSearchIT" -Dtests.opensearch.host=localhost:9200 -Dtests.opensearch.user=admin -Dtests.opensearch.password=admin
./gradlew :data-prepper-plugins:opensearch:integrationTest --tests "org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchIT" -Dtests.opensearch.host=localhost:9200 -Dtests.opensearch.user=admin -Dtests.opensearch.password=admin
./gradlew :data-prepper-plugins:opensearch:integrationTest -Dtests.opensearch.host=localhost:9200 -Dtests.opensearch.user=admin -Dtests.opensearch.password=admin -Dtests.opensearch.bundle=true
- name: Upload Unit Test Results
if: always()
Expand Down
4 changes: 2 additions & 2 deletions data-prepper-plugins/opensearch/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -117,8 +117,8 @@ jacocoTestCoverageVerification {
element = 'CLASS'
excludes = [
// TODO: find a way to include the classes covered by OpenSearch test framework
'com.amazon.dataprepper.plugins.sink.opensearch.OpenSearchSink',
'com.amazon.dataprepper.plugins.sink.opensearch.IndexStateManagement'
'org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchSink',
'org.opensearch.dataprepper.plugins.sink.opensearch.IndexStateManagement'
]
limit {
minimum = 0.90
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* SPDX-License-Identifier: Apache-2.0
*/

package com.amazon.dataprepper.plugins.sink.opensearch;
package org.opensearch.dataprepper.plugins.sink.opensearch;

import org.junit.Test;
import org.opensearch.client.Request;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
* under the License.
*/

package com.amazon.dataprepper.plugins.sink.opensearch;
package org.opensearch.dataprepper.plugins.sink.opensearch;

import org.apache.http.Header;
import org.apache.http.HttpHost;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* SPDX-License-Identifier: Apache-2.0
*/

package com.amazon.dataprepper.plugins.sink.opensearch;
package org.opensearch.dataprepper.plugins.sink.opensearch;

import org.opensearch.client.Request;
import org.opensearch.client.Response;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* SPDX-License-Identifier: Apache-2.0
*/

package com.amazon.dataprepper.plugins.sink.opensearch;
package org.opensearch.dataprepper.plugins.sink.opensearch;

import com.amazon.dataprepper.metrics.MetricNames;
import com.amazon.dataprepper.metrics.MetricsTestUtil;
Expand All @@ -12,11 +12,6 @@
import com.amazon.dataprepper.model.event.EventType;
import com.amazon.dataprepper.model.event.JacksonEvent;
import com.amazon.dataprepper.model.record.Record;
import com.amazon.dataprepper.plugins.sink.opensearch.bulk.BulkAction;
import com.amazon.dataprepper.plugins.sink.opensearch.index.IndexConfiguration;
import com.amazon.dataprepper.plugins.sink.opensearch.index.IndexConstants;
import com.amazon.dataprepper.plugins.sink.opensearch.index.IndexManager;
import com.amazon.dataprepper.plugins.sink.opensearch.index.IndexType;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
Expand All @@ -35,6 +30,11 @@
import org.opensearch.common.Strings;
import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.BulkAction;
import org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration;
import org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConstants;
import org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexManager;
import org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexType;

import javax.ws.rs.HttpMethod;
import java.io.BufferedReader;
Expand All @@ -57,12 +57,6 @@
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

import static com.amazon.dataprepper.plugins.sink.opensearch.OpenSearchIntegrationHelper.createContentParser;
import static com.amazon.dataprepper.plugins.sink.opensearch.OpenSearchIntegrationHelper.createOpenSearchClient;
import static com.amazon.dataprepper.plugins.sink.opensearch.OpenSearchIntegrationHelper.getHosts;
import static com.amazon.dataprepper.plugins.sink.opensearch.OpenSearchIntegrationHelper.isOSBundle;
import static com.amazon.dataprepper.plugins.sink.opensearch.OpenSearchIntegrationHelper.waitForClusterStateUpdatesToFinish;
import static com.amazon.dataprepper.plugins.sink.opensearch.OpenSearchIntegrationHelper.wipeAllTemplates;
import static org.apache.http.HttpStatus.SC_OK;
import static org.awaitility.Awaitility.await;
import static org.hamcrest.CoreMatchers.containsString;
Expand All @@ -71,6 +65,12 @@
import static org.hamcrest.CoreMatchers.notNullValue;
import static org.hamcrest.CoreMatchers.nullValue;
import static org.hamcrest.Matchers.closeTo;
import static org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchIntegrationHelper.createContentParser;
import static org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchIntegrationHelper.createOpenSearchClient;
import static org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchIntegrationHelper.getHosts;
import static org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchIntegrationHelper.isOSBundle;
import static org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchIntegrationHelper.waitForClusterStateUpdatesToFinish;
import static org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchIntegrationHelper.wipeAllTemplates;

public class OpenSearchSinkIT {
private static final String PLUGIN_NAME = "opensearch";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
* and limitations under the License.
*/
package com.amazon.dataprepper.plugins.sink.opensearch;
package org.opensearch.dataprepper.plugins.sink.opensearch;

import org.apache.http.Header;
import org.apache.http.HttpEntityEnclosingRequest;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* SPDX-License-Identifier: Apache-2.0
*/

package com.amazon.dataprepper.plugins.sink.opensearch;
package org.opensearch.dataprepper.plugins.sink.opensearch;

import org.opensearch.common.unit.TimeValue;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,9 @@
* SPDX-License-Identifier: Apache-2.0
*/

package com.amazon.dataprepper.plugins.sink.opensearch;
package org.opensearch.dataprepper.plugins.sink.opensearch;

import com.amazon.dataprepper.metrics.PluginMetrics;
import com.amazon.dataprepper.plugins.sink.opensearch.bulk.AccumulatingBulkRequest;
import io.micrometer.core.instrument.Counter;
import org.opensearch.OpenSearchException;
import org.opensearch.action.bulk.BackoffPolicy;
Expand All @@ -16,6 +15,7 @@
import org.opensearch.client.opensearch.core.bulk.BulkOperation;
import org.opensearch.client.opensearch.core.bulk.BulkResponseItem;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.AccumulatingBulkRequest;
import org.opensearch.rest.RestStatus;

import java.io.IOException;
Expand All @@ -26,7 +26,7 @@
import java.util.function.BiConsumer;
import java.util.function.Supplier;

import static com.amazon.dataprepper.plugins.sink.opensearch.ErrorCauseStringCreator.toSingleLineDisplayString;
import static org.opensearch.dataprepper.plugins.sink.opensearch.ErrorCauseStringCreator.toSingleLineDisplayString;

public final class BulkRetryStrategy {
public static final String DOCUMENTS_SUCCESS = "documentsSuccess";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* SPDX-License-Identifier: Apache-2.0
*/

package com.amazon.dataprepper.plugins.sink.opensearch;
package org.opensearch.dataprepper.plugins.sink.opensearch;

import com.amazon.dataprepper.model.configuration.PluginSetting;
import org.apache.http.HttpHost;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* SPDX-License-Identifier: Apache-2.0
*/

package com.amazon.dataprepper.plugins.sink.opensearch;
package org.opensearch.dataprepper.plugins.sink.opensearch;

import org.opensearch.client.opensearch._types.ErrorCause;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,23 +3,14 @@
* SPDX-License-Identifier: Apache-2.0
*/

package com.amazon.dataprepper.plugins.sink.opensearch;
package org.opensearch.dataprepper.plugins.sink.opensearch;

import com.amazon.dataprepper.model.annotations.DataPrepperPlugin;
import com.amazon.dataprepper.model.configuration.PluginSetting;
import com.amazon.dataprepper.model.event.Event;
import com.amazon.dataprepper.model.record.Record;
import com.amazon.dataprepper.model.sink.AbstractSink;
import com.amazon.dataprepper.model.sink.Sink;
import com.amazon.dataprepper.plugins.sink.opensearch.bulk.AccumulatingBulkRequest;
import com.amazon.dataprepper.plugins.sink.opensearch.bulk.BulkAction;
import com.amazon.dataprepper.plugins.sink.opensearch.bulk.BulkOperationWriter;
import com.amazon.dataprepper.plugins.sink.opensearch.bulk.JavaClientAccumulatingBulkRequest;
import com.amazon.dataprepper.plugins.sink.opensearch.bulk.PreSerializedJsonpMapper;
import com.amazon.dataprepper.plugins.sink.opensearch.bulk.SerializedJson;
import com.amazon.dataprepper.plugins.sink.opensearch.index.IndexManager;
import com.amazon.dataprepper.plugins.sink.opensearch.index.IndexManagerFactory;
import com.amazon.dataprepper.plugins.sink.opensearch.index.IndexType;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.DistributionSummary;
Expand All @@ -34,6 +25,15 @@
import org.opensearch.client.transport.OpenSearchTransport;
import org.opensearch.client.transport.rest_client.RestClientTransport;
import org.opensearch.common.unit.ByteSizeUnit;
import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.AccumulatingBulkRequest;
import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.BulkAction;
import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.BulkOperationWriter;
import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.JavaClientAccumulatingBulkRequest;
import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.PreSerializedJsonpMapper;
import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.SerializedJson;
import org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexManager;
import org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexManagerFactory;
import org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@
* SPDX-License-Identifier: Apache-2.0
*/

package com.amazon.dataprepper.plugins.sink.opensearch;
package org.opensearch.dataprepper.plugins.sink.opensearch;

import com.amazon.dataprepper.model.configuration.PluginSetting;
import com.amazon.dataprepper.plugins.sink.opensearch.index.IndexConfiguration;
import org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration;

import static com.google.common.base.Preconditions.checkNotNull;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* SPDX-License-Identifier: Apache-2.0
*/

package com.amazon.dataprepper.plugins.sink.opensearch;
package org.opensearch.dataprepper.plugins.sink.opensearch;

/**
* Represents a function that accepts one argument and produces a result or throws Exception.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* SPDX-License-Identifier: Apache-2.0
*/

package com.amazon.dataprepper.plugins.sink.opensearch;
package org.opensearch.dataprepper.plugins.sink.opensearch;

import com.amazon.dataprepper.model.configuration.PluginSetting;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* SPDX-License-Identifier: Apache-2.0
*/

package com.amazon.dataprepper.plugins.sink.opensearch.bulk;
package org.opensearch.dataprepper.plugins.sink.opensearch.bulk;

import java.util.List;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* SPDX-License-Identifier: Apache-2.0
*/

package com.amazon.dataprepper.plugins.sink.opensearch.bulk;
package org.opensearch.dataprepper.plugins.sink.opensearch.bulk;

import com.fasterxml.jackson.annotation.JsonCreator;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
* GitHub history for details.
*/

package com.amazon.dataprepper.plugins.sink.opensearch.bulk;
package org.opensearch.dataprepper.plugins.sink.opensearch.bulk;

import org.opensearch.client.opensearch.core.bulk.BulkOperation;
import org.opensearch.common.unit.ByteSizeValue;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* SPDX-License-Identifier: Apache-2.0
*/

package com.amazon.dataprepper.plugins.sink.opensearch.bulk;
package org.opensearch.dataprepper.plugins.sink.opensearch.bulk;

import org.opensearch.client.opensearch.core.BulkRequest;
import org.opensearch.client.opensearch.core.bulk.BulkOperation;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* SPDX-License-Identifier: Apache-2.0
*/

package com.amazon.dataprepper.plugins.sink.opensearch.bulk;
package org.opensearch.dataprepper.plugins.sink.opensearch.bulk;

import jakarta.json.spi.JsonProvider;
import jakarta.json.stream.JsonGenerator;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* SPDX-License-Identifier: Apache-2.0
*/

package com.amazon.dataprepper.plugins.sink.opensearch.bulk;
package org.opensearch.dataprepper.plugins.sink.opensearch.bulk;

import java.nio.charset.StandardCharsets;
import java.util.Objects;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* SPDX-License-Identifier: Apache-2.0
*/

package com.amazon.dataprepper.plugins.sink.opensearch.bulk;
package org.opensearch.dataprepper.plugins.sink.opensearch.bulk;

class SerializedJsonImpl implements SerializedJson {
private byte[] document;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* SPDX-License-Identifier: Apache-2.0
*/

package com.amazon.dataprepper.plugins.sink.opensearch.bulk;
package org.opensearch.dataprepper.plugins.sink.opensearch.bulk;

/**
* Represents an OpenSearch document with a serialized document size.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,13 @@
* SPDX-License-Identifier: Apache-2.0
*/

package com.amazon.dataprepper.plugins.sink.opensearch.index;
package org.opensearch.dataprepper.plugins.sink.opensearch.index;

import com.amazon.dataprepper.model.configuration.PluginSetting;
import com.amazon.dataprepper.plugins.sink.opensearch.bulk.BulkAction;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.lang3.EnumUtils;
import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.BulkAction;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* SPDX-License-Identifier: Apache-2.0
*/

package com.amazon.dataprepper.plugins.sink.opensearch.index;
package org.opensearch.dataprepper.plugins.sink.opensearch.index;

import java.util.HashMap;
import java.util.HashSet;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,8 @@
* SPDX-License-Identifier: Apache-2.0
*/

package com.amazon.dataprepper.plugins.sink.opensearch.index;
package org.opensearch.dataprepper.plugins.sink.opensearch.index;

import com.amazon.dataprepper.plugins.sink.opensearch.OpenSearchSinkConfiguration;
import com.google.common.collect.ImmutableSet;
import org.opensearch.OpenSearchException;
import org.opensearch.action.admin.cluster.settings.ClusterGetSettingsRequest;
Expand All @@ -18,6 +17,7 @@
import org.opensearch.client.indices.IndexTemplateMetadata;
import org.opensearch.client.indices.IndexTemplatesExistRequest;
import org.opensearch.client.indices.PutIndexTemplateRequest;
import org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchSinkConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@
* SPDX-License-Identifier: Apache-2.0
*/

package com.amazon.dataprepper.plugins.sink.opensearch.index;
package org.opensearch.dataprepper.plugins.sink.opensearch.index;

import com.amazon.dataprepper.plugins.sink.opensearch.OpenSearchSinkConfiguration;
import org.opensearch.client.RestHighLevelClient;
import org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchSinkConfiguration;

import java.util.Optional;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* SPDX-License-Identifier: Apache-2.0
*/

package com.amazon.dataprepper.plugins.sink.opensearch.index;
package org.opensearch.dataprepper.plugins.sink.opensearch.index;

import java.util.Arrays;
import java.util.HashMap;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* SPDX-License-Identifier: Apache-2.0
*/

package com.amazon.dataprepper.plugins.sink.opensearch.index;
package org.opensearch.dataprepper.plugins.sink.opensearch.index;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
Expand Down
Loading

0 comments on commit 48d8bc3

Please sign in to comment.