Skip to content

Commit

Permalink
Merge pull request #8372 from mandy-chessell/oak2024
Browse files Browse the repository at this point in the history
Changes to Egeria's namespace name
  • Loading branch information
mandy-chessell authored Sep 3, 2024
2 parents 5580a37 + 35ac5e6 commit 57a4747
Show file tree
Hide file tree
Showing 11 changed files with 239 additions and 23 deletions.
2 changes: 1 addition & 1 deletion CoreContentPackGUIDMap.json

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion content-packs/CoreContentPack.omarchive

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
# openLineage requests

#@openLineageBackend=http://localhost:5000
@openLineageBackend=https://localhost:9443/servers/integration-daemon/open-metadata/integration-services/lineage-integrator/users/airflownpa
@openLineageBackend=http://egeria.pdr-associates.com:5050
#@openLineageBackend=https://localhost:9443/servers/integration-daemon/open-metadata/integration-services/lineage-integrator/users/airflownpa
@openLineageNamespace=restTestNameSpace
@openLineageRunId=001

Expand Down Expand Up @@ -320,15 +321,15 @@ Content-Type: application/json
"producer": "https://egeria-project.org/",
"schemaURL": "https://openlineage.io/spec/1-0-2/OpenLineage.json#/$defs/RunEvent",
"eventType": "COMPLETE",
"eventTime": "2021-11-05T10:28:18.969066Z[Europe/London]",
"eventTime": "{{$isoTimestamp}}",
"run": {
"runId": "ce649429-2b5d-4897-899d-0e3a0c1fef44",
"facets": {
"nominalTime": {
"_producer": "https://egeria-project.org/",
"_schemaURL": "https://openlineage.io/spec/facets/1-0-0/NominalTimeRunFacet.json#/$defs/NominalTimeRunFacet",
"nominalStartTime": "2021-11-05T10:28:10.076Z[Europe/London]",
"nominalEndTime": "2021-11-05T10:28:13.502Z[Europe/London]"
"nominalStartTime": "{{$isoTimestamp}}",
"nominalEndTime": "{{$isoTimestamp}}"
},
"additionalProperties": {}
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
/* Copyright Contributors to the ODPi Egeria project. */
package org.odpi.openmetadata.adapters.connectors.integration.openlineage;

import org.odpi.openmetadata.adapters.connectors.integration.openlineage.ffdc.OpenLineageIntegrationConnectorAuditCode;
import org.odpi.openmetadata.adapters.connectors.integration.openlineage.ffdc.OpenLineageIntegrationConnectorErrorCode;
import org.odpi.openmetadata.frameworks.connectors.ffdc.ConnectorCheckedException;
import org.odpi.openmetadata.frameworks.connectors.ffdc.InvalidParameterException;
Expand Down Expand Up @@ -103,11 +104,24 @@ public void storeEvent(OpenLineageRunEvent openLineageEvent,
String rawEvent,
String logStoreURL) throws InvalidParameterException
{
final String methodName = "storeEvent";

if (rawEvent != null)
{
HttpEntity<?> request = new HttpEntity<>((Object)rawEvent, header);

restTemplate.exchange(logStoreURL, HttpMethod.POST, request, Void.class);
try
{
HttpEntity<?> request = new HttpEntity<>((Object) rawEvent, header);

restTemplate.exchange(logStoreURL, HttpMethod.POST, request, Void.class);
}
catch (Exception error)
{
auditLog.logMessage(methodName, OpenLineageIntegrationConnectorAuditCode.UNEXPECTED_EXCEPTION.getMessageDefinition(connectorName,
error.getClass().getName(),
methodName,
error.getMessage()),
logStoreURL);
}
}
else
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
import org.odpi.openmetadata.frameworks.auditlog.AuditLogReportingComponent;
import org.odpi.openmetadata.frameworks.auditlog.ComponentDevelopmentStatus;
import org.odpi.openmetadata.frameworks.connectors.properties.beans.ConnectorType;
import org.odpi.openmetadata.frameworks.openmetadata.refdata.DeployedImplementationType;
import org.odpi.openmetadata.frameworks.integration.connectors.IntegrationConnectorProvider;
import org.odpi.openmetadata.frameworks.openmetadata.refdata.DeployedImplementationType;

/**
* APIBasedOpenLineageLogStoreProvider is the OCF connector provider for the API based open lineage log store.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,10 @@
public class GovernanceActionOpenLineageIntegrationConnector extends LineageIntegratorConnector implements AssetManagerEventListener
{
private static final URI producer = URI.create("https://egeria-project.org/");
private static final String defaultNameSpace = "GovernanceActions";
private final ZoneId zoneId = ZoneId.systemDefault();

private String namespace = defaultNameSpace;

/**
* Default constructor
Expand All @@ -53,6 +55,13 @@ public synchronized void start() throws ConnectorCheckedException

final String methodName = "start";

namespace = super.getStringConfigurationProperty("namespace", connectionProperties.getConfigurationProperties());

if (namespace == null || namespace.isBlank())
{
namespace = defaultNameSpace;
}

try
{
super.getContext().registerListener(this);
Expand Down Expand Up @@ -191,18 +200,6 @@ else if (EngineActionStatus.INVALID.getName().equals(engineActionStatus))
event.setEventType("ABORT");
}

String namespace = engineAction.getProcessName();

/*
* This is a workaround for Marquez that limits its namespaces to 64 chars.
*/
if (namespace.length() > 64)
{
String[] tokens = namespace.split(":");

namespace = tokens[tokens.length - 1];
}

OpenLineageJob job = new OpenLineageJob();

if (engineAction.getProcessStepName() != null)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
/* Copyright Contributors to the ODPi Egeria project. */
package org.odpi.openmetadata.adapters.connectors.integration.openlineage;

import org.odpi.openmetadata.adapters.connectors.integration.openlineage.controls.OpenLineagePublishConfigurationProperty;
import org.odpi.openmetadata.frameworks.auditlog.AuditLogReportingComponent;
import org.odpi.openmetadata.frameworks.auditlog.ComponentDevelopmentStatus;
import org.odpi.openmetadata.frameworks.connectors.properties.beans.ConnectorType;
Expand Down Expand Up @@ -52,6 +53,8 @@ public GovernanceActionOpenLineageIntegrationProvider()
*/
super.setConnectorClassName(connectorClassName);

super.supportedConfigurationProperties = OpenLineagePublishConfigurationProperty.getConfigurationPropertyTypes();

/*
* Set up the connector type that should be included in a connection used to configure this connector.
*/
Expand All @@ -63,6 +66,7 @@ public GovernanceActionOpenLineageIntegrationProvider()
connectorType.setDescription(connectorDescription);
connectorType.setConnectorProviderClassName(this.getClass().getName());
connectorType.setSupportedAssetTypeName(supportedAssetTypeName);
connectorType.setRecognizedConfigurationProperties(OpenLineagePublishConfigurationProperty.getRecognizedConfigurationProperties());
connectorType.setSupportedDeployedImplementationType(DeployedImplementationType.LINEAGE_INTEGRATION_CONNECTOR.getDeployedImplementationType());

super.connectorTypeBean = connectorType;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,177 @@
/* SPDX-License-Identifier: Apache-2.0 */
/* Copyright Contributors to the ODPi Egeria project. */

package org.odpi.openmetadata.adapters.connectors.integration.openlineage.controls;

import org.odpi.openmetadata.frameworks.connectors.controls.ConfigurationPropertyType;

import java.util.ArrayList;
import java.util.List;

/**
* OMAGServerPlatformConfigurationProperty defines the configuration properties used with the OMAG Server Platform connectors.
*/
public enum OpenLineagePublishConfigurationProperty
{
/**
* Namespace is used to group open lineage events together.
*/
NAMESPACE ("namespace",
"Namespace is used to group open lineage events together.",
"string",
"GovernanceActions",
false),

;

public final String name;
public final String description;
public final String dataType;
public final String example;
public final boolean isPlaceholder;


/**
* Create a specific Enum constant.
*
* @param name name of the request parameter
* @param description description of the request parameter
* @param dataType type of value of the request parameter
* @param example example of the request parameter
* @param isPlaceholder is this also used as a placeholder property?
*/
OpenLineagePublishConfigurationProperty(String name,
String description,
String dataType,
String example,
boolean isPlaceholder)
{
this.name = name;
this.description = description;
this.dataType = dataType;
this.example = example;
this.isPlaceholder = isPlaceholder;
}


/**
* Return the name of the request parameter.
*
* @return string name
*/
public String getName()
{
return name;
}


/**
* Return the description of the configuration property.
*
* @return text
*/
public String getDescription()
{
return description;
}


/**
* Return the data type for the configuration property.
*
* @return data type name
*/
public String getDataType()
{
return dataType;
}


/**
* Return an example of the configuration property to help users understand how to set it up.
*
* @return example
*/
public String getExample()
{
return example;
}


/**
* Return whether this value is also used as a placeholder property.
*
* @return boolean
*/
public boolean isPlaceholder()
{
return isPlaceholder;
}



/**
* Get recognizedConfigurationProperties for the connector.
*
* @return list of property names
*/
public static List<String> getRecognizedConfigurationProperties()
{
List<String> recognizedConfigurationProperties = new ArrayList<>();

for (OpenLineagePublishConfigurationProperty configurationProperty : OpenLineagePublishConfigurationProperty.values())
{
recognizedConfigurationProperties.add(configurationProperty.getName());
}
return recognizedConfigurationProperties;
}


/**
* Retrieve all the defined configuration properties
*
* @return list of configuration property types
*/
public static List<ConfigurationPropertyType> getConfigurationPropertyTypes()
{
List<ConfigurationPropertyType> configurationPropertyTypes = new ArrayList<>();

for (OpenLineagePublishConfigurationProperty configurationProperty : OpenLineagePublishConfigurationProperty.values())
{
configurationPropertyTypes.add(configurationProperty.getConfigurationPropertyType());
}

return configurationPropertyTypes;
}


/**
* Return a summary of this enum to use in a connector provider.
*
* @return request parameter type
*/
public ConfigurationPropertyType getConfigurationPropertyType()
{
ConfigurationPropertyType configurationPropertyType = new ConfigurationPropertyType();

configurationPropertyType.setName(name);
configurationPropertyType.setDescription(description);
configurationPropertyType.setDataType(dataType);
configurationPropertyType.setExample(example);
configurationPropertyType.setRequired(isPlaceholder);

return configurationPropertyType;
}


/**
* Output of this enum class and main value.
*
* @return string showing enum value
*/
@Override
public String toString()
{
return "ConfigurationProperty{ name=" + name + "}";
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ public enum PlaceholderProperty
* The network address of the server
*/
SERVER_NETWORK_ADDRESS("serverNetworkAddress", "The network address of the server.", "string", "http://localhost:8080"),
API_OPERATION("apiOperation", "The name of the operation to append onto the server host name/port number, or set it to an empty string if not needed.", "string", "/api/v1/lineage"),

/**
* The userId to store in the userId attribute of the connection.
Expand Down Expand Up @@ -312,6 +313,7 @@ public static List<PlaceholderPropertyType> getHTTPEndpointPlaceholderPropertyTy

placeholderPropertyTypes.add(PlaceholderProperty.HOST_URL.getPlaceholderType());
placeholderPropertyTypes.add(PlaceholderProperty.PORT_NUMBER.getPlaceholderType());
placeholderPropertyTypes.add(PlaceholderProperty.API_OPERATION.getPlaceholderType());
placeholderPropertyTypes.add(PlaceholderProperty.SERVER_NAME.getPlaceholderType());
placeholderPropertyTypes.add(PlaceholderProperty.DESCRIPTION.getPlaceholderType());

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -301,6 +301,27 @@ Content-Type: application/json
}


###
# @name createElementFromTemplate - Marquez endpoint
# Create a new element from a template.
# Further Information: https://egeria-project.org/features/templated-cataloguing/overview/
POST {{baseURL}}/servers/{{viewServer}}/api/open-metadata/automated-curation/catalog-templates/new-element
Authorization: Bearer {{token}}
Content-Type: application/json

{
"templateGUID" : "9ea4bff4-d193-492f-bcad-6e68c07c6f9e",
"isOwnAnchor" : true,
"placeholderPropertyValues" : {
"description" : "Link to Marquez",
"portNumber" : "5050",
"serverName" : "Marquez",
"hostURL" : "http://egeria.pdr-associates.com",
"apiOperation" : "/api/v1/lineage"
}
}


###
# @name createElementFromTemplate (Apache Kafka Server)
# Create a new element from a template.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public enum EndpointTemplateDefinition implements TemplateDefinition
REST_ENDPOINT_TEMPLATE("9ea4bff4-d193-492f-bcad-6e68c07c6f9e",
DeployedImplementationType.REST_API_ENDPOINT,
"JDBC",
PlaceholderProperty.HOST_URL.getPlaceholder() + ":" + PlaceholderProperty.PORT_NUMBER.getPlaceholder(),
PlaceholderProperty.HOST_URL.getPlaceholder() + ":" + PlaceholderProperty.PORT_NUMBER.getPlaceholder() + PlaceholderProperty.API_OPERATION.getPlaceholder(),
PlaceholderProperty.getHTTPEndpointPlaceholderPropertyTypes()),


Expand Down

0 comments on commit 57a4747

Please sign in to comment.