From cae8c4436df9064e54e5b6151797dcdb5f2ca614 Mon Sep 17 00:00:00 2001 From: Lula Villalobos Date: Thu, 6 May 2021 11:04:55 -0700 Subject: [PATCH] remove atlas metastore listener (#87) * remove atlas metastore listener * delete atlas from dockefile and remove atlas-application.properties * remove everything related to atlas metastore listener * update changelog * update changelog * fix changelog description * update date in Changelog --- CHANGELOG.md | 6 + Dockerfile | 15 -- README.md | 1 - files/atlas-2.0.0-hive-2.3.3.patch | 207 --------------------- files/atlas-application.properties | 288 ----------------------------- files/startup.sh | 15 +- 6 files changed, 7 insertions(+), 525 deletions(-) delete mode 100644 files/atlas-2.0.0-hive-2.3.3.patch delete mode 100644 files/atlas-application.properties diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ba685e..903a042 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [2.0.0] - 2021-05-06 +### Changed +- Remove Atlas MetaStore listener in favor of internal processes that subscribe to the Kafka HMS event listener and push changes to Ranger. + +Note: This release is a *BREAKING* change that removes all support for the Apache Atlas HMS listener. + ## [1.17.1] - 2020-11-18 ### Changed - Enabled ranger audit log summarization. diff --git a/Dockerfile b/Dockerfile index e586a18..2aa925f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,6 @@ ENV APIARY_GLUESYNC_LISTENER_VERSION 4.2.0 ENV APIARY_RANGER_PLUGIN_VERSION 5.0.1 ENV APIARY_METASTORE_METRICS_VERSION 4.2.0 ENV APIARY_METASTORE_AUTH_VERSION 4.2.0 -ENV ATLAS_VERSION 2.0.0 ENV KAFKA_VERSION 2.3.1 ENV COMMONS_CODEC_VERSION 1.12 ENV GETHOSTNAME4J_VERSION 0.0.3 @@ -40,9 +39,6 @@ wget -q https://search.maven.org/remotecontent?filepath=com/expediagroup/apiary/ wget -q https://search.maven.org/remotecontent?filepath=com/expediagroup/apiary/apiary-ranger-metastore-plugin/${APIARY_RANGER_PLUGIN_VERSION}/apiary-ranger-metastore-plugin-${APIARY_RANGER_PLUGIN_VERSION}-all.jar -O apiary-ranger-metastore-plugin-${APIARY_RANGER_PLUGIN_VERSION}-all.jar && \ wget -q https://search.maven.org/remotecontent?filepath=com/expediagroup/apiary/apiary-metastore-metrics/${APIARY_METASTORE_METRICS_VERSION}/apiary-metastore-metrics-${APIARY_METASTORE_METRICS_VERSION}-all.jar -O apiary-metastore-metrics-${APIARY_METASTORE_METRICS_VERSION}-all.jar && \ wget -q https://search.maven.org/remotecontent?filepath=com/expediagroup/apiary/apiary-metastore-auth/${APIARY_METASTORE_AUTH_VERSION}/apiary-metastore-auth-${APIARY_METASTORE_AUTH_VERSION}.jar -O apiary-metastore-auth-${APIARY_METASTORE_AUTH_VERSION}.jar && \ -wget -q https://search.maven.org/remotecontent?filepath=org/apache/atlas/atlas-notification/${ATLAS_VERSION}/atlas-notification-${ATLAS_VERSION}.jar -O atlas-notification-${ATLAS_VERSION}.jar && \ -wget -q https://search.maven.org/remotecontent?filepath=org/apache/atlas/atlas-intg/${ATLAS_VERSION}/atlas-intg-${ATLAS_VERSION}.jar -O atlas-intg-${ATLAS_VERSION}.jar && \ -wget -q https://search.maven.org/remotecontent?filepath=org/apache/atlas/atlas-common/${ATLAS_VERSION}/atlas-common-${ATLAS_VERSION}.jar -O atlas-common-${ATLAS_VERSION}.jar && \ wget -q https://search.maven.org/remotecontent?filepath=org/apache/kafka/kafka-clients/${KAFKA_VERSION}/kafka-clients-${KAFKA_VERSION}.jar -O kafka-clients-${KAFKA_VERSION}.jar && \ wget -q https://search.maven.org/remotecontent?filepath=commons-codec/commons-codec/${COMMONS_CODEC_VERSION}/commons-codec-${COMMONS_CODEC_VERSION}.jar -O commons-codec-${COMMONS_CODEC_VERSION}.jar && \ wget -q https://search.maven.org/remotecontent?filepath=com/kstruct/gethostname4j/${GETHOSTNAME4J_VERSION}/gethostname4j-${GETHOSTNAME4J_VERSION}.jar -O gethostname4j-${GETHOSTNAME4J_VERSION}.jar && \ @@ -55,22 +51,11 @@ ENV MAVEN_VERSION 3.6.3 RUN wget -q -O - http://www-us.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz|tar -C /opt -xzf - && \ ln -sf /opt/apache-maven-${MAVEN_VERSION}/bin/mvn /bin/mvn -COPY files/atlas-${ATLAS_VERSION}-hive-2.3.3.patch /tmp/atlas-${ATLAS_VERSION}-hive-2.3.3.patch -RUN cd /tmp && \ - wget -q https://www-us.apache.org/dist/atlas/${ATLAS_VERSION}/apache-atlas-${ATLAS_VERSION}-sources.tar.gz && \ - tar xfz apache-atlas-${ATLAS_VERSION}-sources.tar.gz && \ - cd apache-atlas-sources-${ATLAS_VERSION}/ && \ - patch -p1 < /tmp/atlas-${ATLAS_VERSION}-hive-2.3.3.patch && \ - sed -s 's#http://repo1.maven.org#https://repo1.maven.org#' -i pom.xml && \ - cd addons/hive-bridge && mvn package -Dhive.version=2.3.3 && cp -a target/hive-bridge-${ATLAS_VERSION}.jar /usr/lib/apiary/ && \ - cd /tmp && rm -rf /root/.m2 && rm -rf /tmp/apache-atlas-sources-${ATLAS_VERSION}/ && rm -f /tmp/apache-atlas-${ATLAS_VERSION}-sources.tar.gz - COPY files/core-site.xml /etc/hadoop/conf/core-site.xml COPY files/hive-site.xml /etc/hive/conf/hive-site.xml COPY files/hive-log4j2.properties /etc/hive/conf/hive-log4j2.properties COPY files/ranger-hive-security.xml /etc/hive/conf/ranger-hive-security.xml COPY files/ranger-hive-audit.xml /etc/hive/conf/ranger-hive-audit.xml -COPY files/atlas-application.properties /etc/hive/conf/atlas-application.properties COPY files/jmx-exporter.yaml /etc/hive/conf/jmx-exporter.yaml diff --git a/README.md b/README.md index 22d544d..924660a 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,6 @@ For more information please refer to the main [Apiary](https://github.com/Expedi |APIARY_S3_INVENTORY_PREFIX|No (defaults to `EntireBucketDaily`)|Prefix used by S3 Inventory when creating data in the inventory bucket.| |APIARY_S3_INVENTORY_TABLE_FORMAT|No (defaults to `ORC`)|Format of S3 inventory data - `ORC`, `Parquet`, or `CSV`| |APIARY_SYSTEM_SCHEMA|No (defaults to `apiary_system`)|Name for internal system database.| -|ATLAS_KAFKA_BOOTSTRAP_SERVERS|No|Atlas hive-bridge kafka bootstrap servers.| |AWS_REGION|Yes|AWS region to configure various AWS clients.| |ENABLE_GLUESYNC|No|Option to turn on GlueSync Hive Metastore listener.| |ENABLE_METRICS|No|Option to enable sending Hive Metastore metrics to CloudWatch.| diff --git a/files/atlas-2.0.0-hive-2.3.3.patch b/files/atlas-2.0.0-hive-2.3.3.patch deleted file mode 100644 index 141e320..0000000 --- a/files/atlas-2.0.0-hive-2.3.3.patch +++ /dev/null @@ -1,207 +0,0 @@ -diff -Naur apache-atlas-sources-2.0.0/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java apache-atlas-sources-2.0.0-patched/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java ---- apache-atlas-sources-2.0.0/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java 2019-05-06 12:55:33.000000000 -0500 -+++ apache-atlas-sources-2.0.0-patched/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java 2019-11-13 19:39:54.587194726 -0600 -@@ -77,7 +77,7 @@ - this.knownObjects = knownObjects; - this.metastoreHook = metastoreHook; - this.metastoreEvent = listenerEvent; -- this.metastoreHandler = (listenerEvent != null) ? metastoreEvent.getIHMSHandler() : null; -+ this.metastoreHandler = (listenerEvent != null) ? metastoreEvent.getHandler() : null; - - init(); - } -@@ -248,10 +248,6 @@ - case CREATEDATABASE: - databases.add(((CreateDatabaseEvent) metastoreEvent).getDatabase()); - break; -- case ALTERDATABASE: -- databases.add(((AlterDatabaseEvent) metastoreEvent).getOldDatabase()); -- databases.add(((AlterDatabaseEvent) metastoreEvent).getNewDatabase()); -- break; - case CREATETABLE: - tables.add(toTable(((CreateTableEvent) metastoreEvent).getTable())); - break; -@@ -289,4 +285,4 @@ - private static boolean isCreateAlterOperation(String operationName) { - return operationName != null && operationName.startsWith(CREATE_OPERATION) || operationName.startsWith(ALTER_OPERATION); - } --} -\ No newline at end of file -+} -diff -Naur apache-atlas-sources-2.0.0/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterDatabase.java apache-atlas-sources-2.0.0-patched/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterDatabase.java ---- apache-atlas-sources-2.0.0/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterDatabase.java 2019-05-03 00:22:00.000000000 -0500 -+++ apache-atlas-sources-2.0.0-patched/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterDatabase.java 1969-12-31 18:00:00.000000000 -0600 -@@ -1,76 +0,0 @@ --/** -- * Licensed to the Apache Software Foundation (ASF) under one -- * or more contributor license agreements. See the NOTICE file -- * distributed with this work for additional information -- * regarding copyright ownership. The ASF licenses this file -- * to you under the Apache License, Version 2.0 (the -- * "License"); you may not use this file except in compliance -- * with the License. You may obtain a copy of the License at -- *

-- * http://www.apache.org/licenses/LICENSE-2.0 -- *

-- * Unless required by applicable law or agreed to in writing, software -- * distributed under the License is distributed on an "AS IS" BASIS, -- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -- * See the License for the specific language governing permissions and -- * limitations under the License. -- */ -- --package org.apache.atlas.hive.hook.events; -- --import org.apache.atlas.hive.hook.AtlasHiveHookContext; --import org.apache.atlas.model.instance.AtlasEntity; --import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo; --import org.apache.atlas.model.notification.HookNotification; --import org.apache.atlas.model.notification.HookNotification.EntityUpdateRequestV2; --import org.apache.commons.collections.CollectionUtils; --import org.apache.hadoop.hive.metastore.api.Database; --import org.apache.hadoop.hive.metastore.events.AlterDatabaseEvent; --import org.slf4j.Logger; --import org.slf4j.LoggerFactory; -- --import java.util.Collections; --import java.util.List; -- --public class AlterDatabase extends CreateDatabase { -- private static final Logger LOG = LoggerFactory.getLogger(AlterDatabase.class); -- -- public AlterDatabase(AtlasHiveHookContext context) { -- super(context); -- } -- -- @Override -- public List getNotificationMessages() throws Exception { -- List ret = null; -- AtlasEntitiesWithExtInfo entities = context.isMetastoreHook() ? getHiveMetastoreEntities() : getHiveEntities(); -- -- if (entities != null && CollectionUtils.isNotEmpty(entities.getEntities())) { -- ret = Collections.singletonList(new EntityUpdateRequestV2(getUserName(), entities)); -- } -- -- return ret; -- } -- -- public AtlasEntitiesWithExtInfo getHiveMetastoreEntities() throws Exception { -- AtlasEntitiesWithExtInfo ret = new AtlasEntitiesWithExtInfo(); -- AlterDatabaseEvent dbEvent = (AlterDatabaseEvent) context.getMetastoreEvent(); -- Database oldDb = dbEvent.getOldDatabase(); -- Database newDb = dbEvent.getNewDatabase(); -- -- if (newDb != null) { -- AtlasEntity dbEntity = toDbEntity(newDb); -- -- ret.addEntity(dbEntity); -- } else { -- LOG.error("AlterDatabase.getEntities(): failed to retrieve db"); -- } -- -- addProcessedEntities(ret); -- -- return ret; -- } -- -- public AtlasEntitiesWithExtInfo getHiveEntities() throws Exception { -- return super.getHiveEntities(); -- } --} -\ No newline at end of file -diff -Naur apache-atlas-sources-2.0.0/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java apache-atlas-sources-2.0.0-patched/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java ---- apache-atlas-sources-2.0.0/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java 2019-05-06 12:55:33.000000000 -0500 -+++ apache-atlas-sources-2.0.0-patched/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java 2019-11-13 19:42:30.087556171 -0600 -@@ -37,7 +37,6 @@ - import org.apache.hadoop.hive.metastore.api.Order; - import org.apache.hadoop.hive.metastore.api.SerDeInfo; - import org.apache.hadoop.hive.metastore.api.StorageDescriptor; --import org.apache.hadoop.hive.metastore.utils.SecurityUtils; - import org.apache.hadoop.hive.ql.hooks.*; - import org.apache.hadoop.hive.ql.hooks.LineageInfo.BaseColumnInfo; - import org.apache.hadoop.hive.ql.hooks.LineageInfo.DependencyKey; -@@ -47,6 +46,7 @@ - import org.apache.hadoop.security.UserGroupInformation; - import org.slf4j.Logger; - import org.slf4j.LoggerFactory; -+import org.apache.hadoop.hive.shims.Utils; - - import java.io.IOException; - import java.net.URI; -@@ -689,7 +689,7 @@ - - if (context.isMetastoreHook()) { - try { -- ugi = SecurityUtils.getUGI(); -+ ugi = Utils.getUGI(); - } catch (Exception e) { - //do nothing - } -diff -Naur apache-atlas-sources-2.0.0/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java apache-atlas-sources-2.0.0-patched/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java ---- apache-atlas-sources-2.0.0/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java 2019-05-06 12:55:33.000000000 -0500 -+++ apache-atlas-sources-2.0.0-patched/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java 2019-11-13 19:40:12.087235438 -0600 -@@ -174,12 +174,6 @@ - event = new DropDatabase(context); - break; - -- case ALTERDATABASE: -- case ALTERDATABASE_OWNER: -- case ALTERDATABASE_LOCATION: -- event = new AlterDatabase(context); -- break; -- - case CREATETABLE: - event = new CreateTable(context, true); - break; -diff -Naur apache-atlas-sources-2.0.0/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveMetastoreHookImpl.java apache-atlas-sources-2.0.0-patched/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveMetastoreHookImpl.java ---- apache-atlas-sources-2.0.0/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveMetastoreHookImpl.java 2019-05-03 00:22:00.000000000 -0500 -+++ apache-atlas-sources-2.0.0-patched/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveMetastoreHookImpl.java 2019-11-13 19:41:21.239396244 -0600 -@@ -24,7 +24,6 @@ - import org.apache.hadoop.hive.metastore.MetaStoreEventListener; - import org.apache.hadoop.hive.metastore.api.FieldSchema; - import org.apache.hadoop.hive.metastore.events.*; --import org.apache.hadoop.hive.metastore.utils.SecurityUtils; - import org.apache.hadoop.hive.ql.metadata.Table; - import org.apache.hadoop.hive.ql.plan.HiveOperation; - import org.apache.hadoop.hive.shims.Utils; -@@ -64,13 +63,6 @@ - } - - @Override -- public void onAlterDatabase(AlterDatabaseEvent dbEvent) { -- HiveOperationContext context = new HiveOperationContext(ALTERDATABASE, dbEvent); -- -- hook.handleEvent(context); -- } -- -- @Override - public void onCreateTable(CreateTableEvent tableEvent) { - HiveOperationContext context = new HiveOperationContext(CREATETABLE, tableEvent); - -@@ -126,10 +118,6 @@ - event = new DropDatabase(context); - break; - -- case ALTERDATABASE: -- event = new AlterDatabase(context); -- break; -- - case CREATETABLE: - event = new CreateTable(context, true); - break; -@@ -161,7 +149,7 @@ - } - - if (event != null) { -- final UserGroupInformation ugi = SecurityUtils.getUGI() == null ? Utils.getUGI() : SecurityUtils.getUGI(); -+ final UserGroupInformation ugi = Utils.getUGI(); - - super.notifyEntities(event.getNotificationMessages(), ugi); - } -@@ -190,4 +178,4 @@ - - return isColumnRename; - } --} -\ No newline at end of file -+} diff --git a/files/atlas-application.properties b/files/atlas-application.properties deleted file mode 100644 index 49cb3c0..0000000 --- a/files/atlas-application.properties +++ /dev/null @@ -1,288 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -######### Graph Database Configs ######### - -# Graph Database - -#Configures the graph database to use. Defaults to JanusGraph -#atlas.graphdb.backend=org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase - -# Graph Storage -# Set atlas.graph.storage.backend to the correct value for your desired storage -# backend. Possible values: -# -# hbase -# cassandra -# embeddedcassandra - Should only be set by building Atlas with -Pdist,embedded-cassandra-solr -# berkeleyje -# -# See the configuration documentation for more information about configuring the various storage backends. -# -atlas.graph.storage.backend=hbase2 -atlas.graph.storage.hbase.table=apache_atlas_janus - -#Hbase -#For standalone mode , specify localhost -#for distributed mode, specify zookeeper quorum here -atlas.graph.storage.hostname=localhost -atlas.graph.storage.hbase.regions-per-server=1 -atlas.graph.storage.lock.wait-time=10000 - -# Gremlin Query Optimizer -# -# Enables rewriting gremlin queries to maximize performance. This flag is provided as -# a possible way to work around any defects that are found in the optimizer until they -# are resolved. -#atlas.query.gremlinOptimizerEnabled=true - -# Delete handler -# -# This allows the default behavior of doing "soft" deletes to be changed. -# -# Allowed Values: -# org.apache.atlas.repository.store.graph.v1.SoftDeleteHandlerV1 - all deletes are "soft" deletes -# org.apache.atlas.repository.store.graph.v1.HardDeleteHandlerV1 - all deletes are "hard" deletes -# -#atlas.DeleteHandlerV1.impl=org.apache.atlas.repository.store.graph.v1.SoftDeleteHandlerV1 - -# Entity audit repository -# -# This allows the default behavior of logging entity changes to hbase to be changed. -# -# Allowed Values: -# org.apache.atlas.repository.audit.HBaseBasedAuditRepository - log entity changes to hbase -# org.apache.atlas.repository.audit.CassandraBasedAuditRepository - log entity changes to cassandra -# org.apache.atlas.repository.audit.NoopEntityAuditRepository - disable the audit repository -# -atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.HBaseBasedAuditRepository - -# if Cassandra is used as a backend for audit from the above property, uncomment and set the following -# properties appropriately. If using the embedded cassandra profile, these properties can remain -# commented out. -# atlas.EntityAuditRepository.keyspace=atlas_audit -# atlas.EntityAuditRepository.replicationFactor=1 - - -# Graph Search Index -atlas.graph.index.search.backend=none - -#Solr -#Solr cloud mode properties -atlas.graph.index.search.solr.mode=cloud -atlas.graph.index.search.solr.zookeeper-url=localhost:2181 -atlas.graph.index.search.solr.zookeeper-connect-timeout=60000 -atlas.graph.index.search.solr.zookeeper-session-timeout=60000 -atlas.graph.index.search.solr.wait-searcher=true - -#Solr http mode properties -#atlas.graph.index.search.solr.mode=http -#atlas.graph.index.search.solr.http-urls=http://localhost:8983/solr - -# Solr-specific configuration property -atlas.graph.index.search.max-result-set-size=150 - -######### Notification Configs ######### -#atlas.notification.embedded=true -#atlas.kafka.data=${sys:atlas.home}/data/kafka -#atlas.kafka.zookeeper.connect=localhost:2181 -atlas.kafka.bootstrap.servers=ATLAS_KAFKA_BOOTSTRAP_SERVERS -atlas.kafka.zookeeper.session.timeout.ms=400 -atlas.kafka.zookeeper.connection.timeout.ms=200 -atlas.kafka.zookeeper.sync.time.ms=20 -atlas.kafka.auto.commit.interval.ms=1000 -atlas.kafka.hook.group.id=atlas - -atlas.kafka.enable.auto.commit=false -atlas.kafka.auto.offset.reset=earliest -atlas.kafka.session.timeout.ms=30000 -atlas.kafka.offsets.topic.replication.factor=1 -atlas.kafka.poll.timeout.ms=1000 - -atlas.notification.create.topics=true -atlas.notification.replicas=1 -atlas.notification.topics=ATLAS_HOOK,ATLAS_ENTITIES -atlas.notification.log.failed.messages=true -atlas.notification.consumer.retry.interval=500 -atlas.notification.hook.retry.interval=1000 -# Enable for Kerberized Kafka clusters -#atlas.notification.kafka.service.principal=kafka/_HOST@EXAMPLE.COM -#atlas.notification.kafka.keytab.location=/etc/security/keytabs/kafka.service.keytab - -## Server port configuration -#atlas.server.http.port=21000 -#atlas.server.https.port=21443 - -######### Security Properties ######### - -# SSL config -atlas.enableTLS=false - -#truststore.file=/path/to/truststore.jks -#cert.stores.credential.provider.path=jceks://file/path/to/credentialstore.jceks - -#following only required for 2-way SSL -#keystore.file=/path/to/keystore.jks - -# Authentication config - -atlas.authentication.method.kerberos=false -atlas.authentication.method.file=true - -#### ldap.type= LDAP or AD -atlas.authentication.method.ldap.type=none - -#### user credentials file -atlas.authentication.method.file.filename=${sys:atlas.home}/conf/users-credentials.properties - -### groups from UGI -#atlas.authentication.method.ldap.ugi-groups=true - -######## LDAP properties ######### -#atlas.authentication.method.ldap.url=ldap://:389 -#atlas.authentication.method.ldap.userDNpattern=uid={0},ou=People,dc=example,dc=com -#atlas.authentication.method.ldap.groupSearchBase=dc=example,dc=com -#atlas.authentication.method.ldap.groupSearchFilter=(member=uid={0},ou=Users,dc=example,dc=com) -#atlas.authentication.method.ldap.groupRoleAttribute=cn -#atlas.authentication.method.ldap.base.dn=dc=example,dc=com -#atlas.authentication.method.ldap.bind.dn=cn=Manager,dc=example,dc=com -#atlas.authentication.method.ldap.bind.password= -#atlas.authentication.method.ldap.referral=ignore -#atlas.authentication.method.ldap.user.searchfilter=(uid={0}) -#atlas.authentication.method.ldap.default.role= - - -######### Active directory properties ####### -#atlas.authentication.method.ldap.ad.domain=example.com -#atlas.authentication.method.ldap.ad.url=ldap://:389 -#atlas.authentication.method.ldap.ad.base.dn=(sAMAccountName={0}) -#atlas.authentication.method.ldap.ad.bind.dn=CN=team,CN=Users,DC=example,DC=com -#atlas.authentication.method.ldap.ad.bind.password= -#atlas.authentication.method.ldap.ad.referral=ignore -#atlas.authentication.method.ldap.ad.user.searchfilter=(sAMAccountName={0}) -#atlas.authentication.method.ldap.ad.default.role= - -######### JAAS Configuration ######## - -#atlas.jaas.KafkaClient.loginModuleName = com.sun.security.auth.module.Krb5LoginModule -#atlas.jaas.KafkaClient.loginModuleControlFlag = required -#atlas.jaas.KafkaClient.option.useKeyTab = true -#atlas.jaas.KafkaClient.option.storeKey = true -#atlas.jaas.KafkaClient.option.serviceName = kafka -#atlas.jaas.KafkaClient.option.keyTab = /etc/security/keytabs/atlas.service.keytab -#atlas.jaas.KafkaClient.option.principal = atlas/_HOST@EXAMPLE.COM - -######### Server Properties ######### -atlas.rest.address=http://localhost:21000 -# If enabled and set to true, this will run setup steps when the server starts -#atlas.server.run.setup.on.start=false - -######### Entity Audit Configs ######### -atlas.audit.hbase.tablename=apache_atlas_entity_audit -atlas.audit.zookeeper.session.timeout.ms=1000 -atlas.audit.hbase.zookeeper.quorum=localhost:2181 - -######### High Availability Configuration ######## -atlas.server.ha.enabled=false -#### Enabled the configs below as per need if HA is enabled ##### -#atlas.server.ids=id1 -#atlas.server.address.id1=localhost:21000 -#atlas.server.ha.zookeeper.connect=localhost:2181 -#atlas.server.ha.zookeeper.retry.sleeptime.ms=1000 -#atlas.server.ha.zookeeper.num.retries=3 -#atlas.server.ha.zookeeper.session.timeout.ms=20000 -## if ACLs need to be set on the created nodes, uncomment these lines and set the values ## -#atlas.server.ha.zookeeper.acl=: -#atlas.server.ha.zookeeper.auth=: - - - -######### Atlas Authorization ######### -atlas.authorizer.impl=simple -atlas.authorizer.simple.authz.policy.file=atlas-simple-authz-policy.json - -######### Type Cache Implementation ######## -# A type cache class which implements -# org.apache.atlas.typesystem.types.cache.TypeCache. -# The default implementation is org.apache.atlas.typesystem.types.cache.DefaultTypeCache which is a local in-memory type cache. -#atlas.TypeCache.impl= - -######### Performance Configs ######### -#atlas.graph.storage.lock.retries=10 -#atlas.graph.storage.cache.db-cache-time=120000 - -######### CSRF Configs ######### -atlas.rest-csrf.enabled=true -atlas.rest-csrf.browser-useragents-regex=^Mozilla.*,^Opera.*,^Chrome.* -atlas.rest-csrf.methods-to-ignore=GET,OPTIONS,HEAD,TRACE -atlas.rest-csrf.custom-header=X-XSRF-HEADER - -############ KNOX Configs ################ -#atlas.sso.knox.browser.useragent=Mozilla,Chrome,Opera -#atlas.sso.knox.enabled=true -#atlas.sso.knox.providerurl=https://:8443/gateway/knoxsso/api/v1/websso -#atlas.sso.knox.publicKey= - -############ Atlas Metric/Stats configs ################ -# Format: atlas.metric.query.. -atlas.metric.query.cache.ttlInSecs=900 -#atlas.metric.query.general.typeCount= -#atlas.metric.query.general.typeUnusedCount= -#atlas.metric.query.general.entityCount= -#atlas.metric.query.general.tagCount= -#atlas.metric.query.general.entityDeleted= -# -#atlas.metric.query.entity.typeEntities= -#atlas.metric.query.entity.entityTagged= -# -#atlas.metric.query.tags.entityTags= - -######### Compiled Query Cache Configuration ######### - -# The size of the compiled query cache. Older queries will be evicted from the cache -# when we reach the capacity. - -#atlas.CompiledQueryCache.capacity=1000 - -# Allows notifications when items are evicted from the compiled query -# cache because it has become full. A warning will be issued when -# the specified number of evictions have occurred. If the eviction -# warning threshold <= 0, no eviction warnings will be issued. - -#atlas.CompiledQueryCache.evictionWarningThrottle=0 - - -######### Full Text Search Configuration ######### - -#Set to false to disable full text search. -#atlas.search.fulltext.enable=true - -######### Gremlin Search Configuration ######### - -#Set to false to disable gremlin search. -atlas.search.gremlin.enable=false - - -########## Add http headers ########### - -#atlas.headers.Access-Control-Allow-Origin=* -#atlas.headers.Access-Control-Allow-Methods=GET,OPTIONS,HEAD,PUT,POST -#atlas.headers.= - -atlas.metadata.namespace=ATLAS_CLUSTER_NAME -atlas.cluster.name=ATLAS_CLUSTER_NAME diff --git a/files/startup.sh b/files/startup.sh index 7876e17..65bf1ef 100755 --- a/files/startup.sh +++ b/files/startup.sh @@ -74,17 +74,6 @@ if [ -n "$ENABLE_METRICS" ]; then fi fi -# If Atlas metastore plugin is being used, set Atlas config properties -if [[ ! -z $ATLAS_KAFKA_BOOTSTRAP_SERVERS ]] -then - # Update Atlas kafka URL - sed "s/ATLAS_KAFKA_BOOTSTRAP_SERVERS/$ATLAS_KAFKA_BOOTSTRAP_SERVERS/" -i /etc/hive/conf/atlas-application.properties - # Update Atlas cluster name - # For backward compatability, if ATLAS_CLUSTER_NAME env var is not set, use INSTANCE_NAME - [[ -z ${ATLAS_CLUSTER_NAME} ]] && ATLAS_CLUSTER_NAME=${INSTANCE_NAME} - sed "s/ATLAS_CLUSTER_NAME/${ATLAS_CLUSTER_NAME}/g" -i /etc/hive/conf/atlas-application.properties -fi - #configure kafka metastore listener if [[ ! -z $KAFKA_BOOTSTRAP_SERVERS ]]; then sed "s/KAFKA_BOOTSTRAP_SERVERS/$KAFKA_BOOTSTRAP_SERVERS/" -i /etc/hive/conf/hive-site.xml @@ -145,7 +134,6 @@ sed "s/HIVE_METASTORE_LOG_LEVEL/$HIVE_METASTORE_LOG_LEVEL/" -i /etc/hive/conf/hi [[ ! -z $SNS_ARN ]] && export METASTORE_LISTENERS="${METASTORE_LISTENERS},com.expediagroup.apiary.extensions.events.metastore.listener.ApiarySnsListener" [[ ! -z $KAFKA_BOOTSTRAP_SERVERS ]] && export METASTORE_LISTENERS="${METASTORE_LISTENERS},com.expediagroup.apiary.extensions.events.metastore.kafka.listener.KafkaMetaStoreEventListener" -[[ ! -z $ATLAS_KAFKA_BOOTSTRAP_SERVERS ]] && export METASTORE_LISTENERS="${METASTORE_LISTENERS},org.apache.atlas.hive.hook.HiveMetastoreHookImpl" [[ ! -z $ENABLE_GLUESYNC ]] && export METASTORE_LISTENERS="${METASTORE_LISTENERS},com.expediagroup.apiary.extensions.gluesync.listener.ApiaryGlueSync" #remove leading , when external METASTORE_LISTENERS are not defined export METASTORE_LISTENERS=$(echo $METASTORE_LISTENERS|sed 's/^,//') @@ -160,12 +148,11 @@ sed "s/METASTORE_PRELISTENERS/${METASTORE_PRELISTENERS}/" -i /etc/hive/conf/hive export AUX_CLASSPATH="/usr/share/java/mariadb-connector-java.jar" [[ ! -z $SNS_ARN ]] && export AUX_CLASSPATH="$AUX_CLASSPATH:/usr/lib/apiary/apiary-metastore-listener-${APIARY_EXTENSIONS_VERSION}-all.jar" -[[ ! -z $KAFKA_BOOTSTRAP_SERVERS ]] && export AUX_CLASSPATH="$AUX_CLASSPATH:/usr/lib/apiary/kafka-metastore-listener-${APIARY_EXTENSIONS_VERSION}-all.jar" +[[ ! -z $KAFKA_BOOTSTRAP_SERVERS ]] && export AUX_CLASSPATH="$AUX_CLASSPATH:/usr/lib/apiary/kafka-metastore-listener-${APIARY_EXTENSIONS_VERSION}-all.jar:/usr/lib/apiary/kafka-clients-${KAFKA_VERSION}.jar" [[ ! -z $ENABLE_GLUESYNC ]] && export AUX_CLASSPATH="$AUX_CLASSPATH:/usr/lib/apiary/apiary-gluesync-listener-${APIARY_GLUESYNC_LISTENER_VERSION}-all.jar" [[ ! -z $RANGER_POLICY_MANAGER_URL ]] && export AUX_CLASSPATH="$AUX_CLASSPATH:/usr/lib/apiary/apiary-ranger-metastore-plugin-${APIARY_RANGER_PLUGIN_VERSION}-all.jar:/usr/lib/apiary/commons-codec-${COMMONS_CODEC_VERSION}.jar:/usr/lib/apiary/gethostname4j-${GETHOSTNAME4J_VERSION}.jar:/usr/lib/apiary/jna-${JNA_VERSION}.jar" [[ ! -z $HIVE_DB_WHITELIST ]] && export AUX_CLASSPATH="$AUX_CLASSPATH:/usr/lib/apiary/apiary-metastore-auth-${APIARY_METASTORE_AUTH_VERSION}.jar" [[ ! -z $ENABLE_METRICS ]] && export AUX_CLASSPATH="$AUX_CLASSPATH:/usr/lib/apiary/apiary-metastore-metrics-${APIARY_METASTORE_METRICS_VERSION}-all.jar" -[[ ! -z $ATLAS_KAFKA_BOOTSTRAP_SERVERS ]] && export AUX_CLASSPATH="$AUX_CLASSPATH:/usr/lib/apiary/hive-bridge-${ATLAS_VERSION}.jar:/usr/lib/apiary/atlas-notification-${ATLAS_VERSION}.jar:/usr/lib/apiary/atlas-intg-${ATLAS_VERSION}.jar:/usr/lib/apiary/atlas-common-${ATLAS_VERSION}.jar:/usr/lib/apiary/kafka-clients-${KAFKA_VERSION}.jar" #configure container credentials provider when running in ECS if [ ! -z ${AWS_CONTAINER_CREDENTIALS_RELATIVE_URI} ]; then