fixes #35 configure metastore to connect to RDS using IAM credentials

Dockerfile

@@ -6,6 +6,7 @@ from amazonlinux:latest
 ENV VAULT_VERSION 0.10.3
 ENV RANGER_VERSION 1.1.0
 ENV APIARY_METASTORE_LISTENER_VERSION 0.1.0
+ENV IAM_JDBC_VERSION 1.1.0
 
 COPY files/RPM-GPG-KEY-emr /etc/pki/rpm-gpg/RPM-GPG-KEY-emr
 COPY files/emr-apps.repo /etc/yum.repos.d/emr-apps.repo
@@ -20,6 +21,7 @@ RUN yum -y install java-1.8.0-openjdk \
   unzip \
   jq \
   emrfs \
+  maven \
   && yum clean all \
   && rm -rf /var/cache/yum
 
@@ -41,6 +43,17 @@ wget -qN https://search.maven.org/remotecontent?filepath=org/eclipse/persistence
 COPY src /src
 RUN cd src && javac -cp "/usr/lib/hadoop/*:/usr/lib/hive/lib/*:/usr/share/aws/aws-java-sdk/*" *.java && jar cf /usr/lib/hive/lib/MetastoreListeners.jar *.class && rm -f *.class
 
+RUN wget -q -O - https://github.com/rikturnbull/iam-jdbc-driver/archive/v${IAM_JDBC_VERSION}.tar.gz|tar -C /tmp -xzf - && \
+cd /tmp/iam-jdbc-driver-${IAM_JDBC_VERSION} && \
+sed 's/com.mysql.jdbc.Driver/org.mariadb.jdbc.Driver/' -i src/main/java/uk/co/controlz/aws/IAMJDBCDriver.java && \
+sed 's/properties.getProperty(PROPERTY_AWS_REGION)/System.getenv("AWS_REGION")/' -i src/main/java/uk/co/controlz/aws/IAMJDBCDriver.java && \
+sed 's/<dependencies>/<dependencies>\n<dependency>\n<groupId>org.mariadb.jdbc<\/groupId>\n<artifactId>mariadb-java-client<\/artifactId>\n<version>2.3.0<\/version>\n<\/dependency>\n/g' -i pom.xml && \
+mvn package  && cp -a target/iam-jdbc-driver-${IAM_JDBC_VERSION}.jar /usr/lib/hive/lib/ && \
+rm -rf /root/.m2 && rm -rf /tmp/iam-jdbc-driver-${IAM_JDBC_VERSION}
+
+#RDS CA certificate, required to use jdbc with ssl
+RUN wget -q https://s3.amazonaws.com/rds-downloads/rds-ca-2015-root.pem -O /etc/pki/ca-trust/source/anchors/rds-ca-2015-root.pem && update-ca-trust && update-ca-trust enable
+
 RUN echo 'export HADOOP_CLASSPATH="$HADOOP_CLASSPATH:/usr/share/aws/emr/emrfs/conf:/usr/share/aws/emr/emrfs/lib/*:/usr/share/aws/emr/emrfs/auxlib/*"' >> /etc/hadoop/conf/hadoop-env.sh
 COPY files/core-site.xml /etc/hadoop/conf/core-site.xml
 COPY files/emrfs-site.xml /usr/share/aws/emr/emrfs/conf/emrfs-site.xml

files/hive-site.xml

@@ -20,7 +20,7 @@
 
 <property>
     <name>javax.jdo.option.ConnectionDriverName</name>
-    <value>org.mariadb.jdbc.Driver</value>
+    <value>uk.co.controlz.aws.IAMJDBCDriver</value>
 </property>
 
 <property>

files/startup.sh

@@ -6,11 +6,9 @@ export VAULT_SKIP_VERIFY=true
 export VAULT_TOKEN=`vault login -method=aws -path=${VAULT_LOGIN_PATH} -token-only`
 
 if [ x"$instance_type" = x"readwrite" ]; then
-    dbuser=`vault read -field=username ${vault_path}/hive_rwuser`
-    dbpass=`vault read -field=password ${vault_path}/hive_rwuser`
+    dbuser="iamrw"
 else
-    dbuser=`vault read -field=username ${vault_path}/hive_rouser`
-    dbpass=`vault read -field=password ${vault_path}/hive_rouser`
+    dbuser="iamro"
 fi
 
 #configure LDAP group mapping, required for ranger authorization
@@ -47,7 +45,8 @@ fi
 
 #check if database is initialized, test only from rw instances and only if DB is managed by apiary
 if [ -z $EXTERNAL_DATABASE ] && [ x"$instance_type" = x"readwrite" ]; then
-MYSQL_OPTIONS="-h$dbhost -u$dbuser -p$dbpass $dbname -N"
+TOKEN=$(aws rds generate-db-auth-token --hostname $dbhost --port 3306 --region $AWS_REGION --username $dbuser)
+MYSQL_OPTIONS="-h$dbhost --ssl-ca=/etc/pki/ca-trust/source/anchors/rds-ca-2015-root.pem -u$dbuser -p$TOKEN $dbname -N"
 schema_version=`echo "select SCHEMA_VERSION from VERSION"|mysql $MYSQL_OPTIONS`
 if [ x"$schema_version" != x"2.3.0" ]; then
     cd /usr/lib/hive/scripts/metastore/upgrade/mysql
@@ -91,4 +90,4 @@ sed "s/METASTORE_PRELISTENERS/${METASTORE_PRELISTENERS}/" -i /etc/hive/conf/hive
 #export HADOOP_OPTS="$HADOOP_OPTS -Dorg.apache.commons.logging.LogFactory=org.apache.commons.logging.impl.LogFactoryImpl -Dorg.apache.commons.logging.Log=org.apache.commons.logging.impl.SimpleLog"
 
 export AUX_CLASSPATH="/usr/share/java/mariadb-connector-java.jar:/usr/lib/apiary/apiary-metastore-listener-${APIARY_METASTORE_LISTENER_VERSION}-all.jar:/usr/share/aws/aws-java-sdk/*"
-su hive -s/bin/bash -c "/usr/lib/hive/bin/hive --service metastore --hiveconf hive.root.logger=${loglevel},console --hiveconf javax.jdo.option.ConnectionURL=jdbc:mysql://${dbhost}:3306/${dbname} --hiveconf javax.jdo.option.ConnectionUserName='${dbuser}' --hiveconf javax.jdo.option.ConnectionPassword='${dbpass}'"
+su hive -s/bin/bash -c "/usr/lib/hive/bin/hive --service metastore --hiveconf hive.root.logger=${loglevel},console --hiveconf javax.jdo.option.ConnectionURL=jdbc:mysqliam://${dbhost}:3306/${dbname}?useSSL=true\&requireSSL=true --hiveconf javax.jdo.option.ConnectionUserName=${dbuser}"