upgrade tablestore as a new backend

JanusGraph · Oct 16, 2024 · 3f2034a · 3f2034a
1 parent 1890ed4
commit 3f2034a
Show file tree

Hide file tree

Showing 44 changed files with 3,619 additions and 21 deletions.
diff --git a/janusgraph-all/pom.xml b/janusgraph-all/pom.xml
@@ -82,21 +82,6 @@
             <artifactId>janusgraph-bigtable</artifactId>
             <version>${project.version}</version>
         </dependency>
-        <dependency>
-            <groupId>org.janusgraph</groupId>
-            <artifactId>janusgraph-lindorm</artifactId>
-            <version>${project.version}</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>org.apache.commons</groupId>
-                    <artifactId>commons-crypto</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.jruby.joni</groupId>
-                    <artifactId>joni</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
         <dependency>
             <groupId>org.janusgraph</groupId>
             <artifactId>janusgraph-tablestore</artifactId>
@@ -110,6 +95,10 @@
                     <groupId>ch.qos.reload4j</groupId>
                     <artifactId>reload4j</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>org.apache.httpcomponents</groupId>
+                    <artifactId>httpasyncclient</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>
@@ -129,6 +118,10 @@
                     <groupId>org.jruby.jcodings</groupId>
                     <artifactId>jcodings</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>org.jruby.jcodings</groupId>
+                    <artifactId>jcodings</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>

diff --git a/janusgraph-core/src/main/java/org/janusgraph/core/util/ReflectiveConfigOptionLoader.java b/janusgraph-core/src/main/java/org/janusgraph/core/util/ReflectiveConfigOptionLoader.java
@@ -122,6 +122,7 @@ public void loadStandard(Class<?> caller) {
          */
         List<String> classnames = Collections.unmodifiableList(Arrays.asList(
             "org.janusgraph.diskstorage.hbase.HBaseStoreManager",
+            "org.janusgraph.diskstorage.tablestore.TableStoreStoreManager",
             "org.janusgraph.diskstorage.cql.CQLConfigOptions",
             "org.janusgraph.diskstorage.es.ElasticSearchIndex",
             "org.janusgraph.diskstorage.solr.SolrIndex",

diff --git a/janusgraph-core/src/main/java/org/janusgraph/diskstorage/Backend.java b/janusgraph-core/src/main/java/org/janusgraph/diskstorage/Backend.java
@@ -171,6 +171,7 @@ public class Backend implements LockerProvider, AutoCloseable {
         put(StandardStoreManager.BDB_JE, STORAGE_DIRECTORY);
         put(StandardStoreManager.CQL, STORAGE_HOSTS);
         put(StandardStoreManager.HBASE, STORAGE_HOSTS);
+        put(StandardStoreManager.TABLESTORE,STORAGE_HOSTS);
         //put(StandardStorageBackend.IN_MEMORY, null);
         put(StandardStoreManager.SCYLLA, STORAGE_HOSTS);
     }});

diff --git a/janusgraph-core/src/main/java/org/janusgraph/diskstorage/StandardStoreManager.java b/janusgraph-core/src/main/java/org/janusgraph/diskstorage/StandardStoreManager.java
@@ -29,6 +29,7 @@ public enum StandardStoreManager {
     BDB_JE("org.janusgraph.diskstorage.berkeleyje.BerkeleyJEStoreManager", "berkeleyje"),
     CQL("org.janusgraph.diskstorage.cql.CQLStoreManager", "cql"),
     HBASE("org.janusgraph.diskstorage.hbase.HBaseStoreManager", "hbase"),
+    TABLESTORE("org.janusgraph.diskstorage.tablestore.TableStoreStoreManager", "tablestore"),
     IN_MEMORY("org.janusgraph.diskstorage.inmemory.InMemoryStoreManager", "inmemory"),
     SCYLLA("org.janusgraph.diskstorage.cql.ScyllaStoreManager", "scylla");
 

diff --git a/janusgraph-tablestore/docker/Dockerfile b/janusgraph-tablestore/docker/Dockerfile
@@ -0,0 +1,59 @@
+# Copyright 2021 JanusGraph Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM debian:bookworm-20240211 as builder
+
+ARG HBASE_VERSION=2.5.0
+ARG HBASE_DIST="https://archive.apache.org/dist/hbase"
+
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y curl
+
+RUN curl -SL ${HBASE_DIST}/${HBASE_VERSION}/hbase-${HBASE_VERSION}-bin.tar.gz | tar -x -z && mv hbase-${HBASE_VERSION} /opt/hbase
+WORKDIR /opt/hbase
+RUN rm -rf ./docs ./LEGAL ./*.txt ./*/*.cmd
+
+COPY ./hbase-site.xml /opt/hbase/conf/hbase-site.xml
+COPY ./hbase-policy.xml /opt/hbase/conf/hbase-policy.xml
+
+FROM openjdk:8-jre-buster
+
+ARG HBASE_UID=1000
+ARG HBASE_GID=1000
+
+ENV LOGS_DIR=/data/logs
+
+COPY --from=builder /opt/hbase/ /opt/hbase/
+
+RUN ln -sf /opt/hbase/bin/* /usr/bin/
+RUN mkdir -p $LOGS_DIR /data/hbase /data/wal /data/run /opt/hbase/logs
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y socat
+WORKDIR /opt/hbase
+COPY ./entrypoint.sh /entrypoint.sh
+
+EXPOSE 2181
+EXPOSE 16000
+EXPOSE 16010
+EXPOSE 16020
+EXPOSE 16030
+
+ENTRYPOINT ["/entrypoint.sh"]
+
+RUN groupadd -g ${HBASE_GID} hbase && \
+    useradd -s /bin/bash hbase -u ${HBASE_UID} -g ${HBASE_GID} && \
+    chmod +x /entrypoint.sh && \
+    chown -R ${HBASE_UID}:${HBASE_GID} /data && \
+    chown -R ${HBASE_UID}:${HBASE_GID} /opt/hbase/logs
+USER hbase
diff --git a/janusgraph-tablestore/docker/entrypoint.sh b/janusgraph-tablestore/docker/entrypoint.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+#
+# Copyright 2022 JanusGraph Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# re export zookeeper
+socat tcp-listen:2182,fork,reuseaddr tcp-connect:localhost:2181 &
+
+/usr/bin/hbase master start
diff --git a/janusgraph-tablestore/docker/hbase-policy.xml b/janusgraph-tablestore/docker/hbase-policy.xml
@@ -0,0 +1,53 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+  <property>
+    <name>security.client.protocol.acl</name>
+    <value>*</value>
+    <description>ACL for HRegionInterface protocol implementations (ie. 
+    clients talking to HRegionServers)
+    The ACL is a comma-separated list of user and group names. The user and 
+    group list is separated by a blank. For e.g. "alice,bob users,wheel". 
+    A special value of "*" means all users are allowed.</description>
+  </property>
+
+  <property>
+    <name>security.admin.protocol.acl</name>
+    <value>*</value>
+    <description>ACL for HMasterInterface protocol implementation (ie. 
+    clients talking to HMaster for admin operations).
+    The ACL is a comma-separated list of user and group names. The user and 
+    group list is separated by a blank. For e.g. "alice,bob users,wheel". 
+    A special value of "*" means all users are allowed.</description>
+  </property>
+
+  <property>
+    <name>security.masterregion.protocol.acl</name>
+    <value>*</value>
+    <description>ACL for HMasterRegionInterface protocol implementations
+    (for HRegionServers communicating with HMaster)
+    The ACL is a comma-separated list of user and group names. The user and 
+    group list is separated by a blank. For e.g. "alice,bob users,wheel". 
+    A special value of "*" means all users are allowed.</description>
+  </property>
+</configuration>
diff --git a/janusgraph-tablestore/docker/hbase-site.xml b/janusgraph-tablestore/docker/hbase-site.xml
@@ -0,0 +1,45 @@
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+
+  <property>
+    <name>hbase.zookeeper.quorum</name>
+    <value>localhost</value>
+  </property>
+  <property>
+    <name>hbase.rootdir</name>
+    <value>file:////data/hbase</value>
+  </property>
+  <property>
+    <name>hbase.cluster.distributed</name>
+    <value>false</value>
+  </property>
+  <property>
+    <name>hbase.wal.dir</name>
+    <value>file:////data/wal</value>
+  </property>
+  <property>
+    <name>hbase.unsafe.stream.capability.enforce</name>
+    <value>false</value>
+  </property>
+
+  <property>
+    <name>hbase.master.info.bindAddress</name>
+    <value>0.0.0.0</value>
+  </property>
+
+  <property>
+    <name>hbase.regionserver.info.bindAddress</name>
+    <value>0.0.0.0</value>
+  </property>
+
+  <property>
+    <name>hbase.master.ipc.address</name>
+    <value>0.0.0.0</value>
+  </property>
+
+  <property>
+    <name>hbase.regionserver.ipc.address</name>
+    <value>0.0.0.0</value>
+  </property>
+
+</configuration>