diff --git a/linkis-engineconn-plugins/hbase/src/main/assembly/distribution.xml b/linkis-engineconn-plugins/hbase/src/main/assembly/distribution.xml new file mode 100644 index 0000000000..d9c4bfebe5 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/assembly/distribution.xml @@ -0,0 +1,247 @@ + + + + + linkis-engineplugin-hbase + + dir + zip + + true + hbase + + + + + + /dist/${hbase.version}/lib + true + true + false + false + true + + + antlr:antlr:jar + asm:asm:jar + cglib:cglib:jar + com.amazonaws:aws-java-sdk-autoscaling:jar + com.amazonaws:aws-java-sdk-core:jar + com.amazonaws:aws-java-sdk-ec2:jar + com.amazonaws:aws-java-sdk-route53:jar + com.amazonaws:aws-java-sdk-sts:jar + com.amazonaws:jmespath-java:jar + com.fasterxml.jackson.core:jackson-annotations:jar + com.fasterxml.jackson.core:jackson-core:jar + com.fasterxml.jackson.core:jackson-databind:jar + com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:jar + com.fasterxml.jackson.datatype:jackson-datatype-jdk8:jar + com.fasterxml.jackson.datatype:jackson-datatype-jsr310:jar + com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:jar + com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar + com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar + com.fasterxml.jackson.module:jackson-module-parameter-names:jar + com.fasterxml.jackson.module:jackson-module-paranamer:jar + com.fasterxml.jackson.module:jackson-module-scala_2.11:jar + com.github.andrewoma.dexx:dexx-collections:jar + com.github.vlsi.compactmap:compactmap:jar + com.google.code.findbugs:annotations:jar + com.google.code.gson:gson:jar + com.google.guava:guava:jar + com.google.protobuf:protobuf-java:jar + com.ning:async-http-client:jar + com.sun.xml.bind:jaxb-impl:jar + commons-beanutils:commons-beanutils:jar + commons-beanutils:commons-beanutils-core:jar + commons-cli:commons-cli:jar + commons-collections:commons-collections:jar + commons-configuration:commons-configuration:jar + commons-dbcp:commons-dbcp:jar + commons-digester:commons-digester:jar + commons-io:commons-io:jar + commons-logging:commons-logging:jar + commons-jxpath:commons-jxpath:jar + commons-lang:commons-lang:jar + commons-net:commons-net:jar + commons-pool:commons-pool:jar + io.micrometer:micrometer-core:jar + io.netty:netty-all:jar + io.netty:netty-buffer:jar + io.netty:netty-codec:jar + io.netty:netty-codec-http:jar + io.netty:netty-common:jar + io.netty:netty-handler:jar + io.netty:netty-transport:jar + io.netty:netty-transport-native-epoll:jar + io.reactivex:rxjava:jar + io.reactivex:rxnetty:jar + io.reactivex:rxnetty-contexts:jar + io.reactivex:rxnetty-servo:jar + javax.activation:activation:jar + javax.annotation:javax.annotation-api:jar + javax.inject:javax.inject:jar + javax.servlet:javax.servlet-api:jar + javax.servlet.jsp:jsp-api:jar + javax.validation:validation-api:jar + javax.websocket:javax.websocket-api:jar + javax.ws.rs:javax.ws.rs-api:jar + javax.xml.bind:jaxb-api:jar + javax.xml.stream:stax-api:jar + mysql:mysql-connector-java:jar + org.antlr:antlr-runtime:jar + org.antlr:stringtemplate:jar + org.apache.commons:commons-compress:jar + org.apache.commons:commons-math:jar + org.apache.commons:commons-math3:jar + org.apache.curator:curator-framework:jar + org.apache.curator:curator-recipes:jar + org.apache.directory.api:api-asn1-api:jar + org.apache.directory.api:api-util:jar + org.apache.directory.server:apacheds-i18n:jar + org.apache.directory.server:apacheds-kerberos-codec:jar + org.apache.hadoop:hadoop-annotations:jar + org.apache.hadoop:hadoop-auth:jar + org.apache.hadoop:hadoop-common:jar + org.apache.hadoop:hadoop-hdfs:jar + org.apache.htrace:htrace-core:jar + org.apache.logging.log4j:log4j-api:jar + org.apache.logging.log4j:log4j-core:jar + org.apache.logging.log4j:log4j-jul:jar + org.apache.logging.log4j:log4j-slf4j-impl:jar + log4j:log4j:jar + org.apache.zookeeper:zookeeper:jar + org.aspectj:aspectjweaver:jar + org.bouncycastle:bcpkix-jdk15on:jar + org.bouncycastle:bcprov-jdk15on:jar + org.codehaus.jackson:jackson-jaxrs:jar + org.codehaus.jackson:jackson-xc:jar + org.codehaus.jettison:jettison:jar + org.codehaus.woodstox:stax2-api:jar + org.codehaus.woodstox:woodstox-core-asl:jar + org.eclipse.jetty:jetty-annotations:jar + org.eclipse.jetty:jetty-client:jar + org.eclipse.jetty:jetty-continuation:jar + org.eclipse.jetty:jetty-http:jar + org.eclipse.jetty:jetty-io:jar + org.eclipse.jetty:jetty-jndi:jar + org.eclipse.jetty:jetty-plus:jar + org.eclipse.jetty:jetty-security:jar + org.eclipse.jetty:jetty-server:jar + org.eclipse.jetty:jetty-servlet:jar + org.eclipse.jetty:jetty-servlets:jar + org.eclipse.jetty:jetty-util:jar + org.eclipse.jetty:jetty-webapp:jar + org.eclipse.jetty:jetty-xml:jar + org.eclipse.jetty.websocket:javax-websocket-client-impl:jar + org.eclipse.jetty.websocket:javax-websocket-server-impl:jar + org.eclipse.jetty.websocket:websocket-api:jar + org.eclipse.jetty.websocket:websocket-client:jar + org.eclipse.jetty.websocket:websocket-common:jar + org.eclipse.jetty.websocket:websocket-server:jar + org.eclipse.jetty.websocket:websocket-servlet:jar + org.eclipse.jetty.orbit:javax.servlet:jar + org.eclipse.jetty.aggregate:jetty-all:jar + org.fusesource.leveldbjni:leveldbjni-all:jar + org.glassfish.hk2:class-model:jar + org.glassfish.hk2:config-types:jar + org.glassfish.hk2.external:aopalliance-repackaged:jar + org.glassfish.hk2.external:asm-all-repackaged:jar + org.glassfish.hk2.external:bean-validator:jar + org.glassfish.hk2.external:javax.inject:jar + org.glassfish.hk2:hk2:jar + org.glassfish.hk2:hk2-api:jar + org.glassfish.hk2:hk2-config:jar + org.glassfish.hk2:hk2-core:jar + org.glassfish.hk2:hk2-locator:jar + org.glassfish.hk2:hk2-runlevel:jar + org.glassfish.hk2:hk2-utils:jar + org.glassfish.hk2:osgi-resource-locator:jar + org.glassfish.hk2:spring-bridge:jar + org.glassfish.jersey.bundles:jaxrs-ri:jar + org.glassfish.jersey.bundles.repackaged:jersey-guava:jar + org.glassfish.jersey.containers:jersey-container-servlet:jar + org.glassfish.jersey.containers:jersey-container-servlet-core:jar + org.glassfish.jersey.core:jersey-client:jar + org.glassfish.jersey.core:jersey-common:jar + org.glassfish.jersey.core:jersey-server:jar + org.glassfish.jersey.ext:jersey-entity-filtering:jar + org.glassfish.jersey.ext:jersey-spring3:jar + org.glassfish.jersey.media:jersey-media-jaxb:jar + org.glassfish.jersey.media:jersey-media-json-jackson:jar + org.glassfish.jersey.media:jersey-media-multipart:jar + org.hdrhistogram:HdrHistogram:jar + org.javassist:javassist:jar + org.json4s:json4s-ast_2.11:jar + org.json4s:json4s-core_2.11:jar + org.json4s:json4s-jackson_2.11:jar + org.jsoup:jsoup:jar + org.jvnet.mimepull:mimepull:jar + org.jvnet:tiger-types:jar + org.latencyutils:LatencyUtils:jar + org.mortbay.jasper:apache-el:jar + org.mortbay.jetty:jetty-util:jar + org.mortbay.jetty:jetty:jar + tomcat:jasper-compiler:jar + tomcat:jasper-runtime:jar + org.ow2.asm:asm-analysis:jar + org.ow2.asm:asm-commons:jar + org.ow2.asm:asm-tree:jar + org.reflections:reflections:jar + org.slf4j:jul-to-slf4j:jar + org.slf4j:slf4j-api:jar + org.tukaani:xz:jar + org.yaml:snakeyaml:jar + software.amazon.ion:ion-java:jar + xerces:xercesImpl:jar + xmlenc:xmlenc:jar + xmlpull:xmlpull:jar + xpp3:xpp3_min:jar + + + + + + + + + ${basedir}/src/main/resources + + * + + 0777 + dist/${hbase.version}/conf + unix + + + + ${basedir}/target + + *.jar + + + *doc.jar + + 0777 + plugin/${hbase.version} + + + + + + diff --git a/linkis-engineconn-plugins/hbase/src/main/java/org/apache/linkis/manager/engineplugin/hbase/HBaseConnectionManager.java b/linkis-engineconn-plugins/hbase/src/main/java/org/apache/linkis/manager/engineplugin/hbase/HBaseConnectionManager.java new file mode 100644 index 0000000000..03d0bdf9da --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/java/org/apache/linkis/manager/engineplugin/hbase/HBaseConnectionManager.java @@ -0,0 +1,341 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.engineplugin.hbase; + +import static org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant.DEFAULT_HBASE_DFS_ROOT_DIR; +import static org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant.DEFAULT_KRB5_CONF_PATH; +import static org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant.DEFAULT_ZOOKEEPER_CLIENT_PORT; +import static org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant.DEFAULT_ZOOKEEPER_NODE_PARENT; +import static org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant.DEFAULT_ZOOKEEPER_QUORUM; +import static org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant.HADOOP_SECURITY_AUTH; +import static org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant.HBASE_AUTH; +import static org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant.HBASE_DFS_ROOT_DIR; +import static org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant.HBASE_MASTER_KERBEROS_PRINCIPAL; +import static org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant.HBASE_REGION_SERVER_KERBEROS_PRINCIPAL; +import static org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant.HBASE_SECURITY_AUTH; +import static org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant.KERBEROS; +import static org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant.KERBEROS_KEYTAB_FILE; +import static org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant.KERBEROS_PRINCIPAL; +import static org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant.KERBEROS_PROXY_USER; +import static org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant.KRB5_CONF_PATH; +import static org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant.MASTER_SERVER_KERBEROS_PRINCIPAL; +import static org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant.REGION_SERVER_KERBEROS_PRINCIPAL; +import static org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant.SIMPLE; +import static org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant.UNIQUE_KEY_DELIMITER; +import static org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant.ZOOKEEPER_CLIENT_PORT; +import static org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant.ZOOKEEPER_NODE_PARENT; +import static org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant.ZOOKEEPER_QUORUM; + +import java.io.File; +import java.io.IOException; +import java.security.PrivilegedAction; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReentrantLock; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.linkis.manager.engineplugin.hbase.errorcode.HBaseErrorCodeSummary; +import org.apache.linkis.manager.engineplugin.hbase.exception.HBaseParamsIllegalException; +import org.apache.linkis.manager.engineplugin.hbase.exception.JobExecutorException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class HBaseConnectionManager { + private static final Logger LOG = LoggerFactory.getLogger(HBaseConnectionManager.class); + private final ConcurrentHashMap connectionMap; + private final ReentrantLock lock = new ReentrantLock(); + private static final AtomicBoolean kerberosEnvInit = new AtomicBoolean(false); + private static final int KERBEROS_RE_LOGIN_MAX_RETRY = 5; + private static final long KERBEROS_RE_LOGIN_INTERVAL = 30 * 60 * 1000L; + private static volatile HBaseConnectionManager instance = null; + + private HBaseConnectionManager() { + connectionMap = new ConcurrentHashMap<>(); + } + + public static HBaseConnectionManager getInstance() { + if (instance == null) { + synchronized (HBaseConnectionManager.class) { + if (instance == null) { + instance = new HBaseConnectionManager(); + } + } + } + return instance; + } + + public Connection getConnection(Properties prop) { + Map propMap = new HashMap<>(); + if (prop == null) { + return getConnection(propMap); + } + for (String key : prop.stringPropertyNames()) { + propMap.put(key, prop.getProperty(key)); + } + return getConnection(propMap); + } + + public Connection getConnection(Map prop) { + if (prop == null) { + prop = new HashMap<>(0); + } + Configuration configuration = buildConfiguration(prop); + String clusterConnUniqueKey = generateUniqueConnectionKey(configuration, prop); + LOG.info("Start to get connection for cluster {}.", clusterConnUniqueKey); + try { + lock.lock(); + if (!connectionMap.containsKey(clusterConnUniqueKey)) { + if (isKerberosAuthType(prop) && kerberosEnvInit.compareAndSet(false, true)) { + doKerberosLogin(configuration, prop); + } + Connection connection; + String proxyUser = getKerberosProxyUser(prop); + UserGroupInformation kerberosLoginUser = UserGroupInformation.getLoginUser(); + String kerberosLoginShortUserName = kerberosLoginUser.getShortUserName(); + if (StringUtils.isNotBlank(proxyUser) && !proxyUser.equals(kerberosLoginShortUserName)) { + UserGroupInformation ugi = UserGroupInformation.createProxyUser(proxyUser, kerberosLoginUser); + connection = ugi.doAs((PrivilegedAction) () -> { + try { + return ConnectionFactory.createConnection(configuration); + } catch (IOException e) { + LOG.error(HBaseErrorCodeSummary.HBASE_CLIENT_CONN_CREATE_FAILED.getErrorDesc(), e); + throw new JobExecutorException( + HBaseErrorCodeSummary.HBASE_CLIENT_CONN_CREATE_FAILED.getErrorCode(), + HBaseErrorCodeSummary.HBASE_CLIENT_CONN_CREATE_FAILED.getErrorDesc()); + } + }); + LOG.info("Successfully create a connection {} and proxy user {}", connection, proxyUser); + } else { + connection = ConnectionFactory.createConnection(configuration); + LOG.info("Successfully create a connection {}.", connection); + } + connectionMap.put(clusterConnUniqueKey, connection); + return connection; + } + } catch (IOException e) { + LOG.error(HBaseErrorCodeSummary.HBASE_CLIENT_CONN_CREATE_FAILED.getErrorDesc(), e); + throw new JobExecutorException(HBaseErrorCodeSummary.HBASE_CLIENT_CONN_CREATE_FAILED.getErrorCode(), + HBaseErrorCodeSummary.HBASE_CLIENT_CONN_CREATE_FAILED.getErrorDesc()); + } finally { + lock.unlock(); + } + return connectionMap.get(clusterConnUniqueKey); + } + + private void doKerberosLogin(Configuration configuration, Map prop) { + String principal = getKerberosPrincipal(prop); + String keytab = getKerberosKeytabFile(prop); + File file = new File(keytab); + if (!file.exists()) { + kerberosEnvInit.set(false); + throw new HBaseParamsIllegalException(HBaseErrorCodeSummary.KERBEROS_KEYTAB_FILE_NOT_EXISTS.getErrorCode(), + HBaseErrorCodeSummary.KERBEROS_KEYTAB_FILE_NOT_EXISTS.getErrorDesc()); + } + if (!file.isFile()) { + kerberosEnvInit.set(false); + throw new HBaseParamsIllegalException(HBaseErrorCodeSummary.KERBEROS_KEYTAB_NOT_FILE.getErrorCode(), + HBaseErrorCodeSummary.KERBEROS_KEYTAB_NOT_FILE.getErrorDesc()); + } + try { + UserGroupInformation.setConfiguration(configuration); + UserGroupInformation.loginUserFromKeytab(principal, keytab); + LOG.info("Login successfully via keytab: {} and principal: {}", keytab, principal); + doKerberosReLogin(); + } catch (IOException e) { + kerberosEnvInit.set(false); + throw new JobExecutorException(HBaseErrorCodeSummary.KERBEROS_AUTH_FAILED.getErrorCode(), + HBaseErrorCodeSummary.KERBEROS_AUTH_FAILED.getErrorDesc()); + } + } + + private boolean runKerberosLogin() { + Configuration conf = new org.apache.hadoop.conf.Configuration(); + conf.set("hadoop.security.authentication", KERBEROS); + UserGroupInformation.setConfiguration(conf); + try { + if (UserGroupInformation.isLoginKeytabBased()) { + LOG.info("Trying re login from keytab."); + UserGroupInformation.getLoginUser().reloginFromKeytab(); + return true; + } else if (UserGroupInformation.isLoginTicketBased()) { + LOG.info("Trying re login from ticket cache"); + UserGroupInformation.getLoginUser().reloginFromTicketCache(); + return true; + } + } catch (Exception e) { + LOG.error("Unable to run kinit.", e); + } + return false; + } + + private void doKerberosReLogin() { + if (!UserGroupInformation.isSecurityEnabled()) { + return; + } + + Thread reLoginThread = new Thread(new Runnable() { + @Override + public void run() { + while (true) { + int times = 0; + + while (times < KERBEROS_RE_LOGIN_MAX_RETRY) { + if (runKerberosLogin()) { + LOG.info("Ran kerberos re login command successfully."); + break; + } else { + times++; + LOG.info("Run kerberos re login failed for {} time(s).", times); + } + } + try { + Thread.sleep(KERBEROS_RE_LOGIN_INTERVAL); + } catch (InterruptedException e) { + LOG.warn("Ignore error", e); + } + } + } + }); + reLoginThread.setName("KerberosReLoginThread"); + reLoginThread.setDaemon(true); + reLoginThread.start(); + } + + private Configuration buildConfiguration(Map prop) { + Configuration configuration = HBaseConfiguration.create(); + if (prop.isEmpty()) { + return configuration; + } + String zkQuorum = HBasePropertiesParser.getString(prop, ZOOKEEPER_QUORUM, DEFAULT_ZOOKEEPER_QUORUM); + configuration.set(HConstants.ZOOKEEPER_QUORUM, zkQuorum); + int zkClientPort = HBasePropertiesParser.getInt(prop, ZOOKEEPER_CLIENT_PORT, DEFAULT_ZOOKEEPER_CLIENT_PORT); + configuration.set(HConstants.ZOOKEEPER_CLIENT_PORT, String.valueOf(zkClientPort)); + String zNodeParent = + HBasePropertiesParser.getString(prop, ZOOKEEPER_NODE_PARENT, DEFAULT_ZOOKEEPER_NODE_PARENT); + configuration.set(HConstants.ZOOKEEPER_ZNODE_PARENT, zNodeParent); + String dfsRootDir = HBasePropertiesParser.getString(prop, HBASE_DFS_ROOT_DIR, DEFAULT_HBASE_DFS_ROOT_DIR); + configuration.set(HConstants.HBASE_DIR, dfsRootDir); + if (isKerberosAuthType(prop)) { + configuration.set(HBASE_AUTH, KERBEROS); + configuration.set(HADOOP_SECURITY_AUTH, KERBEROS); + String regionServerPrincipal = + HBasePropertiesParser.getString(prop, HBASE_REGION_SERVER_KERBEROS_PRINCIPAL, ""); + if (StringUtils.isBlank(regionServerPrincipal)) { + throw new HBaseParamsIllegalException( + HBaseErrorCodeSummary.REGION_SERVER_KERBEROS_PRINCIPAL_NOT_NULL.getErrorCode(), + HBaseErrorCodeSummary.REGION_SERVER_KERBEROS_PRINCIPAL_NOT_NULL.getErrorDesc()); + } + configuration.set(REGION_SERVER_KERBEROS_PRINCIPAL, regionServerPrincipal); + String masterPrincipal = HBasePropertiesParser.getString(prop, HBASE_MASTER_KERBEROS_PRINCIPAL, ""); + if (StringUtils.isBlank(masterPrincipal)) { + throw new HBaseParamsIllegalException( + HBaseErrorCodeSummary.MASTER_KERBEROS_PRINCIPAL_NOT_NULL.getErrorCode(), + HBaseErrorCodeSummary.MASTER_KERBEROS_PRINCIPAL_NOT_NULL.getErrorDesc()); + } + configuration.set(MASTER_SERVER_KERBEROS_PRINCIPAL, masterPrincipal); + String krb5Conf = HBasePropertiesParser.getString(prop, KRB5_CONF_PATH, DEFAULT_KRB5_CONF_PATH); + System.setProperty(KRB5_CONF_PATH, krb5Conf); + } + return configuration; + } + + private String getSecurityAuth(Map prop) { + return HBasePropertiesParser.getString(prop, HBASE_SECURITY_AUTH, SIMPLE); + } + + private boolean isKerberosAuthType(Map prop) { + String authType = getSecurityAuth(prop); + if (StringUtils.isBlank(authType)) { + return false; + } + return KERBEROS.equalsIgnoreCase(authType.trim()); + } + + private String getKerberosPrincipal(Map prop) { + String kerberosPrincipal = HBasePropertiesParser.getString(prop, KERBEROS_PRINCIPAL, ""); + if (StringUtils.isBlank(kerberosPrincipal)) { + throw new HBaseParamsIllegalException(HBaseErrorCodeSummary.KERBEROS_PRINCIPAL_NOT_NULL.getErrorCode(), + HBaseErrorCodeSummary.KERBEROS_PRINCIPAL_NOT_NULL.getErrorDesc()); + } + return kerberosPrincipal; + } + + private String getKerberosKeytabFile(Map prop) { + String keytabFile = HBasePropertiesParser.getString(prop, KERBEROS_KEYTAB_FILE, ""); + if (StringUtils.isBlank(keytabFile)) { + throw new HBaseParamsIllegalException(HBaseErrorCodeSummary.KERBEROS_KEYTAB_NOT_NULL.getErrorCode(), + HBaseErrorCodeSummary.KERBEROS_KEYTAB_NOT_NULL.getErrorDesc()); + } + return keytabFile; + } + + private String generateUniqueConnectionKey(Configuration configuration, Map prop) { + String zkQuorum = configuration.get(HConstants.ZOOKEEPER_QUORUM); + String zkClientPort = configuration.get(HConstants.ZOOKEEPER_CLIENT_PORT); + StringBuilder sb = new StringBuilder(zkQuorum); + sb.append(UNIQUE_KEY_DELIMITER); + sb.append(zkClientPort); + if (supportKerberosProxyUser(prop)) { + sb.append(UNIQUE_KEY_DELIMITER); + sb.append(getKerberosProxyUser(prop)); + } + return sb.toString(); + } + + public String generateUniqueConnectionKey(Map prop) { + Configuration configuration = buildConfiguration(prop); + return generateUniqueConnectionKey(configuration, prop); + } + + private boolean supportKerberosProxyUser(Map prop) { + if (!isKerberosAuthType(prop)) { + return false; + } + String proxyUser = getKerberosProxyUser(prop); + return StringUtils.isNotBlank(proxyUser); + } + + private String getKerberosProxyUser(Map prop) { + if (prop == null || prop.isEmpty()) { + return ""; + } + return HBasePropertiesParser.getString(prop, KERBEROS_PROXY_USER, ""); + } + + public void destroy() { + try { + for (Connection connection : connectionMap.values()) { + connection.close(); + } + connectionMap.clear(); + } catch (IOException e) { + LOG.warn("An exception occurred while destroy resources.", e); + } + + } + +} diff --git a/linkis-engineconn-plugins/hbase/src/main/java/org/apache/linkis/manager/engineplugin/hbase/HBasePropertiesParser.java b/linkis-engineconn-plugins/hbase/src/main/java/org/apache/linkis/manager/engineplugin/hbase/HBasePropertiesParser.java new file mode 100644 index 0000000000..34cbf9af6c --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/java/org/apache/linkis/manager/engineplugin/hbase/HBasePropertiesParser.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.engineplugin.hbase; + +import java.util.Map; + +public class HBasePropertiesParser extends PropertiesParser { + public static long getLong(Map prop, String key, long defaultValue) { + return getValue(prop, key, defaultValue, Long::parseLong); + } + + public static int getInt(Map prop, String key, int defaultValue) { + return getValue(prop, key, defaultValue, Integer::parseInt); + } + + public static boolean getBool(Map prop, String key, boolean defaultValue) { + return getValue(prop, key, defaultValue, "true"::equalsIgnoreCase); + } +} diff --git a/linkis-engineconn-plugins/hbase/src/main/java/org/apache/linkis/manager/engineplugin/hbase/PropertiesParser.java b/linkis-engineconn-plugins/hbase/src/main/java/org/apache/linkis/manager/engineplugin/hbase/PropertiesParser.java new file mode 100644 index 0000000000..06b9c53c56 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/java/org/apache/linkis/manager/engineplugin/hbase/PropertiesParser.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.engineplugin.hbase; + +import java.util.Map; + +import org.apache.commons.lang3.StringUtils; + +public abstract class PropertiesParser { + interface TypeConversion { + /** + * String type data is converted to T type + * + * @param oriV origin type + * @return T which is target type + */ + T convertTo(String oriV); + } + + public static String getString(Map prop, String key, String defaultValue) { + return prop.getOrDefault(key, defaultValue); + } + + public static T getValue( + Map prop, String key, T defaultValue, TypeConversion typeConversion) { + String valueStr = getString(prop, key, ""); + if (StringUtils.isBlank(valueStr)) { + return defaultValue; + } + try { + return typeConversion.convertTo(valueStr); + } catch (Exception e) { + return defaultValue; + } + } +} diff --git a/linkis-engineconn-plugins/hbase/src/main/java/org/apache/linkis/manager/engineplugin/hbase/constant/HBaseEngineConnConstant.java b/linkis-engineconn-plugins/hbase/src/main/java/org/apache/linkis/manager/engineplugin/hbase/constant/HBaseEngineConnConstant.java new file mode 100644 index 0000000000..c39bc47f23 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/java/org/apache/linkis/manager/engineplugin/hbase/constant/HBaseEngineConnConstant.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.engineplugin.hbase.constant; + +import org.apache.hadoop.hbase.HConstants; + +public class HBaseEngineConnConstant { + private HBaseEngineConnConstant() {} + + private static final String LINKIS_PREFIX = "wds.linkis."; + + public static final String ZOOKEEPER_QUORUM = LINKIS_PREFIX + HConstants.ZOOKEEPER_QUORUM; + public static final String DEFAULT_ZOOKEEPER_QUORUM = "localhost"; + public static final String ZOOKEEPER_CLIENT_PORT = LINKIS_PREFIX + HConstants.ZOOKEEPER_CLIENT_PORT; + public static final int DEFAULT_ZOOKEEPER_CLIENT_PORT = 2181; + public static final String ZOOKEEPER_NODE_PARENT = LINKIS_PREFIX + HConstants.ZOOKEEPER_ZNODE_PARENT; + public static final String DEFAULT_ZOOKEEPER_NODE_PARENT = "/hbase"; + public static final String HBASE_DFS_ROOT_DIR = LINKIS_PREFIX + HConstants.HBASE_DIR; + public static final String DEFAULT_HBASE_DFS_ROOT_DIR = "/hbase"; + public static final String HBASE_AUTH = "hbase.security.authentication"; + public static final String HADOOP_SECURITY_AUTH = "hadoop.security.authentication"; + public static final String HBASE_SECURITY_AUTH = LINKIS_PREFIX + HBASE_AUTH; + public static final String KERBEROS = "kerberos"; + public static final String SIMPLE = "simple"; + public static final String KERBEROS_PRINCIPAL = LINKIS_PREFIX + "hbase.kerberos.principal"; + public static final String KERBEROS_KEYTAB_FILE = LINKIS_PREFIX + "hbase.keytab.file"; + public static final String KERBEROS_PROXY_USER = LINKIS_PREFIX + "hbase.kerberos.proxy.user"; + public static final String REGION_SERVER_KERBEROS_PRINCIPAL = "hbase.regionserver.kerberos.principal"; + public static final String HBASE_REGION_SERVER_KERBEROS_PRINCIPAL = LINKIS_PREFIX + REGION_SERVER_KERBEROS_PRINCIPAL; + public static final String MASTER_SERVER_KERBEROS_PRINCIPAL = "hbase.master.kerberos.principal"; + public static final String HBASE_MASTER_KERBEROS_PRINCIPAL = LINKIS_PREFIX + MASTER_SERVER_KERBEROS_PRINCIPAL; + public static final String KRB5_CONF_PATH = "java.security.krb5.conf"; + public static final String DEFAULT_KRB5_CONF_PATH = "/etc/krb5.conf"; + public static final String UNIQUE_KEY_DELIMITER = "#"; + + public static final String HBASE_SHELL_SESSION_INIT_TIMEOUT_MS = LINKIS_PREFIX + "hbase.shell.session.init.timeout.ms"; + public static final long DEFAULT_SHELL_SESSION_INIT_TIMEOUT_MS = 2 * 60 * 1000L; + + public static final String HBASE_SHELL_SESSION_INIT_MAX_TIMES = LINKIS_PREFIX + "hbase.shell.session.init.max.times"; + public static final int DEFAULT_SHELL_SESSION_INIT_MAX_TIMES = 10; + + public static final String HBASE_SHELL_SESSION_INIT_RETRY_INTERVAL_MS = LINKIS_PREFIX + "hbase.shell.session.init.retry.interval"; + public static final long DEFAULT_SHELL_SESSION_INIT_RETRY_INTERVAL_MS = 500L; + + public static final String HBASE_SHELL_SESSION_IDLE_MS = LINKIS_PREFIX + "hbase.shell.session.idle"; + public static final long DEFAULT_SHELL_SESSION_IDLE_MS = 2 * 60 * 60 * 1000L; + + public static final String HBASE_SHELL_DEBUG_LOG = LINKIS_PREFIX + "hbase.shell.session.debug.log"; + public static final boolean DEFAULT_SHELL_DEBUG_LOG = false; +} diff --git a/linkis-engineconn-plugins/hbase/src/main/java/org/apache/linkis/manager/engineplugin/hbase/errorcode/HBaseErrorCodeSummary.java b/linkis-engineconn-plugins/hbase/src/main/java/org/apache/linkis/manager/engineplugin/hbase/errorcode/HBaseErrorCodeSummary.java new file mode 100644 index 0000000000..1c8bec963f --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/java/org/apache/linkis/manager/engineplugin/hbase/errorcode/HBaseErrorCodeSummary.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.engineplugin.hbase.errorcode; + +import org.apache.linkis.common.errorcode.ErrorCodeUtils; +import org.apache.linkis.common.errorcode.LinkisErrorCode; + +public enum HBaseErrorCodeSummary implements LinkisErrorCode { + KERBEROS_PRINCIPAL_NOT_NULL( + 27000, + "In the hbase authentication mode of kerberos, the kerberos principal cannot be empty(kerberos的hbase认证方式下，kerberos principal不能为空)"), + KERBEROS_KEYTAB_NOT_NULL( + 27001, + "In the hbase authentication mode of kerberos, the kerberos keytab cannot be empty(kerberos的hbase认证方式下，kerberos keytab不能为空)"), + KERBEROS_KEYTAB_FILE_NOT_EXISTS( + 27002, + "The kerberos keytab file must exists(kerberos keytab文件必须存在)"), + KERBEROS_KEYTAB_NOT_FILE( + 27003, + "The kerberos keytab file must be a file(kerberos keytab文件必须是个文件)"), + KERBEROS_AUTH_FAILED( + 27004, + "kerberos authentication failed(kerberos 认证失败)"), + REGION_SERVER_KERBEROS_PRINCIPAL_NOT_NULL( + 27005, + "In the hbase authentication mode of kerberos, the region server kerberos principal cannot be empty(kerberos的hbase认证方式下，region server kerberos principal不能为空)"), + MASTER_KERBEROS_PRINCIPAL_NOT_NULL( + 27006, + "In the hbase authentication mode of kerberos, the hmaster kerberos principal cannot be empty(kerberos的hbase认证方式下，hmaster kerberos principal不能为空)"), + HBASE_CLIENT_CONN_CREATE_FAILED( + 27007, + "HBase client connection failed to be created(HBase客户端连接创建失败)"), + HBASE_SHELL_ENV_INIT_FAILED( + 27008, + "HBase shell environment initialization failed(HBase shell环境初始化失败)"); + + private final int errorCode; + + private final String errorDesc; + + HBaseErrorCodeSummary(int errorCode, String errorDesc) { + ErrorCodeUtils.validateErrorCode(errorCode, 26000, 29999); + this.errorCode = errorCode; + this.errorDesc = errorDesc; + } + + @Override + public int getErrorCode() { + return errorCode; + } + + @Override + public String getErrorDesc() { + return errorDesc; + } +} diff --git a/linkis-engineconn-plugins/hbase/src/main/java/org/apache/linkis/manager/engineplugin/hbase/shell/HBaseShellCommands.java b/linkis-engineconn-plugins/hbase/src/main/java/org/apache/linkis/manager/engineplugin/hbase/shell/HBaseShellCommands.java new file mode 100644 index 0000000000..ac17b15183 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/java/org/apache/linkis/manager/engineplugin/hbase/shell/HBaseShellCommands.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.engineplugin.hbase.shell; + +import java.io.File; +import java.io.IOException; +import java.net.JarURLConnection; +import java.net.URL; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class HBaseShellCommands { + private final static Logger LOG = LoggerFactory.getLogger(HBaseShellCommands.class); + private final static String COMMANDS_PATH = "hbase-ruby/shell/commands/"; + private volatile static Set commandsSet; + private HBaseShellCommands () { + + } + public static Set getAllCommands() throws IOException { + if (commandsSet == null) { + synchronized (HBaseShellCommands.class) { + if (commandsSet == null) { + Set sortedSet = new TreeSet<>(); + URL commandFilesUrl = HBaseShellCommands.class.getClassLoader().getResource(COMMANDS_PATH); + if (commandFilesUrl == null) { + throw new IOException("The command files path is null!"); + } + String commandFilePath = commandFilesUrl.getPath(); + File commandFile = new File(commandFilePath); + if (!commandFile.exists()) { + LOG.warn("The command files path is not exists, starting read file from jar."); + String jarPath = commandFilesUrl.toString().substring(0, commandFilesUrl.toString().indexOf("!/") + 2); + LOG.info("The path in jar is " + jarPath); + URL jarUrl = new URL(jarPath); + JarURLConnection jarCon = (JarURLConnection) jarUrl.openConnection(); + JarFile jarFile = jarCon.getJarFile(); + Enumeration jarEntries = jarFile.entries(); + while (jarEntries.hasMoreElements()) { + JarEntry entry = jarEntries.nextElement(); + String name = entry.getName(); + if (!entry.isDirectory() && name.startsWith(COMMANDS_PATH)) { + String commandName = name.substring(name.lastIndexOf(File.separator) + 1, + name.lastIndexOf(".rb")); + sortedSet.add(commandName); + } + } + + } else { + String[] files = commandFile.list(); + if (files == null) { + throw new IOException("The command files is null!"); + } + for (String file : files) { + if (file.endsWith(".rb")){ + sortedSet.add(file.substring(0, file.lastIndexOf(".rb"))); + } + } + } + + commandsSet = sortedSet; + } + } + } + return commandsSet; + } + + public static List searchCommand(String subCommand) { + List matchCommands = new ArrayList<>(); + + try { + Set allCommands = getAllCommands(); + for (String command : allCommands) { + if (command.startsWith(subCommand)) { + matchCommands.add(command); + } + } + } catch (IOException e) { + return matchCommands; + } + return matchCommands; + } +} diff --git a/linkis-engineconn-plugins/hbase/src/main/java/org/apache/linkis/manager/engineplugin/hbase/shell/HBaseShellSession.java b/linkis-engineconn-plugins/hbase/src/main/java/org/apache/linkis/manager/engineplugin/hbase/shell/HBaseShellSession.java new file mode 100644 index 0000000000..cdcc9c8973 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/java/org/apache/linkis/manager/engineplugin/hbase/shell/HBaseShellSession.java @@ -0,0 +1,311 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.engineplugin.hbase.shell; + +import static org.apache.linkis.manager.engineplugin.hbase.errorcode.HBaseErrorCodeSummary.HBASE_SHELL_ENV_INIT_FAILED; + +import java.io.IOException; +import java.io.InputStream; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import org.apache.commons.lang3.StringUtils; +import org.apache.linkis.manager.engineplugin.hbase.exception.ExecutorInitException; +import org.jruby.embed.LocalContextScope; +import org.jruby.embed.ScriptingContainer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class HBaseShellSession implements ShellSession { + private static final Logger LOGGER = LoggerFactory.getLogger(HBaseShellSession.class); + private static final String SESSION_PROP_SEPARATOR = "$"; + + private final String sessionId; + private final int sessionInitMaxTimes; + private final long sessionInitRetryInterval; + private final long sessionInitTimeout; + private final long sessionIdle; + private final long sessionInitStartTime; + private final boolean sessionDebugLog; + private final Properties properties; + + private ScriptingContainer scriptingContainer; + private StringWriter writer; + private boolean isConnected; + + public HBaseShellSession(Builder builder) { + this.properties = builder.properties; + this.sessionId = builder.sessionId; + this.sessionInitMaxTimes = builder.sessionInitMaxTimes; + this.sessionInitRetryInterval = builder.sessionInitRetryInterval; + this.sessionInitTimeout = builder.sessionInitTimeout; + this.sessionIdle = builder.sessionIdle; + this.sessionInitStartTime = System.currentTimeMillis(); + this.sessionDebugLog = builder.sessionDebugLog; + } + + static class Builder { + private String sessionId; + private Map properties; + private int sessionInitMaxTimes; + private long sessionInitRetryInterval; + private long sessionInitTimeout; + private long sessionIdle; + private boolean sessionDebugLog; + + public Builder sessionId(String sessionId) { + this.sessionId = sessionId; + return this; + } + + public Builder properties(Map properties) { + this.properties = properties; + return this; + } + + public Builder sessionInitMaxTimes(int sessionInitMaxTimes) { + this.sessionInitMaxTimes = sessionInitMaxTimes; + return this; + } + + public Builder sessionInitRetryInterval(long sessionInitRetryInterval) { + this.sessionInitRetryInterval = sessionInitRetryInterval; + return this; + } + + public Builder sessionInitTimeout(long sessionInitTimeout) { + this.sessionInitTimeout = sessionInitTimeout; + return this; + } + + public Builder sessionIdle(long sessionIdle) { + this.sessionIdle = sessionIdle; + return this; + } + + public Builder sessionDebugLog(boolean sessionDebugLog) { + this.sessionDebugLog = sessionDebugLog; + return this; + } + + public Builder properties(String key, String value) { + if (this.properties == null) { + this.properties = new HashMap<>(); + } + this.properties.put(key, value); + return this; + } + + public HBaseShellSession build() { + return new HBaseShellSession(this); + } + } + + public static Builder sessionBuilder() { + return new Builder(); + } + + @Override + public void open() { + Thread t = new Thread(() -> { + int initMaxTimes = this.getSessionInitMaxTimes(); + + try { + LOGGER.info("Starting create hbase shell session ......"); + createShellRunningEnv(); + } catch (Exception e) { + for (int i = 0; i < initMaxTimes; i++) { + try { + createShellRunningEnv(); + } catch (Exception ex) { + if (i == (initMaxTimes - 1)) { + LOGGER.error("After {} retries, HBase shell session initialization failed.", initMaxTimes, ex); + throw new ExecutorInitException(HBASE_SHELL_ENV_INIT_FAILED.getErrorCode(), + HBASE_SHELL_ENV_INIT_FAILED.getErrorDesc()); + } + shortSpin(this.getSessionInitRetryInterval()); + } + } + } + }); + t.setName("HBaseShellRunningEnvInitThread"); + t.setDaemon(true); + t.start(); + shortSpin(10000); + + CompletableFuture future = CompletableFuture.supplyAsync(this::waitShellSessionConnected); + try { + this.isConnected = future.get(this.getSessionInitTimeout(), TimeUnit.MILLISECONDS); + LOGGER.info("Created hbase shell session successfully."); + } catch (InterruptedException | ExecutionException e) { + this.isConnected = false; + future.cancel(true); + LOGGER.error("Initialize hbase shell session failed.", e); + this.destroy(); + } catch (TimeoutException e) { + LOGGER.error("Initialize hbase shell session timeout.", e); + this.isConnected = false; + future.cancel(true); + this.destroy(); + } + } + + private void shortSpin(long interval) { + if (interval <= 0) { + return; + } + try { + Thread.sleep(interval); + } catch (InterruptedException e) { + LOGGER.warn("Ignore error.", e); + } + } + + private void createShellRunningEnv() throws IOException { + this.scriptingContainer = new ScriptingContainer(LocalContextScope.SINGLETHREAD); + this.writer = new StringWriter(); + scriptingContainer.setOutput(this.writer); + Properties sysProps = System.getProperties(); + String prop = ""; + if (this.isSessionDebugLog()) { + prop = "-d".concat(SESSION_PROP_SEPARATOR); + } + if (properties != null && !properties.isEmpty()) { + StringBuilder sb = new StringBuilder(); + for (String key : properties.stringPropertyNames()) { + sb.append("-D"); + sb.append(key); + sb.append("="); + sb.append(properties.getProperty(key)); + sb.append(SESSION_PROP_SEPARATOR); + } + prop = prop + sb.substring(0, sb.length() - 1); + } + if (StringUtils.isNotBlank(prop)) { + sysProps.setProperty("hbase.ruby.args", prop); + } + try (InputStream in = this.getClass().getClassLoader().getResourceAsStream("hbase-ruby/hirb.rb")) { + this.scriptingContainer.runScriptlet(in, "hirb.rb"); + } + } + + private boolean waitShellSessionConnected() { + while (true) { + Result result = executeCmd("list_namespace"); + String r = result.getResult(); + if (result.isSuccess() && StringUtils.isNotBlank(r)) { + return true; + } + shortSpin(200L); + } + } + + @Override + public Result execute(String cmd) { + if (!this.isConnected()) { + return Result.failed(String.format("The current session [%s] is not connected successfully," + + " please try again.", this)); + } + return executeCmd(cmd); + } + + @Override + public void destroy() { + if (this.scriptingContainer != null) { + this.scriptingContainer.terminate(); + } + this.setConnected(false); + LOGGER.info("The hbase shell session destroy successfully."); + } + + private Result executeCmd(String cmd) { + try { + this.writer.getBuffer().setLength(0); + Object o = this.scriptingContainer.runScriptlet(cmd); + this.writer.flush(); + String res = writer.toString(); + if (StringUtils.isBlank(res) && o != null) { + res = o.toString(); + } + return Result.ok(res); + } catch (Exception e) { + return Result.failed(getStackTrace(e)); + } + } + + public String getStackTrace(Throwable throwable) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw, true); + throwable.printStackTrace(pw); + return sw.getBuffer().toString(); + } + + public String getSessionId() { + return sessionId; + } + + public Properties getProperties() { + return properties; + } + + public int getSessionInitMaxTimes() { + return sessionInitMaxTimes; + } + + public long getSessionInitRetryInterval() { + return sessionInitRetryInterval; + } + + public long getSessionInitTimeout() { + return sessionInitTimeout; + } + + public boolean isConnected() { + return isConnected; + } + + public void setConnected(boolean connected) { + isConnected = connected; + } + + public long getSessionIdle() { + return sessionIdle; + } + + public long getSessionInitStartTime() { + return sessionInitStartTime; + } + + public boolean isSessionDebugLog() { + return sessionDebugLog; + } + + @Override + public String toString() { + return this.getSessionId(); + } + + +} diff --git a/linkis-engineconn-plugins/hbase/src/main/java/org/apache/linkis/manager/engineplugin/hbase/shell/HBaseShellSessionConfig.java b/linkis-engineconn-plugins/hbase/src/main/java/org/apache/linkis/manager/engineplugin/hbase/shell/HBaseShellSessionConfig.java new file mode 100644 index 0000000000..4897a0d0ff --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/java/org/apache/linkis/manager/engineplugin/hbase/shell/HBaseShellSessionConfig.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.engineplugin.hbase.shell; + +import static org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant.DEFAULT_SHELL_DEBUG_LOG; +import static org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant.DEFAULT_SHELL_SESSION_IDLE_MS; +import static org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant.DEFAULT_SHELL_SESSION_INIT_MAX_TIMES; +import static org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant.DEFAULT_SHELL_SESSION_INIT_RETRY_INTERVAL_MS; +import static org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant.DEFAULT_SHELL_SESSION_INIT_TIMEOUT_MS; +import static org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant.HBASE_SHELL_DEBUG_LOG; +import static org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant.HBASE_SHELL_SESSION_IDLE_MS; +import static org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant.HBASE_SHELL_SESSION_INIT_MAX_TIMES; +import static org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant.HBASE_SHELL_SESSION_INIT_RETRY_INTERVAL_MS; +import static org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant.HBASE_SHELL_SESSION_INIT_TIMEOUT_MS; + +import java.util.Map; + +import org.apache.linkis.manager.engineplugin.hbase.HBasePropertiesParser; + +public class HBaseShellSessionConfig { + public static long initTimeout(Map prop) { + return HBasePropertiesParser.getLong(prop, HBASE_SHELL_SESSION_INIT_TIMEOUT_MS, + DEFAULT_SHELL_SESSION_INIT_TIMEOUT_MS); + } + + public static int maxRetryTimes(Map prop) { + return HBasePropertiesParser.getInt(prop, HBASE_SHELL_SESSION_INIT_MAX_TIMES, + DEFAULT_SHELL_SESSION_INIT_MAX_TIMES); + } + + public static long initRetryInterval(Map prop) { + return HBasePropertiesParser.getLong(prop, HBASE_SHELL_SESSION_INIT_RETRY_INTERVAL_MS, + DEFAULT_SHELL_SESSION_INIT_RETRY_INTERVAL_MS); + } + + public static long idleTimeMs(Map prop) { + return HBasePropertiesParser.getLong(prop, HBASE_SHELL_SESSION_IDLE_MS, DEFAULT_SHELL_SESSION_IDLE_MS); + } + + public static boolean openDebugLog(Map prop) { + return HBasePropertiesParser.getBool(prop, HBASE_SHELL_DEBUG_LOG, DEFAULT_SHELL_DEBUG_LOG); + } +} diff --git a/linkis-engineconn-plugins/hbase/src/main/java/org/apache/linkis/manager/engineplugin/hbase/shell/HBaseShellSessionManager.java b/linkis-engineconn-plugins/hbase/src/main/java/org/apache/linkis/manager/engineplugin/hbase/shell/HBaseShellSessionManager.java new file mode 100644 index 0000000000..7dcd5293ee --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/java/org/apache/linkis/manager/engineplugin/hbase/shell/HBaseShellSessionManager.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.engineplugin.hbase.shell; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.linkis.manager.engineplugin.hbase.HBaseConnectionManager; + +public class HBaseShellSessionManager { + private volatile static Map shellSessionMap; + + public static HBaseShellSession getHBaseShellSession(Map prop) { + String sessionId = HBaseConnectionManager.getInstance().generateUniqueConnectionKey(prop); + + if (shellSessionMap == null || !shellSessionMap.containsKey(sessionId)) { + synchronized (HBaseShellSessionManager.class) { + if (shellSessionMap == null || !shellSessionMap.containsKey(sessionId)) { + if (shellSessionMap == null) { + shellSessionMap = new HashMap<>(2); + } + if (!shellSessionMap.containsKey(sessionId)) { + HBaseShellSession shellSession = HBaseShellSession.sessionBuilder() + .sessionId(sessionId) + .sessionInitMaxTimes(HBaseShellSessionConfig.maxRetryTimes(prop)) + .sessionInitRetryInterval(HBaseShellSessionConfig.initRetryInterval(prop)) + .sessionInitTimeout(HBaseShellSessionConfig.initTimeout(prop)) + .sessionIdle(HBaseShellSessionConfig.idleTimeMs(prop)) + .sessionDebugLog(HBaseShellSessionConfig.openDebugLog(prop)) + .properties(prop) + .build(); + shellSession.open(); + shellSessionMap.put(sessionId, shellSession); + return shellSession; + } + } + } + } + return shellSessionMap.get(sessionId); + } + +} diff --git a/linkis-engineconn-plugins/hbase/src/main/java/org/apache/linkis/manager/engineplugin/hbase/shell/Result.java b/linkis-engineconn-plugins/hbase/src/main/java/org/apache/linkis/manager/engineplugin/hbase/shell/Result.java new file mode 100644 index 0000000000..93f611a89b --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/java/org/apache/linkis/manager/engineplugin/hbase/shell/Result.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.engineplugin.hbase.shell; + +public final class Result { + private final boolean success; + private final String result; + + public Result(boolean success, String result) { + this.success = success; + this.result = result; + } + + public static Result of(boolean success, String message) { + return new Result(success, message); + } + + public static Result ok(String message) { + return Result.of(true, message); + } + + public static Result failed(String message) { + return Result.of(false, message); + } + + public static Result ok() { + return Result.of(true, "ok"); + } + + public static Result failed() { + return Result.of(false, "error"); + } + + public boolean isSuccess() { + return success; + } + + public String getResult() { + return result; + } + + @Override + public String toString() { + return "Result{" + + "success=" + success + + ", result='" + result + '\'' + + '}'; + } +} diff --git a/linkis-engineconn-plugins/hbase/src/main/java/org/apache/linkis/manager/engineplugin/hbase/shell/ShellSession.java b/linkis-engineconn-plugins/hbase/src/main/java/org/apache/linkis/manager/engineplugin/hbase/shell/ShellSession.java new file mode 100644 index 0000000000..ef09d6ba40 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/java/org/apache/linkis/manager/engineplugin/hbase/shell/ShellSession.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.engineplugin.hbase.shell; + +public interface ShellSession { + void open(); + + Result execute(String cmd); + + void destroy(); +} diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase/admin.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase/admin.rb new file mode 100644 index 0000000000..97d1ec5487 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase/admin.rb @@ -0,0 +1,1826 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include Java +java_import java.util.Arrays +java_import java.util.regex.Pattern +java_import org.apache.hadoop.hbase.util.Pair +java_import org.apache.hadoop.hbase.util.RegionSplitter +java_import org.apache.hadoop.hbase.util.Bytes +java_import org.apache.hadoop.hbase.ServerName +java_import org.apache.hadoop.hbase.TableName +java_import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder +java_import org.apache.hadoop.hbase.HConstants + +require 'hbase/balancer_utils' + +# Wrapper for org.apache.hadoop.hbase.client.HBaseAdmin + +module Hbase + # rubocop:disable Metrics/ClassLength + class Admin + include HBaseConstants + + def initialize(connection) + @connection = connection + # Java Admin instance + @admin = @connection.getAdmin + @hbck = @connection.getHbck + @conf = @connection.getConfiguration + end + + def close + @admin.close + end + + #---------------------------------------------------------------------------------------------- + # Returns a list of tables in hbase + def list(regex = '.*') + @admin.listTableNames(Pattern.compile(regex)).map(&:getNameAsString) + end + + #---------------------------------------------------------------------------------------------- + # Requests a table or region or region server flush + def flush(name, family = nil) + family_bytes = nil + family_bytes = family.to_java_bytes unless family.nil? + if family_bytes.nil? + @admin.flushRegion(name.to_java_bytes) + else + @admin.flushRegion(name.to_java_bytes, family_bytes) + end + rescue java.lang.IllegalArgumentException + # Unknown region. Try table. + begin + if family_bytes.nil? + @admin.flush(TableName.valueOf(name)) + else + @admin.flush(TableName.valueOf(name), family_bytes) + end + rescue java.lang.IllegalArgumentException + # Unknown table. Try region server. + @admin.flushRegionServer(ServerName.valueOf(name)) + end + end + + #---------------------------------------------------------------------------------------------- + # Requests a table or region or column family compaction + def compact(table_or_region_name, family = nil, type = 'NORMAL') + family_bytes = nil + family_bytes = family.to_java_bytes unless family.nil? + compact_type = nil + if type == 'NORMAL' + compact_type = org.apache.hadoop.hbase.client.CompactType::NORMAL + elsif type == 'MOB' + compact_type = org.apache.hadoop.hbase.client.CompactType::MOB + else + raise ArgumentError, 'only NORMAL or MOB accepted for type!' + end + + begin + @admin.compactRegion(table_or_region_name.to_java_bytes, family_bytes) + rescue java.lang.IllegalArgumentException => e + @admin.compact(TableName.valueOf(table_or_region_name), family_bytes, compact_type) + end + end + + #---------------------------------------------------------------------------------------------- + # Switch compaction on/off at runtime on a region server + def compaction_switch(on_or_off, regionserver_names) + region_servers = regionserver_names.flatten.compact + servers = java.util.ArrayList.new + if region_servers.any? + region_servers.each do |s| + servers.add(s) + end + end + @admin.compactionSwitch(java.lang.Boolean.valueOf(on_or_off), servers) + end + + #---------------------------------------------------------------------------------------------- + # Gets compaction state for specified table + def getCompactionState(table_name) + @admin.getCompactionState(TableName.valueOf(table_name)).name + end + + # Requests to compact all regions on the regionserver + def compact_regionserver(servername, major = false) + @admin.compactRegionServer(ServerName.valueOf(servername), major) + end + + #---------------------------------------------------------------------------------------------- + # Requests a table or region or column family major compaction + def major_compact(table_or_region_name, family = nil, type = 'NORMAL') + family_bytes = nil + family_bytes = family.to_java_bytes unless family.nil? + compact_type = nil + if type == 'NORMAL' + compact_type = org.apache.hadoop.hbase.client.CompactType::NORMAL + elsif type == 'MOB' + compact_type = org.apache.hadoop.hbase.client.CompactType::MOB + else + raise ArgumentError, 'only NORMAL or MOB accepted for type!' + end + + begin + @admin.majorCompactRegion(table_or_region_name.to_java_bytes, family_bytes) + rescue java.lang.IllegalArgumentException => e + @admin.majorCompact(TableName.valueOf(table_or_region_name), family_bytes, compact_type) + end + end + + #---------------------------------------------------------------------------------------------- + # Requests a regionserver's WAL roll + def wal_roll(server_name) + @admin.rollWALWriter(ServerName.valueOf(server_name)) + end + # TODO: remove older hlog_roll version + alias hlog_roll wal_roll + + #---------------------------------------------------------------------------------------------- + # Requests a table or region split + def split(table_or_region_name, split_point = nil) + split_point_bytes = nil + split_point_bytes = split_point.to_java_bytes unless split_point.nil? + begin + @admin.splitRegion(table_or_region_name.to_java_bytes, split_point_bytes) + rescue java.lang.IllegalArgumentException => e + @admin.split(TableName.valueOf(table_or_region_name), split_point_bytes) + end + end + + #---------------------------------------------------------------------------------------------- + # Enable/disable one split or merge switch + # Returns previous switch setting. + def splitormerge_switch(type, enabled) + switch_type = nil + if type == 'SPLIT' + switch_type = org.apache.hadoop.hbase.client::MasterSwitchType::SPLIT + elsif type == 'MERGE' + switch_type = org.apache.hadoop.hbase.client::MasterSwitchType::MERGE + else + raise ArgumentError, 'only SPLIT or MERGE accepted for type!' + end + @admin.setSplitOrMergeEnabled( + java.lang.Boolean.valueOf(enabled), java.lang.Boolean.valueOf(false), + switch_type + )[0] + end + + #---------------------------------------------------------------------------------------------- + # Query the current state of the split or merge switch. + # Returns the switch's state (true is enabled). + def splitormerge_enabled(type) + switch_type = nil + if type == 'SPLIT' + switch_type = org.apache.hadoop.hbase.client::MasterSwitchType::SPLIT + elsif type == 'MERGE' + switch_type = org.apache.hadoop.hbase.client::MasterSwitchType::MERGE + else + raise ArgumentError, 'only SPLIT or MERGE accepted for type!' + end + @admin.isSplitOrMergeEnabled(switch_type) + end + + def locate_region(table_name, row_key) + locator = @connection.getRegionLocator(TableName.valueOf(table_name)) + begin + return locator.getRegionLocation(Bytes.toBytesBinary(row_key)) + ensure + locator.close + end + end + + #---------------------------------------------------------------------------------------------- + # Requests a cluster balance + # Returns true if balancer ran + def balancer(*args) + request = ::Hbase::BalancerUtils.create_balance_request(args) + @admin.balance(request) + end + + #---------------------------------------------------------------------------------------------- + # Enable/disable balancer + # Returns previous balancer switch setting. + def balance_switch(enableDisable) + @admin.setBalancerRunning( + java.lang.Boolean.valueOf(enableDisable), java.lang.Boolean.valueOf(false) + ) + end + + #---------------------------------------------------------------------------------------------- + # Query the current state of the LoadBalancer. + # Returns the balancer's state (true is enabled). + def balancer_enabled? + @admin.isBalancerEnabled + end + + #---------------------------------------------------------------------------------------------- + # Requests clear block cache for table + def clear_block_cache(table_name) + @admin.clearBlockCache(org.apache.hadoop.hbase.TableName.valueOf(table_name)).toString + end + + #---------------------------------------------------------------------------------------------- + # Requests region normalization for all configured tables in the cluster + # Returns true if normalize request was successfully submitted + def normalize(*args) + builder = org.apache.hadoop.hbase.client.NormalizeTableFilterParams::Builder.new + args.each do |arg| + unless arg.is_a?(String) || arg.is_a?(Hash) + raise(ArgumentError, "#{arg.class} of #{arg.inspect} is not of Hash or String type") + end + + if arg.key?(TABLE_NAME) + table_name = arg.delete(TABLE_NAME) + unless table_name.is_a?(String) + raise(ArgumentError, "#{TABLE_NAME} must be of type String") + end + + builder.tableNames(java.util.Collections.singletonList(TableName.valueOf(table_name))) + elsif arg.key?(TABLE_NAMES) + table_names = arg.delete(TABLE_NAMES) + unless table_names.is_a?(Array) + raise(ArgumentError, "#{TABLE_NAMES} must be of type Array") + end + + table_name_list = java.util.LinkedList.new + table_names.each do |tn| + unless tn.is_a?(String) + raise(ArgumentError, "#{TABLE_NAMES} value #{tn} must be of type String") + end + + table_name_list.add(TableName.valueOf(tn)) + end + builder.tableNames(table_name_list) + elsif arg.key?(REGEX) + regex = arg.delete(REGEX) + raise(ArgumentError, "#{REGEX} must be of type String") unless regex.is_a?(String) + + builder.regex(regex) + elsif arg.key?(NAMESPACE) + namespace = arg.delete(NAMESPACE) + unless namespace.is_a?(String) + raise(ArgumentError, "#{NAMESPACE} must be of type String") + end + + builder.namespace(namespace) + else + raise(ArgumentError, "Unrecognized argument #{arg}") + end + end + ntfp = builder.build + @admin.normalize(ntfp) + end + + #---------------------------------------------------------------------------------------------- + # Enable/disable region normalizer + # Returns previous normalizer switch setting. + def normalizer_switch(enableDisable) + @admin.setNormalizerRunning(java.lang.Boolean.valueOf(enableDisable)) + end + + #---------------------------------------------------------------------------------------------- + # Query the current state of region normalizer. + # Returns the state of region normalizer (true is enabled). + def normalizer_enabled? + @admin.isNormalizerEnabled + end + + #---------------------------------------------------------------------------------------------- + # Query the current state of master in maintenance mode. + # Returns the state of maintenance mode (true is on). + def in_maintenance_mode? + @admin.isMasterInMaintenanceMode + end + + #---------------------------------------------------------------------------------------------- + # Request HBCK chore to run + def hbck_chore_run + @hbck.runHbckChore + end + + #---------------------------------------------------------------------------------------------- + # Request a scan of the catalog table (for garbage collection) + # Returns an int signifying the number of entries cleaned + def catalogjanitor_run + @admin.runCatalogScan + end + + #---------------------------------------------------------------------------------------------- + # Enable/disable the catalog janitor + # Returns previous catalog janitor switch setting. + def catalogjanitor_switch(enableDisable) + @admin.enableCatalogJanitor(java.lang.Boolean.valueOf(enableDisable)) + end + + #---------------------------------------------------------------------------------------------- + # Query on the catalog janitor state (enabled/disabled?) + # Returns catalog janitor state (true signifies enabled). + def catalogjanitor_enabled + @admin.isCatalogJanitorEnabled + end + + #---------------------------------------------------------------------------------------------- + # Request cleaner chore to run (for garbage collection of HFiles and WAL files) + def cleaner_chore_run + @admin.runCleanerChore + end + + #---------------------------------------------------------------------------------------------- + # Enable/disable the cleaner chore + # Returns previous cleaner switch setting. + def cleaner_chore_switch(enableDisable) + @admin.setCleanerChoreRunning(java.lang.Boolean.valueOf(enableDisable)) + end + + #---------------------------------------------------------------------------------------------- + # Query on the cleaner chore state (enabled/disabled?) + # Returns cleaner state (true signifies enabled). + def cleaner_chore_enabled + @admin.isCleanerChoreEnabled + end + + #---------------------------------------------------------------------------------------------- + # Enables a table + def enable(table_name) + tableExists(table_name) + return if enabled?(table_name) + @admin.enableTable(TableName.valueOf(table_name)) + end + + #---------------------------------------------------------------------------------------------- + # Enables all tables matching the given regex + def enable_all(regex) + pattern = Pattern.compile(regex.to_s) + failed = java.util.ArrayList.new + @admin.listTableNames(pattern).each do |table_name| + begin + @admin.enableTable(table_name) + rescue java.io.IOException => e + puts "table:#{table_name}, error:#{e.toString}" + failed.add(table_name) + end + end + failed + end + + #---------------------------------------------------------------------------------------------- + # Disables a table + def disable(table_name) + tableExists(table_name) + return if disabled?(table_name) + @admin.disableTable(TableName.valueOf(table_name)) + end + + #---------------------------------------------------------------------------------------------- + # Disables all tables matching the given regex + def disable_all(regex) + pattern = Pattern.compile(regex.to_s) + failed = java.util.ArrayList.new + @admin.listTableNames(pattern).each do |table_name| + begin + @admin.disableTable(table_name) + rescue java.io.IOException => e + puts "table:#{table_name}, error:#{e.toString}" + failed.add(table_name) + end + end + failed + end + + #--------------------------------------------------------------------------------------------- + # Throw exception if table doesn't exist + def tableExists(table_name) + raise ArgumentError, "Table #{table_name} does not exist." unless exists?(table_name) + end + + #---------------------------------------------------------------------------------------------- + # Is table disabled? + def disabled?(table_name) + @admin.isTableDisabled(TableName.valueOf(table_name)) + end + + #---------------------------------------------------------------------------------------------- + # Drops a table + def drop(table_name) + tableExists(table_name) + raise ArgumentError, "Table #{table_name} is enabled. Disable it first." if enabled?( + table_name + ) + + @admin.deleteTable(org.apache.hadoop.hbase.TableName.valueOf(table_name)) + end + + #---------------------------------------------------------------------------------------------- + # Drops a table + def drop_all(regex) + pattern = Pattern.compile(regex.to_s) + failed = java.util.ArrayList.new + @admin.listTableNames(pattern).each do |table_name| + begin + @admin.deleteTable(table_name) + rescue java.io.IOException => e + puts puts "table:#{table_name}, error:#{e.toString}" + failed.add(table_name) + end + end + failed + end + + #---------------------------------------------------------------------------------------------- + # Returns ZooKeeper status dump + def zk_dump + @zk_wrapper = org.apache.hadoop.hbase.zookeeper.ZKWatcher.new( + @admin.getConfiguration, + 'admin', + nil + ) + zk = @zk_wrapper.getRecoverableZooKeeper.getZooKeeper + @zk_main = org.apache.zookeeper.ZooKeeperMain.new(zk) + org.apache.hadoop.hbase.zookeeper.ZKDump.dump(@zk_wrapper) + end + + #---------------------------------------------------------------------------------------------- + # Creates a table + def create(table_name, *args) + # Fail if table name is not a string + raise(ArgumentError, 'Table name must be of type String') unless table_name.is_a?(String) + + # Flatten params array + args = args.flatten.compact + has_columns = false + + # Start defining the table + htd = org.apache.hadoop.hbase.HTableDescriptor.new(org.apache.hadoop.hbase.TableName.valueOf(table_name)) + splits = nil + # Args are either columns or splits, add them to the table definition + # TODO: add table options support + args.each do |arg| + unless arg.is_a?(String) || arg.is_a?(Hash) + raise(ArgumentError, "#{arg.class} of #{arg.inspect} is not of Hash or String type") + end + + # First, handle all the cases where arg is a column family. + if arg.is_a?(String) || arg.key?(NAME) + # If the arg is a string, default action is to add a column to the table. + # If arg has a name, it must also be a column descriptor. + descriptor = hcd(arg, htd) + # Warn if duplicate columns are added + if htd.hasFamily(descriptor.getName) + puts "Family '" + descriptor.getNameAsString + "' already exists, the old one will be replaced" + htd.modifyFamily(descriptor) + else + htd.addFamily(descriptor) + end + has_columns = true + next + end + if arg.key?(REGION_REPLICATION) + region_replication = JInteger.valueOf(arg.delete(REGION_REPLICATION)) + htd.setRegionReplication(region_replication) + end + + # Get rid of the "METHOD", which is deprecated for create. + # We'll do whatever it used to do below if it's table_att. + if (method = arg.delete(METHOD)) + raise(ArgumentError, 'table_att is currently the only supported method') unless method == 'table_att' + end + + # The hash is not a column family. Figure out what's in it. + # First, handle splits. + if arg.key?(SPLITS_FILE) + splits_file = arg.delete(SPLITS_FILE) + unless File.exist?(splits_file) + raise(ArgumentError, "Splits file #{splits_file} doesn't exist") + end + arg[SPLITS] = [] + File.foreach(splits_file) do |line| + arg[SPLITS].push(line.chomp) + end + htd.setValue(SPLITS_FILE, splits_file) + end + + if arg.key?(SPLITS) + splits = Java::byte[][arg[SPLITS].size].new + idx = 0 + arg.delete(SPLITS).each do |split| + splits[idx] = org.apache.hadoop.hbase.util.Bytes.toBytesBinary(split) + idx += 1 + end + elsif arg.key?(NUMREGIONS) || arg.key?(SPLITALGO) + # deprecated region pre-split API; if one of the above is specified, will be ignored. + raise(ArgumentError, 'Number of regions must be specified') unless arg.key?(NUMREGIONS) + raise(ArgumentError, 'Split algorithm must be specified') unless arg.key?(SPLITALGO) + raise(ArgumentError, 'Number of regions must be greater than 1') unless arg[NUMREGIONS] > 1 + num_regions = arg.delete(NUMREGIONS) + split_algo = RegionSplitter.newSplitAlgoInstance(@conf, arg.delete(SPLITALGO)) + splits = split_algo.split(JInteger.valueOf(num_regions)) + end + + # Done with splits; apply formerly-table_att parameters. + update_htd_from_arg(htd, arg) + + arg.each_key do |ignored_key| + puts(format('An argument ignored (unknown or overridden): %s', ignored_key)) + end + end + + # Fail if no column families defined + raise(ArgumentError, 'Table must have at least one column family') unless has_columns + + if splits.nil? + # Perform the create table call + @admin.createTable(htd) + else + # Perform the create table call + @admin.createTable(htd, splits) + end + end + + #---------------------------------------------------------------------------------------------- + #---------------------------------------------------------------------------------------------- + # Assign a region + def assign(region_name) + @admin.assign(region_name.to_java_bytes) + end + + #---------------------------------------------------------------------------------------------- + # Unassign a region + # the force parameter is deprecated, if it is specified, will be ignored. + def unassign(region_name, force = nil) + @admin.unassign(region_name.to_java_bytes) + end + + #---------------------------------------------------------------------------------------------- + # Move a region + def move(encoded_region_name, server = nil) + @admin.move(encoded_region_name.to_java_bytes, server ? server.to_java_bytes : nil) + end + + #---------------------------------------------------------------------------------------------- + # Merge multiple regions + def merge_region(regions, force) + unless regions.is_a?(Array) + raise(ArgumentError, "Type of #{regions.inspect} is #{regions.class}, but expected Array") + end + region_array = Java::byte[][regions.length].new + i = 0 + while i < regions.length + unless regions[i].is_a?(String) + raise( + ArgumentError, + "Type of #{regions[i].inspect} is #{regions[i].class}, but expected String" + ) + end + region_array[i] = regions[i].to_java_bytes + i += 1 + end + org.apache.hadoop.hbase.util.FutureUtils.get( + @admin.mergeRegionsAsync( + region_array, + java.lang.Boolean.valueOf(force) + ) + ) + end + + #---------------------------------------------------------------------------------------------- + # Returns table's structure description + def describe(table_name) + tableExists(table_name) + @admin.getTableDescriptor(TableName.valueOf(table_name)).to_s + end + + def get_column_families(table_name) + tableExists(table_name) + @admin.getTableDescriptor(TableName.valueOf(table_name)).getColumnFamilies + end + + def get_table_attributes(table_name) + tableExists(table_name) + @admin.getTableDescriptor(TableName.valueOf(table_name)).toStringTableAttributes + end + + #---------------------------------------------------------------------------------------------- + # Enable/disable snapshot auto-cleanup based on TTL expiration + # Returns previous snapshot auto-cleanup switch setting. + def snapshot_cleanup_switch(enable_disable) + @admin.snapshotCleanupSwitch( + java.lang.Boolean.valueOf(enable_disable), java.lang.Boolean.valueOf(false) + ) + end + + #---------------------------------------------------------------------------------------------- + # Query the current state of the snapshot auto-cleanup based on TTL + # Returns the snapshot auto-cleanup state (true if enabled) + def snapshot_cleanup_enabled? + @admin.isSnapshotCleanupEnabled + end + + #---------------------------------------------------------------------------------------------- + # Truncates table (deletes all records by recreating the table) + def truncate(table_name_str) + puts "Truncating '#{table_name_str}' table (it may take a while):" + table_name = TableName.valueOf(table_name_str) + + if enabled?(table_name_str) + puts 'Disabling table...' + disable(table_name_str) + end + + puts 'Truncating table...' + @admin.truncateTable(table_name, false) + end + + #---------------------------------------------------------------------------------------------- + # Truncates table while maintaining region boundaries + # (deletes all records by recreating the table) + def truncate_preserve(table_name_str) + puts "Truncating '#{table_name_str}' table (it may take a while):" + table_name = TableName.valueOf(table_name_str) + + if enabled?(table_name_str) + puts 'Disabling table...' + disable(table_name_str) + end + + puts 'Truncating table...' + @admin.truncateTable(table_name, true) + end + + #---------------------------------------------------------------------------------------------- + # Check the status of alter command (number of regions reopened) + def alter_status(table_name) + # Table name should be a string + raise(ArgumentError, 'Table name must be of type String') unless table_name.is_a?(String) + + # Table should exist + raise(ArgumentError, "Can't find a table: #{table_name}") unless exists?(table_name) + + begin + cluster_metrics = @admin.getClusterMetrics + table_region_status = cluster_metrics + .getTableRegionStatesCount + .get(org.apache.hadoop.hbase.TableName.valueOf(table_name)) + if table_region_status.getTotalRegions != 0 + updated_regions = table_region_status.getTotalRegions - + table_region_status.getRegionsInTransition - + table_region_status.getClosedRegions + puts "#{updated_regions}/#{table_region_status.getTotalRegions} regions updated." + else + puts 'All regions updated.' + end + sleep 1 + end while !table_region_status.nil? && table_region_status.getRegionsInTransition != 0 + puts 'Done.' + end + + #---------------------------------------------------------------------------------------------- + # Change table structure or table options + def alter(table_name_str, wait = true, *args) + # Table name should be a string + raise(ArgumentError, 'Table name must be of type String') unless + table_name_str.is_a?(String) + + # Table should exist + raise(ArgumentError, "Can't find a table: #{table_name_str}") unless exists?(table_name_str) + + # There should be at least one argument + raise(ArgumentError, 'There should be at least one argument but the table name') if args.empty? + + table_name = TableName.valueOf(table_name_str) + + # Get table descriptor + htd = org.apache.hadoop.hbase.HTableDescriptor.new(@admin.getTableDescriptor(table_name)) + hasTableUpdate = false + + # Process all args + args.each do |arg| + # Normalize args to support column name only alter specs + arg = { NAME => arg } if arg.is_a?(String) + + # Normalize args to support shortcut delete syntax + arg = { METHOD => 'delete', NAME => arg['delete'] } if arg['delete'] + + # There are 3 possible options. + # 1) Column family spec. Distinguished by having a NAME and no METHOD. + method = arg.delete(METHOD) + if method.nil? && arg.key?(NAME) + descriptor = hcd(arg, htd) + column_name = descriptor.getNameAsString + + # If column already exist, then try to alter it. Create otherwise. + if htd.hasFamily(column_name.to_java_bytes) + htd.modifyFamily(descriptor) + else + htd.addFamily(descriptor) + end + hasTableUpdate = true + next + end + + # 2) Method other than table_att, with some args. + name = arg.delete(NAME) + if !method.nil? && method != 'table_att' + # Delete column family + if method == 'delete' + raise(ArgumentError, 'NAME parameter missing for delete method') unless name + htd.removeFamily(name.to_java_bytes) + hasTableUpdate = true + # Unset table attributes + elsif method == 'table_att_unset' + raise(ArgumentError, 'NAME parameter missing for table_att_unset method') unless name + if name.is_a?(Array) + name.each do |key| + if htd.getValue(key).nil? + raise ArgumentError, "Could not find attribute: #{key}" + end + htd.remove(key) + end + else + if htd.getValue(name).nil? + raise ArgumentError, "Could not find attribute: #{name}" + end + htd.remove(name) + end + hasTableUpdate = true + elsif method == 'table_remove_coprocessor' + classname = arg.delete(CLASSNAME) + raise(ArgumentError, 'CLASSNAME parameter missing for table_remove_coprocessor method') unless classname + if classname.is_a?(Array) + classname.each do |key| + htd.removeCoprocessor(key) + end + else + htd.removeCoprocessor(classname) + end + hasTableUpdate = true + # Unset table configuration + elsif method == 'table_conf_unset' + raise(ArgumentError, 'NAME parameter missing for table_conf_unset method') unless name + if name.is_a?(Array) + name.each do |key| + if htd.getConfigurationValue(key).nil? + raise ArgumentError, "Could not find configuration: #{key}" + end + htd.removeConfiguration(key) + end + else + if htd.getConfigurationValue(name).nil? + raise ArgumentError, "Could not find configuration: #{name}" + end + htd.removeConfiguration(name) + end + hasTableUpdate = true + # Unknown method + else + raise ArgumentError, "Unknown method: #{method}" + end + + arg.each_key do |unknown_key| + puts(format('Unknown argument ignored: %s', unknown_key)) + end + + next + end + + # 3) Some args for the table, optionally with METHOD => table_att (deprecated) + update_htd_from_arg(htd, arg) + + # set a coprocessor attribute + valid_coproc_keys = [] + next unless arg.is_a?(Hash) + arg.each do |key, value| + k = String.new(key) # prepare to strip + k.strip! + + next unless k =~ /coprocessor/i + v = String.new(value) + v.strip! + # TODO: We should not require user to config the coprocessor with our inner format. + htd.addCoprocessorWithSpec(v) + valid_coproc_keys << key + end + + valid_coproc_keys.each do |key| + arg.delete(key) + end + + hasTableUpdate = true + + arg.each_key do |unknown_key| + puts(format('Unknown argument ignored: %s', unknown_key)) + end + + next + end + + # Bulk apply all table modifications. + if hasTableUpdate + @admin.modifyTable(table_name, htd) + + if wait == true + puts 'Updating all regions with the new schema...' + alter_status(table_name_str) + end + end + end + + def status(format, type) + status = @admin.getClusterMetrics + if format == 'detailed' + puts(format('version %s', status.getHBaseVersion)) + # Put regions in transition first because usually empty + puts(format('%d regionsInTransition', status.getRegionStatesInTransition.size)) + for v in status.getRegionStatesInTransition + puts(format(' %s', v)) + end + master = status.getMaster + unless master.nil? + puts(format('active master: %s:%d %d', master.getHostname, master.getPort, master.getStartcode)) + for task in status.getMasterTasks + puts(format(' %s', task.toString)) + end + end + puts(format('%d backup masters', status.getBackupMastersSize)) + for server in status.getBackupMasters + puts(format(' %s:%d %d', server.getHostname, server.getPort, server.getStartcode)) + end + master_coprocs = java.util.Arrays.toString(@admin.getMasterCoprocessors) + unless master_coprocs.nil? + puts(format('master coprocessors: %s', master_coprocs)) + end + puts(format('%d live servers', status.getServersSize)) + for server in status.getServers + puts(format(' %s:%d %d', server.getHostname, server.getPort, server.getStartcode)) + puts(format(' %s', status.getLoad(server).toString)) + for name, region in status.getLoad(server).getRegionsLoad + puts(format(' %s', region.getNameAsString.dump)) + puts(format(' %s', region.toString)) + end + for task in status.getLoad(server).getTasks + puts(format(' %s', task.toString)) + end + end + puts(format('%d dead servers', status.getDeadServersSize)) + for server in status.getDeadServerNames + puts(format(' %s', server)) + end + elsif format == 'replication' + puts(format('version %s', version: status.getHBaseVersion)) + puts(format('%d live servers', servers: status.getServersSize)) + status.getServers.each do |server_status| + sl = status.getLoad(server_status) + r_sink_string = ' SINK:' + r_source_string = ' SOURCE:' + r_load_sink = sl.getReplicationLoadSink + next if r_load_sink.nil? + if r_load_sink.getTimestampsOfLastAppliedOp() == r_load_sink.getTimestampStarted() + # If we have applied no operations since we've started replication, + # assume that we're not acting as a sink and don't print the normal information + r_sink_string << " TimeStampStarted=" + r_load_sink.getTimestampStarted().to_s + r_sink_string << ", Waiting for OPs... " + else + r_sink_string << " TimeStampStarted=" + r_load_sink.getTimestampStarted().to_s + r_sink_string << ", AgeOfLastAppliedOp=" + r_load_sink.getAgeOfLastAppliedOp().to_s + r_sink_string << ", TimeStampsOfLastAppliedOp=" + + (java.util.Date.new(r_load_sink.getTimestampsOfLastAppliedOp())).toString() + end + + r_load_source_map = sl.getReplicationLoadSourceMap + build_source_string(r_load_source_map, r_source_string) + puts(format(' %s:', host: server_status.getHostname)) + if type.casecmp('SOURCE').zero? + puts(format('%s', source: r_source_string)) + elsif type.casecmp('SINK').zero? + puts(format('%s', sink: r_sink_string)) + else + puts(format('%s', source: r_source_string)) + puts(format('%s', sink: r_sink_string)) + end + end + elsif format == 'tasks' + master = status.getMaster + unless master.nil? + puts(format('active master: %s:%d %d', master.getHostname, master.getPort, master.getStartcode)) + printed = false + for task in status.getMasterTasks + next unless task.getState.name == 'RUNNING' + puts(format(' %s', task.toString)) + printed = true + end + if !printed + puts(' no active tasks') + end + end + puts(format('%d live servers', status.getServersSize)) + for server in status.getServers + puts(format(' %s:%d %d', server.getHostname, server.getPort, server.getStartcode)) + printed = false + for task in status.getLoad(server).getTasks + next unless task.getState.name == 'RUNNING' + puts(format(' %s', task.toString)) + printed = true + end + if !printed + puts(' no active tasks') + end + end + elsif format == 'simple' + load = 0 + regions = 0 + master = status.getMaster + puts(format('active master: %s:%d %d', master.getHostname, master.getPort, master.getStartcode)) + puts(format('%d backup masters', status.getBackupMastersSize)) + for server in status.getBackupMasters + puts(format(' %s:%d %d', server.getHostname, server.getPort, server.getStartcode)) + end + puts(format('%d live servers', status.getServersSize)) + for server in status.getServers + puts(format(' %s:%d %d', server.getHostname, server.getPort, server.getStartcode)) + puts(format(' %s', status.getLoad(server).toString)) + load += status.getLoad(server).getNumberOfRequests + regions += status.getLoad(server).getNumberOfRegions + end + puts(format('%d dead servers', status.getDeadServers)) + for server in status.getDeadServerNames + puts(format(' %s', server)) + end + puts(format('Aggregate load: %d, regions: %d', load, regions)) + else + puts "1 active master, #{status.getBackupMastersSize} backup masters, #{status.getServersSize} servers, #{status.getDeadServers} dead, #{format('%.4f', status.getAverageLoad)} average load" + end + end + + def build_source_string(r_load_source_map, r_source_string) + r_load_source_map.each do |peer, sources| + r_source_string << ' PeerID=' + peer + sources.each do |source_load| + build_queue_title(source_load, r_source_string) + build_running_source_stats(source_load, r_source_string) + end + end + end + + def build_queue_title(source_load, r_source_string) + r_source_string << if source_load.isRecovered + "\n Recovered Queue: " + else + "\n Normal Queue: " + end + r_source_string << source_load.getQueueId + end + + def build_running_source_stats(source_load, r_source_string) + if source_load.isRunning + build_shipped_stats(source_load, r_source_string) + build_load_general_stats(source_load, r_source_string) + r_source_string << ', Replication Lag=' + + source_load.getReplicationLag.to_s + else + r_source_string << "\n " + r_source_string << 'No Reader/Shipper threads runnning yet.' + end + end + + def build_shipped_stats(source_load, r_source_string) + r_source_string << if source_load.getTimestampOfLastShippedOp.zero? + "\n " \ + 'No Ops shipped since last restart' + else + "\n AgeOfLastShippedOp=" + + source_load.getAgeOfLastShippedOp.to_s + + ', TimeStampOfLastShippedOp=' + + java.util.Date.new(source_load + .getTimestampOfLastShippedOp).toString + end + end + + def build_load_general_stats(source_load, r_source_string) + r_source_string << ', SizeOfLogQueue=' + + source_load.getSizeOfLogQueue.to_s + r_source_string << ', EditsReadFromLogQueue=' + + source_load.getEditsRead.to_s + r_source_string << ', OpsShippedToTarget=' + + source_load.getOPsShipped.to_s + build_edits_for_source(source_load, r_source_string) + end + + def build_edits_for_source(source_load, r_source_string) + if source_load.hasEditsSinceRestart + r_source_string << ', TimeStampOfNextToReplicate=' + + java.util.Date.new(source_load + .getTimeStampOfNextToReplicate).toString + else + r_source_string << ', No edits for this source' + r_source_string << ' since it started' + end + end + + #---------------------------------------------------------------------------------------------- + # + # Helper methods + # + + # Does table exist? + def exists?(table_name) + @admin.tableExists(TableName.valueOf(table_name)) + end + + #---------------------------------------------------------------------------------------------- + # Is table enabled + def enabled?(table_name) + @admin.isTableEnabled(TableName.valueOf(table_name)) + end + + #---------------------------------------------------------------------------------------------- + # Return a new HColumnDescriptor made of passed args + def hcd(arg, htd) + # String arg, single parameter constructor + return org.apache.hadoop.hbase.HColumnDescriptor.new(arg) if arg.is_a?(String) + + raise(ArgumentError, "Column family #{arg} must have a name") unless name = arg.delete(NAME) + + family = htd.getFamily(name.to_java_bytes) + # create it if it's a new family + family ||= org.apache.hadoop.hbase.HColumnDescriptor.new(name.to_java_bytes) + + family.setBlockCacheEnabled(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::BLOCKCACHE))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::BLOCKCACHE) + family.setScope(JInteger.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::REPLICATION_SCOPE))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::REPLICATION_SCOPE) + family.setCacheDataOnWrite(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::CACHE_DATA_ON_WRITE))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::CACHE_DATA_ON_WRITE) + family.setCacheIndexesOnWrite(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::CACHE_INDEX_ON_WRITE))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::CACHE_INDEX_ON_WRITE) + family.setCacheBloomsOnWrite(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::CACHE_BLOOMS_ON_WRITE))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::CACHE_BLOOMS_ON_WRITE) + family.setEvictBlocksOnClose(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::EVICT_BLOCKS_ON_CLOSE))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::EVICT_BLOCKS_ON_CLOSE) + family.setInMemory(JBoolean.valueOf(arg.delete(ColumnFamilyDescriptorBuilder::IN_MEMORY))) if arg.include?(ColumnFamilyDescriptorBuilder::IN_MEMORY) + if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::IN_MEMORY_COMPACTION) + family.setInMemoryCompaction( + org.apache.hadoop.hbase.MemoryCompactionPolicy.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::IN_MEMORY_COMPACTION)) + ) + end + family.setTimeToLive(arg.delete(ColumnFamilyDescriptorBuilder::TTL)) if arg.include?(ColumnFamilyDescriptorBuilder::TTL) + family.setDataBlockEncoding(org.apache.hadoop.hbase.io.encoding.DataBlockEncoding.valueOf(arg.delete(ColumnFamilyDescriptorBuilder::DATA_BLOCK_ENCODING))) if arg.include?(ColumnFamilyDescriptorBuilder::DATA_BLOCK_ENCODING) + family.setBlocksize(arg.delete(ColumnFamilyDescriptorBuilder::BLOCKSIZE)) if arg.include?(ColumnFamilyDescriptorBuilder::BLOCKSIZE) + family.setMaxVersions(JInteger.valueOf(arg.delete(HConstants::VERSIONS))) if arg.include?(HConstants::VERSIONS) + family.setMinVersions(JInteger.valueOf(arg.delete(ColumnFamilyDescriptorBuilder::MIN_VERSIONS))) if arg.include?(ColumnFamilyDescriptorBuilder::MIN_VERSIONS) + family.setKeepDeletedCells(org.apache.hadoop.hbase.KeepDeletedCells.valueOf(arg.delete(ColumnFamilyDescriptorBuilder::KEEP_DELETED_CELLS).to_s.upcase)) if arg.include?(ColumnFamilyDescriptorBuilder::KEEP_DELETED_CELLS) + family.setCompressTags(JBoolean.valueOf(arg.delete(ColumnFamilyDescriptorBuilder::COMPRESS_TAGS))) if arg.include?(ColumnFamilyDescriptorBuilder::COMPRESS_TAGS) + family.setPrefetchBlocksOnOpen(JBoolean.valueOf(arg.delete(ColumnFamilyDescriptorBuilder::PREFETCH_BLOCKS_ON_OPEN))) if arg.include?(ColumnFamilyDescriptorBuilder::PREFETCH_BLOCKS_ON_OPEN) + family.setMobEnabled(JBoolean.valueOf(arg.delete(ColumnFamilyDescriptorBuilder::IS_MOB))) if arg.include?(ColumnFamilyDescriptorBuilder::IS_MOB) + family.setMobThreshold(JLong.valueOf(arg.delete(ColumnFamilyDescriptorBuilder::MOB_THRESHOLD))) if arg.include?(ColumnFamilyDescriptorBuilder::MOB_THRESHOLD) + family.setNewVersionBehavior(JBoolean.valueOf(arg.delete(ColumnFamilyDescriptorBuilder::NEW_VERSION_BEHAVIOR))) if arg.include?(ColumnFamilyDescriptorBuilder::NEW_VERSION_BEHAVIOR) + if arg.include?(ColumnFamilyDescriptorBuilder::BLOOMFILTER) + bloomtype = arg.delete(ColumnFamilyDescriptorBuilder::BLOOMFILTER).upcase.to_sym + if org.apache.hadoop.hbase.regionserver.BloomType.constants.include?(bloomtype) + family.setBloomFilterType(org.apache.hadoop.hbase.regionserver.BloomType.valueOf(bloomtype)) + else + raise(ArgumentError, "BloomFilter type #{bloomtype} is not supported. Use one of " + org.apache.hadoop.hbase.regionserver.StoreFile::BloomType.constants.join(' ')) + end + end + if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::COMPRESSION) + compression = arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::COMPRESSION).upcase.to_sym + if org.apache.hadoop.hbase.io.compress.Compression::Algorithm.constants.include?(compression) + family.setCompressionType(org.apache.hadoop.hbase.io.compress.Compression::Algorithm.valueOf(compression)) + else + raise(ArgumentError, "Compression #{compression} is not supported. Use one of " + org.apache.hadoop.hbase.io.compress.Compression::Algorithm.constants.join(' ')) + end + end + if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::ENCRYPTION) + algorithm = arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::ENCRYPTION).upcase + family.setEncryptionType(algorithm) + if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::ENCRYPTION_KEY) + key = org.apache.hadoop.hbase.io.crypto.Encryption.generateSecretKey( + @conf, algorithm, arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::ENCRYPTION_KEY) + ) + family.setEncryptionKey(org.apache.hadoop.hbase.security.EncryptionUtil.wrapKey(@conf, key, + algorithm)) + end + end + if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::COMPRESSION_COMPACT) + compression = arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::COMPRESSION_COMPACT).upcase.to_sym + if org.apache.hadoop.hbase.io.compress.Compression::Algorithm.constants.include?(compression) + family.setCompactionCompressionType(org.apache.hadoop.hbase.io.compress.Compression::Algorithm.valueOf(compression)) + else + raise(ArgumentError, "Compression #{compression} is not supported. Use one of " + org.apache.hadoop.hbase.io.compress.Compression::Algorithm.constants.join(' ')) + end + end + if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::COMPRESSION_COMPACT_MAJOR) + compression = arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::COMPRESSION_COMPACT_MAJOR).upcase.to_sym + if org.apache.hadoop.hbase.io.compress.Compression::Algorithm.constants.include?(compression) + family.setMajorCompactionCompressionType(org.apache.hadoop.hbase.io.compress.Compression::Algorithm.valueOf(compression)) + else + raise(ArgumentError, "Compression #{compression} is not supported. Use one of " + org.apache.hadoop.hbase.io.compress.Compression::Algorithm.constants.join(' ')) + end + end + if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::COMPRESSION_COMPACT_MINOR) + compression = arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::COMPRESSION_COMPACT_MINOR).upcase.to_sym + if org.apache.hadoop.hbase.io.compress.Compression::Algorithm.constants.include?(compression) + family.setMinorCompactionCompressionType(org.apache.hadoop.hbase.io.compress.Compression::Algorithm.valueOf(compression)) + else + raise(ArgumentError, "Compression #{compression} is not supported. Use one of " + org.apache.hadoop.hbase.io.compress.Compression::Algorithm.constants.join(' ')) + end + end + if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::STORAGE_POLICY) + storage_policy = arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::STORAGE_POLICY).upcase + family.setStoragePolicy(storage_policy) + end + if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::MOB_COMPACT_PARTITION_POLICY) + mob_partition_policy = arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::MOB_COMPACT_PARTITION_POLICY).upcase.to_sym + if org.apache.hadoop.hbase.client.MobCompactPartitionPolicy.constants.include?(mob_partition_policy) + family.setMobCompactPartitionPolicy(org.apache.hadoop.hbase.client.MobCompactPartitionPolicy.valueOf(mob_partition_policy)) + else + raise(ArgumentError, "MOB_COMPACT_PARTITION_POLICY #{mob_partition_policy} is not supported. Use one of " + org.apache.hadoop.hbase.client.MobCompactPartitionPolicy.constants.join(' ')) + end + end + + set_user_metadata(family, arg.delete(METADATA)) if arg[METADATA] + set_descriptor_config(family, arg.delete(CONFIGURATION)) if arg[CONFIGURATION] + if arg.include?(org.apache.hadoop.hbase + .HColumnDescriptor::DFS_REPLICATION) + family.setDFSReplication(JInteger.valueOf(arg.delete(org.apache.hadoop.hbase + .HColumnDescriptor::DFS_REPLICATION))) + end + + arg.each_key do |unknown_key| + puts(format('Unknown argument ignored for column family %s: %s', name, unknown_key)) + end + + family + end + + #---------------------------------------------------------------------------------------------- + # Enables/disables a region by name + def online(region_name, on_off) + # Open meta table + meta = @connection.getTable(org.apache.hadoop.hbase.TableName::META_TABLE_NAME) + + # Read region info + # FIXME: fail gracefully if can't find the region + region_bytes = region_name.to_java_bytes + g = org.apache.hadoop.hbase.client.Get.new(region_bytes) + g.addColumn(org.apache.hadoop.hbase.HConstants::CATALOG_FAMILY, org.apache.hadoop.hbase.HConstants::REGIONINFO_QUALIFIER) + hri_bytes = meta.get(g).value + + # Change region status + hri = org.apache.hadoop.hbase.util.Writables.getWritable(hri_bytes, org.apache.hadoop.hbase.HRegionInfo.new) + hri.setOffline(on_off) + + # Write it back + put = org.apache.hadoop.hbase.client.Put.new(region_bytes) + put.addColumn(org.apache.hadoop.hbase.HConstants::CATALOG_FAMILY, + org.apache.hadoop.hbase.HConstants::REGIONINFO_QUALIFIER, + org.apache.hadoop.hbase.util.Writables.getBytes(hri)) + meta.put(put) + end + + # Apply user metadata to table/column descriptor + def set_user_metadata(descriptor, metadata) + raise(ArgumentError, "#{METADATA} must be a Hash type") unless metadata.is_a?(Hash) + for k, v in metadata + v = v.to_s unless v.nil? + descriptor.setValue(k, v) + end + end + + #---------------------------------------------------------------------------------------------- + # Take a snapshot of specified table + def snapshot(table, snapshot_name, *args) + # Table name should be a string + raise(ArgumentError, 'Table name must be of type String') unless table.is_a?(String) + + # Snapshot name should be a string + raise(ArgumentError, 'Snapshot name must be of type String') unless + snapshot_name.is_a?(String) + + table_name = TableName.valueOf(table) + if args.empty? + @admin.snapshot(snapshot_name, table_name) + else + args.each do |arg| + ttl = arg[TTL] + ttl = ttl ? ttl.to_java(:long) : -1 + snapshot_props = java.util.HashMap.new + snapshot_props.put("TTL", ttl) + max_filesize = arg[MAX_FILESIZE] + max_filesize = max_filesize ? max_filesize.to_java(:long) : -1 + snapshot_props.put("MAX_FILESIZE", max_filesize) + if arg[SKIP_FLUSH] == true + @admin.snapshot(snapshot_name, table_name, + org.apache.hadoop.hbase.client.SnapshotType::SKIPFLUSH, snapshot_props) + else + @admin.snapshot(snapshot_name, table_name, snapshot_props) + end + end + end + end + + #---------------------------------------------------------------------------------------------- + # Restore specified snapshot + def restore_snapshot(snapshot_name, restore_acl = false) + conf = @connection.getConfiguration + take_fail_safe_snapshot = conf.getBoolean('hbase.snapshot.restore.take.failsafe.snapshot', false) + @admin.restoreSnapshot(snapshot_name, take_fail_safe_snapshot, restore_acl) + end + + #---------------------------------------------------------------------------------------------- + # Create a new table by cloning the snapshot content + def clone_snapshot(snapshot_name, table, restore_acl = false, clone_sft = nil) + @admin.cloneSnapshot(snapshot_name, TableName.valueOf(table), restore_acl, clone_sft) + end + + #---------------------------------------------------------------------------------------------- + # Delete specified snapshot + def delete_snapshot(snapshot_name) + @admin.deleteSnapshot(snapshot_name) + end + + #---------------------------------------------------------------------------------------------- + # Deletes the snapshots matching the given regex + def delete_all_snapshot(regex) + @admin.deleteSnapshots(Pattern.compile(regex)).to_a + end + + #---------------------------------------------------------------------------------------------- + # Deletes the table snapshots matching the given regex + def delete_table_snapshots(tableNameRegex, snapshotNameRegex = '.*') + @admin.deleteTableSnapshots(Pattern.compile(tableNameRegex), + Pattern.compile(snapshotNameRegex)).to_a + end + + #---------------------------------------------------------------------------------------------- + # Returns a list of snapshots + def list_snapshot(regex = '.*') + @admin.listSnapshots(Pattern.compile(regex)).to_a + end + + #---------------------------------------------------------------------------------------------- + # Returns a list of table snapshots + def list_table_snapshots(tableNameRegex, snapshotNameRegex = '.*') + @admin.listTableSnapshots(Pattern.compile(tableNameRegex), + Pattern.compile(snapshotNameRegex)).to_a + end + + #---------------------------------------------------------------------------------------------- + # Returns the ClusterStatus of the cluster + def getClusterStatus + @admin.getClusterStatus + end + + #---------------------------------------------------------------------------------------------- + # Returns a list of regionservers + def getRegionServers + @admin.getClusterStatus.getServers.map { |serverName| serverName } + end + + #---------------------------------------------------------------------------------------------- + # Returns servername corresponding to passed server_name_string + def getServerName(server_name_string) + regionservers = getRegionServers + + if ServerName.isFullServerName(server_name_string) + return ServerName.valueOf(server_name_string) + else + name_list = server_name_string.split(',') + + regionservers.each do|sn| + if name_list[0] == sn.hostname && (name_list[1].nil? ? true : (name_list[1] == sn.port.to_s)) + return sn + end + end + end + + return nil + end + + #---------------------------------------------------------------------------------------------- + # Returns a list of servernames + def getServerNames(servers, should_return_all_if_servers_empty) + regionservers = getRegionServers + servernames = [] + + if servers.empty? + # if no servers were specified as arguments, get a list of all servers + if should_return_all_if_servers_empty + servernames = regionservers + end + else + # Strings replace with ServerName objects in servers array + i = 0 + while i < servers.length + server = servers[i] + + if ServerName.isFullServerName(server) + servernames.push(ServerName.valueOf(server)) + else + name_list = server.split(',') + j = 0 + while j < regionservers.length + sn = regionservers[j] + if name_list[0] == sn.hostname && (name_list[1].nil? ? true : (name_list[1] == sn.port.to_s)) + servernames.push(sn) + end + j += 1 + end + end + i += 1 + end + end + + servernames + end + + # Apply config specific to a table/column to its descriptor + def set_descriptor_config(descriptor, config) + raise(ArgumentError, "#{CONFIGURATION} must be a Hash type") unless config.is_a?(Hash) + for k, v in config + v = v.to_s unless v.nil? + descriptor.setConfiguration(k, v) + end + end + + #---------------------------------------------------------------------------------------------- + # Updates the configuration of one regionserver. + def update_config(serverName) + @admin.updateConfiguration(ServerName.valueOf(serverName)) + end + + #---------------------------------------------------------------------------------------------- + # Updates the configuration of all the regionservers. + def update_all_config + @admin.updateConfiguration + end + + #---------------------------------------------------------------------------------------------- + # Returns namespace's structure description + def describe_namespace(namespace_name) + namespace = @admin.getNamespaceDescriptor(namespace_name) + + return namespace.to_s unless namespace.nil? + + raise(ArgumentError, "Failed to find namespace named #{namespace_name}") + end + + #---------------------------------------------------------------------------------------------- + # Returns a list of namespaces in hbase + def list_namespace(regex = '.*') + pattern = java.util.regex.Pattern.compile(regex) + list = @admin.listNamespaces + list.select { |s| pattern.match(s) } + end + + #---------------------------------------------------------------------------------------------- + # Returns a list of tables in namespace + def list_namespace_tables(namespace_name) + unless namespace_name.nil? + return @admin.listTableNamesByNamespace(namespace_name).map(&:getQualifierAsString) + end + + raise(ArgumentError, "Failed to find namespace named #{namespace_name}") + end + + #---------------------------------------------------------------------------------------------- + # Creates a namespace + def create_namespace(namespace_name, *args) + # Fail if table name is not a string + raise(ArgumentError, 'Namespace name must be of type String') unless namespace_name.is_a?(String) + + # Flatten params array + args = args.flatten.compact + + # Start defining the table + nsb = org.apache.hadoop.hbase.NamespaceDescriptor.create(namespace_name) + args.each do |arg| + unless arg.is_a?(Hash) + raise(ArgumentError, "#{arg.class} of #{arg.inspect} is not of Hash or String type") + end + for k, v in arg + v = v.to_s unless v.nil? + nsb.addConfiguration(k, v) + end + end + @admin.createNamespace(nsb.build) + end + + #---------------------------------------------------------------------------------------------- + # modify a namespace + def alter_namespace(namespace_name, *args) + # Fail if table name is not a string + raise(ArgumentError, 'Namespace name must be of type String') unless namespace_name.is_a?(String) + + nsd = @admin.getNamespaceDescriptor(namespace_name) + + raise(ArgumentError, 'Namespace does not exist') unless nsd + nsb = org.apache.hadoop.hbase.NamespaceDescriptor.create(nsd) + + # Flatten params array + args = args.flatten.compact + + # Start defining the table + args.each do |arg| + unless arg.is_a?(Hash) + raise(ArgumentError, "#{arg.class} of #{arg.inspect} is not of Hash type") + end + method = arg[METHOD] + if method == 'unset' + nsb.removeConfiguration(arg[NAME]) + elsif method == 'set' + arg.delete(METHOD) + for k, v in arg + v = v.to_s unless v.nil? + + nsb.addConfiguration(k, v) + end + else + raise(ArgumentError, "Unknown method #{method}") + end + end + @admin.modifyNamespace(nsb.build) + end + + #---------------------------------------------------------------------------------------------- + # Get namespace's rsgroup + def get_namespace_rsgroup(namespace_name) + # Fail if namespace name is not a string + raise(ArgumentError, 'Namespace name must be of type String') unless namespace_name.is_a?(String) + nsd = @admin.getNamespaceDescriptor(namespace_name) + raise(ArgumentError, 'Namespace does not exist') unless nsd + nsd.getConfigurationValue("hbase.rsgroup.name") + end + + #---------------------------------------------------------------------------------------------- + # Drops a table + def drop_namespace(namespace_name) + @admin.deleteNamespace(namespace_name) + end + + #---------------------------------------------------------------------------------------------- + # Get security capabilities + def get_security_capabilities + @admin.getSecurityCapabilities + end + + # List all procedures + def list_procedures + @admin.getProcedures + end + + # List all locks + def list_locks + @admin.getLocks + end + + # Parse arguments and update HTableDescriptor accordingly + def update_htd_from_arg(htd, arg) + htd.setOwnerString(arg.delete(org.apache.hadoop.hbase.HTableDescriptor::OWNER)) if arg.include?(org.apache.hadoop.hbase.HTableDescriptor::OWNER) + htd.setMaxFileSize(arg.delete(org.apache.hadoop.hbase.HTableDescriptor::MAX_FILESIZE)) if arg.include?(org.apache.hadoop.hbase.HTableDescriptor::MAX_FILESIZE) + htd.setReadOnly(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HTableDescriptor::READONLY))) if arg.include?(org.apache.hadoop.hbase.HTableDescriptor::READONLY) + htd.setCompactionEnabled(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HTableDescriptor::COMPACTION_ENABLED))) if arg.include?(org.apache.hadoop.hbase.HTableDescriptor::COMPACTION_ENABLED) + htd.setSplitEnabled(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HTableDescriptor::SPLIT_ENABLED))) if arg.include?(org.apache.hadoop.hbase.HTableDescriptor::SPLIT_ENABLED) + htd.setMergeEnabled(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HTableDescriptor::MERGE_ENABLED))) if arg.include?(org.apache.hadoop.hbase.HTableDescriptor::MERGE_ENABLED) + htd.setNormalizationEnabled(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HTableDescriptor::NORMALIZATION_ENABLED))) if arg.include?(org.apache.hadoop.hbase.HTableDescriptor::NORMALIZATION_ENABLED) + htd.setNormalizerTargetRegionCount(JInteger.valueOf(arg.delete(org.apache.hadoop.hbase.HTableDescriptor::NORMALIZER_TARGET_REGION_COUNT))) if arg.include?(org.apache.hadoop.hbase.HTableDescriptor::NORMALIZER_TARGET_REGION_COUNT) + # TODO: Keeping backward compatability for NORMALIZER_TARGET_REGION_SIZE with HBASE-25651 change. Can be removed in later version + if arg.include?(org.apache.hadoop.hbase.HTableDescriptor::NORMALIZER_TARGET_REGION_SIZE) + warn 'Use of NORMALIZER_TARGET_REGION_SIZE has been deprecated and will be removed in future version, please use NORMALIZER_TARGET_REGION_SIZE_MB instead' + htd.setNormalizerTargetRegionSize(JLong.valueOf(arg.delete(org.apache.hadoop.hbase.HTableDescriptor::NORMALIZER_TARGET_REGION_SIZE))) + end + htd.setNormalizerTargetRegionSize(JLong.valueOf(arg.delete(org.apache.hadoop.hbase.HTableDescriptor::NORMALIZER_TARGET_REGION_SIZE_MB))) \ + if arg.include?(org.apache.hadoop.hbase.HTableDescriptor::NORMALIZER_TARGET_REGION_SIZE_MB) + htd.setMemStoreFlushSize(arg.delete(org.apache.hadoop.hbase.HTableDescriptor::MEMSTORE_FLUSHSIZE)) if arg.include?(org.apache.hadoop.hbase.HTableDescriptor::MEMSTORE_FLUSHSIZE) + htd.setDurability(org.apache.hadoop.hbase.client.Durability.valueOf(arg.delete(org.apache.hadoop.hbase.HTableDescriptor::DURABILITY))) if arg.include?(org.apache.hadoop.hbase.HTableDescriptor::DURABILITY) + htd.setPriority(JInteger.valueOf(arg.delete(org.apache.hadoop.hbase.HTableDescriptor::PRIORITY))) if arg.include?(org.apache.hadoop.hbase.HTableDescriptor::PRIORITY) + htd.setFlushPolicyClassName(arg.delete(org.apache.hadoop.hbase.HTableDescriptor::FLUSH_POLICY)) if arg.include?(org.apache.hadoop.hbase.HTableDescriptor::FLUSH_POLICY) + htd.setRegionMemstoreReplication(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HTableDescriptor::REGION_MEMSTORE_REPLICATION))) if arg.include?(org.apache.hadoop.hbase.HTableDescriptor::REGION_MEMSTORE_REPLICATION) + htd.setRegionSplitPolicyClassName(arg.delete(org.apache.hadoop.hbase.HTableDescriptor::SPLIT_POLICY)) if arg.include?(org.apache.hadoop.hbase.HTableDescriptor::SPLIT_POLICY) + htd.setRegionReplication(JInteger.valueOf(arg.delete(org.apache.hadoop.hbase.HTableDescriptor::REGION_REPLICATION))) if arg.include?(org.apache.hadoop.hbase.HTableDescriptor::REGION_REPLICATION) + set_user_metadata(htd, arg.delete(METADATA)) if arg[METADATA] + set_descriptor_config(htd, arg.delete(CONFIGURATION)) if arg[CONFIGURATION] + end + + #---------------------------------------------------------------------------------------------- + # clear compaction queues + def clear_compaction_queues(server_name, queue_name = nil) + names = %w[long short] + queues = java.util.HashSet.new + if queue_name.nil? + queues.add('long') + queues.add('short') + elsif queue_name.is_a?(String) + queues.add(queue_name) + unless names.include?(queue_name) + raise(ArgumentError, "Unknown queue name #{queue_name}") + end + elsif queue_name.is_a?(Array) + queue_name.each do |s| + queues.add(s) + unless names.include?(s) + raise(ArgumentError, "Unknown queue name #{s}") + end + end + else + raise(ArgumentError, "Unknown queue name #{queue_name}") + end + @admin.clearCompactionQueues(ServerName.valueOf(server_name), queues) + end + + #---------------------------------------------------------------------------------------------- + # list dead region servers + def list_deadservers + @admin.listDeadServers.to_a + end + + #---------------------------------------------------------------------------------------------- + # clear dead region servers + def clear_deadservers(dead_servers) + # Flatten params array + dead_servers = dead_servers.flatten.compact + if dead_servers.empty? + servers = list_deadservers + else + servers = java.util.ArrayList.new + dead_servers.each do |s| + servers.add(ServerName.valueOf(s)) + end + end + @admin.clearDeadServers(servers).to_a + end + + #---------------------------------------------------------------------------------------------- + # list unknown region servers + def list_unknownservers + @admin.listUnknownServers.to_a + end + + #---------------------------------------------------------------------------------------------- + # List live region servers + def list_liveservers + @admin.getClusterStatus.getServers.to_a + end + + #--------------------------------------------------------------------------- + # create a new table by cloning the existent table schema. + def clone_table_schema(table_name, new_table_name, preserve_splits = true) + @admin.cloneTableSchema(TableName.valueOf(table_name), + TableName.valueOf(new_table_name), + preserve_splits) + end + + #---------------------------------------------------------------------------------------------- + # List decommissioned RegionServers + def list_decommissioned_regionservers + @admin.listDecommissionedRegionServers + end + + #---------------------------------------------------------------------------------------------- + # Retrieve SlowLog Responses from RegionServers + def get_slowlog_responses(server_names, args, is_large_log = false) + unless server_names.is_a?(Array) || server_names.is_a?(String) + raise(ArgumentError, + "#{server_names.class} of #{server_names.inspect} is not of Array/String type") + end + if server_names == '*' + server_names = getServerNames([], true) + else + server_names_list = to_server_names(server_names) + server_names = getServerNames(server_names_list, false) + end + filter_params = get_filter_params(args) + if args.key? 'LIMIT' + limit = args['LIMIT'] + else + limit = 10 + end + if is_large_log + log_type = 'LARGE_LOG' + else + log_type = 'SLOW_LOG' + end + log_dest = org.apache.hadoop.hbase.client.ServerType::REGION_SERVER + server_names_set = java.util.HashSet.new(server_names) + slow_log_responses = @admin.getLogEntries(server_names_set, log_type, log_dest, limit, + filter_params) + slow_log_responses_arr = [] + slow_log_responses.each { |slow_log_response| + slow_log_responses_arr << slow_log_response.toJsonPrettyPrint + } + slow_log_responses_arr + end + + def get_filter_params(args) + filter_params = java.util.HashMap.new + if args.key? 'REGION_NAME' + region_name = args['REGION_NAME'] + filter_params.put('regionName', region_name) + end + if args.key? 'TABLE_NAME' + table_name = args['TABLE_NAME'] + filter_params.put('tableName', table_name) + end + if args.key? 'CLIENT_IP' + client_address = args['CLIENT_IP'] + filter_params.put('clientAddress', client_address) + end + if args.key? 'USER' + user = args['USER'] + filter_params.put('userName', user) + end + if args.key? 'FILTER_BY_OP' + filter_by_op = args['FILTER_BY_OP'] + if filter_by_op != 'OR' && filter_by_op != 'AND' + raise(ArgumentError, "FILTER_BY_OP should be either OR / AND") + end + if filter_by_op == 'AND' + filter_params.put('filterByOperator', 'AND') + end + end + filter_params + end + + #---------------------------------------------------------------------------------------------- + # Clears SlowLog Responses from RegionServers + def clear_slowlog_responses(server_names) + unless server_names.nil? || server_names.is_a?(Array) || server_names.is_a?(String) + raise(ArgumentError, + "#{server_names.class} of #{server_names.inspect} is not of correct type") + end + if server_names.nil? + server_names = getServerNames([], true) + else + server_names_list = to_server_names(server_names) + server_names = getServerNames(server_names_list, false) + end + clear_log_responses = @admin.clearSlowLogResponses(java.util.HashSet.new(server_names)) + clear_log_success_count = 0 + clear_log_responses.each do |response| + if response + clear_log_success_count += 1 + end + end + puts 'Cleared Slowlog responses from ' \ + "#{clear_log_success_count}/#{clear_log_responses.size} RegionServers" + end + + #---------------------------------------------------------------------------------------------- + # Decommission a list of region servers, optionally offload corresponding regions + def decommission_regionservers(host_or_servers, should_offload) + # Fail if host_or_servers is neither a string nor an array + unless host_or_servers.is_a?(Array) || host_or_servers.is_a?(String) + raise(ArgumentError, + "#{host_or_servers.class} of #{host_or_servers.inspect} is not of Array/String type") + end + + # Fail if should_offload is neither a TrueClass/FalseClass nor a string + unless (!!should_offload == should_offload) || should_offload.is_a?(String) + raise(ArgumentError, "#{should_offload} is not a boolean value") + end + + # If a string is passed, convert it to an array + _host_or_servers = host_or_servers.is_a?(Array) ? + host_or_servers : + java.util.Arrays.asList(host_or_servers) + + # Retrieve the server names corresponding to passed _host_or_servers list + server_names = getServerNames(_host_or_servers, false) + + # Fail, if we can not find any server(s) corresponding to the passed host_or_servers + if server_names.empty? + raise(ArgumentError, + "Could not find any server(s) with specified name(s): #{host_or_servers}") + end + + @admin.decommissionRegionServers(server_names, + java.lang.Boolean.valueOf(should_offload)) + end + + #---------------------------------------------------------------------------------------------- + # Recommission a region server, optionally load a list of passed regions + def recommission_regionserver(server_name_string, encoded_region_names) + # Fail if server_name_string is not a string + unless server_name_string.is_a?(String) + raise(ArgumentError, + "#{server_name_string.class} of #{server_name_string.inspect} is not of String type") + end + + # Fail if encoded_region_names is not an array + unless encoded_region_names.is_a?(Array) + raise(ArgumentError, + "#{encoded_region_names.class} of #{encoded_region_names.inspect} is not of Array type") + end + + # Convert encoded_region_names from string to bytes (element-wise) + region_names_in_bytes = encoded_region_names + .map {|region_name| region_name.to_java_bytes} + .compact + + # Retrieve the server name corresponding to the passed server_name_string + server_name = getServerName(server_name_string) + + # Fail if we can not find a server corresponding to the passed server_name_string + if server_name.nil? + raise(ArgumentError, + "Could not find any server with name #{server_name_string}") + end + + @admin.recommissionRegionServer(server_name, region_names_in_bytes) + end + + #---------------------------------------------------------------------------------------------- + # Retrieve latest balancer decisions made by LoadBalancers + def get_balancer_decisions(args) + if args.key? 'LIMIT' + limit = args['LIMIT'] + else + limit = 250 + end + log_type = 'BALANCER_DECISION' + log_dest = org.apache.hadoop.hbase.client.ServerType::MASTER + balancer_decisions_responses = @admin.getLogEntries(nil, log_type, log_dest, limit, nil) + balancer_decisions_resp_arr = [] + balancer_decisions_responses.each { |balancer_dec_resp| + balancer_decisions_resp_arr << balancer_dec_resp.toJsonPrettyPrint + } + balancer_decisions_resp_arr + end + + #---------------------------------------------------------------------------------------------- + # Retrieve latest balancer rejections made by LoadBalancers + def get_balancer_rejections(args) + if args.key? 'LIMIT' + limit = args['LIMIT'] + else + limit = 250 + end + + log_type = 'BALANCER_REJECTION' + log_dest = org.apache.hadoop.hbase.client.ServerType::MASTER + balancer_rejections_responses = @admin.getLogEntries(nil, log_type, log_dest, limit, nil) + balancer_rejections_resp_arr = [] + balancer_rejections_responses.each { |balancer_dec_resp| + balancer_rejections_resp_arr << balancer_dec_resp.toJsonPrettyPrint + } + balancer_rejections_resp_arr + end + + #---------------------------------------------------------------------------------------------- + # get region info + def get_region_info(regionname) + sn = @admin.getMaster() + puts org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRegionInfo(nil, + @connection.getAdmin(sn), + regionname.to_java_bytes) + end + + #---------------------------------------------------------------------------------------------- + # Stop the active Master + def stop_master + @admin.stopMaster + end + + # Stop the given RegionServer + def stop_regionserver(hostport) + @admin.stopRegionServer(hostport) + end + + #---------------------------------------------------------------------------------------------- + # Get list of server names + def to_server_names(server_names) + if server_names.is_a?(Array) + server_names + else + java.util.Arrays.asList(server_names) + end + end + + #---------------------------------------------------------------------------------------------- + # Change table's sft + def modify_table_sft(tableName, sft) + @admin.modifyTableStoreFileTracker(tableName, sft) + end + + #---------------------------------------------------------------------------------------------- + # Change table column family's sft + def modify_table_family_sft(tableName, family_bytes, sft) + @admin.modifyColumnFamilyStoreFileTracker(tableName, family_bytes, sft) + end + + #---------------------------------------------------------------------------------------------- + # Flush master local region + def flush_master_store() + @admin.flushMasterStore() + end + end + # rubocop:enable Metrics/ClassLength +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase/balancer_utils.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase/balancer_utils.rb new file mode 100644 index 0000000000..9948c0f474 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase/balancer_utils.rb @@ -0,0 +1,57 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include Java + +java_import org.apache.hadoop.hbase.client.BalanceRequest + +module Hbase + class BalancerUtils + def self.create_balance_request(args) + args = args.first if args.first.is_a?(Array) and args.size == 1 + if args.nil? or args.empty? + return BalanceRequest.defaultInstance() + elsif args.size > 2 + raise ArgumentError, "Illegal arguments #{args}. Expected between 0 and 2 arguments, but got #{args.size}." + end + + builder = BalanceRequest.newBuilder() + + index = 0 + args.each do |arg| + if !arg.is_a?(String) + raise ArgumentError, "Illegal argument in index #{index}: #{arg}. All arguments must be strings, but got #{arg.class}." + end + + case arg + when 'force', 'ignore_rit' + builder.setIgnoreRegionsInTransition(true) + when 'dry_run' + builder.setDryRun(true) + else + raise ArgumentError, "Illegal argument in index #{index}: #{arg}. Unknown option #{arg}, expected 'force', 'ignore_rit', or 'dry_run'." + end + + index += 1 + end + + return builder.build() + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase/hbase.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase/hbase.rb new file mode 100644 index 0000000000..a24eb12bb0 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase/hbase.rb @@ -0,0 +1,88 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include Java +java_import org.apache.hadoop.hbase.client.ConnectionFactory +java_import org.apache.hadoop.hbase.HBaseConfiguration +java_import org.apache.linkis.manager.engineplugin.hbase.HBaseConnectionManager + +require 'hbase/admin' +require 'hbase/table' +require 'hbase/taskmonitor' +require 'hbase/quotas' +require 'hbase/security' +require 'hbase/visibility_labels' + +module Hbase + class Hbase + attr_accessor :configuration + + def initialize(properties = nil) + # Create configuration + if properties + self.configuration = HBaseConnectionManagerRuby.getConfiguration(properties) + @connection = HBaseConnectionManagerRuby.getConnection(properties) + else + self.configuration = HBaseConfiguration.create + # Turn off retries in hbase and ipc. Human doesn't want to wait on N retries. + configuration.setInt('hbase.client.retries.number', 7) + configuration.setInt('hbase.ipc.client.connect.max.retries', 3) + @connection = ConnectionFactory.createConnection(configuration) + end + end + + # Returns ruby's Admin class from admin.rb + def admin + ::Hbase::Admin.new(@connection) + end + + def rsgroup_admin + ::Hbase::RSGroupAdmin.new(@connection) + end + + def taskmonitor + ::Hbase::TaskMonitor.new(configuration) + end + + # Create new one each time + def table(table, shell) + ::Hbase::Table.new(@connection.getTable(TableName.valueOf(table)), shell) + end + + def replication_admin + ::Hbase::RepAdmin.new(configuration) + end + + def security_admin + ::Hbase::SecurityAdmin.new(@connection.getAdmin) + end + + def visibility_labels_admin + ::Hbase::VisibilityLabelsAdmin.new(@connection.getAdmin) + end + + def quotas_admin + ::Hbase::QuotasAdmin.new(@connection.getAdmin) + end + + def shutdown + @connection.close + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase/quotas.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase/quotas.rb new file mode 100644 index 0000000000..3488f0f33d --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase/quotas.rb @@ -0,0 +1,448 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include Java +java_import java.util.concurrent.TimeUnit +java_import org.apache.hadoop.hbase.TableName +java_import org.apache.hadoop.hbase.ServerName +java_import org.apache.hadoop.hbase.quotas.ThrottleType +java_import org.apache.hadoop.hbase.quotas.QuotaFilter +java_import org.apache.hadoop.hbase.quotas.QuotaRetriever +java_import org.apache.hadoop.hbase.quotas.QuotaScope +java_import org.apache.hadoop.hbase.quotas.QuotaSettingsFactory +java_import org.apache.hadoop.hbase.quotas.QuotaTableUtil +java_import org.apache.hadoop.hbase.quotas.SpaceViolationPolicy + +module HBaseQuotasConstants + QUOTA_TABLE_NAME = QuotaTableUtil::QUOTA_TABLE_NAME + # RPC Quota constants + GLOBAL_BYPASS = 'GLOBAL_BYPASS'.freeze + THROTTLE_TYPE = 'THROTTLE_TYPE'.freeze + THROTTLE = 'THROTTLE'.freeze + REQUEST = 'REQUEST'.freeze + WRITE = 'WRITE'.freeze + READ = 'READ'.freeze + SCOPE = 'SCOPE'.freeze + CLUSTER = 'CLUSTER'.freeze + MACHINE = 'MACHINE'.freeze + # Space quota constants + SPACE = 'SPACE'.freeze + NO_INSERTS = 'NO_INSERTS'.freeze + NO_WRITES = 'NO_WRITES'.freeze + NO_WRITES_COMPACTIONS = 'NO_WRITES_COMPACTIONS'.freeze + DISABLE = 'DISABLE'.freeze + READ_NUMBER = 'READ_NUMBER'.freeze + READ_SIZE = 'READ_SIZE'.freeze + WRITE_NUMBER = 'WRITE_NUMBER'.freeze + WRITE_SIZE = 'WRITE_SIZE'.freeze + REQUEST_NUMBER = 'REQUEST_NUMBER'.freeze + REQUEST_SIZE = 'REQUEST_SIZE'.freeze + REQUEST_CAPACITY_UNIT = 'REQUEST_CAPACITY_UNIT'.freeze + WRITE_CAPACITY_UNIT = 'WRITE_CAPACITY_UNIT'.freeze + READ_CAPACITY_UNIT = 'READ_CAPACITY_UNIT'.freeze +end + +module Hbase + # rubocop:disable Metrics/ClassLength + class QuotasAdmin + include HBaseConstants + include HBaseQuotasConstants + + def initialize(admin) + @admin = admin + end + + def close + @admin.close + end + + def throttle(args) + raise(ArgumentError, 'Arguments should be a Hash') unless args.is_a?(Hash) + type = args.fetch(THROTTLE_TYPE, REQUEST) + args.delete(THROTTLE_TYPE) + type, limit, time_unit = _parse_limit(args.delete(LIMIT), ThrottleType, type) + scope = _parse_scope(args.fetch(SCOPE, MACHINE)) + args.delete(SCOPE) + if args.key?(USER) + user = args.delete(USER) + if args.key?(TABLE) + table = TableName.valueOf(args.delete(TABLE)) + raise(ArgumentError, 'Unexpected arguments: ' + args.inspect) unless args.empty? + settings = QuotaSettingsFactory.throttleUser(user, table, type, limit, time_unit, scope) + elsif args.key?(NAMESPACE) + namespace = args.delete(NAMESPACE) + raise(ArgumentError, 'Unexpected arguments: ' + args.inspect) unless args.empty? + settings = QuotaSettingsFactory.throttleUser(user, namespace, type, limit, time_unit, scope) + else + raise(ArgumentError, 'Unexpected arguments: ' + args.inspect) unless args.empty? + settings = QuotaSettingsFactory.throttleUser(user, type, limit, time_unit, scope) + end + elsif args.key?(TABLE) + table = TableName.valueOf(args.delete(TABLE)) + raise(ArgumentError, 'Unexpected arguments: ' + args.inspect) unless args.empty? + settings = QuotaSettingsFactory.throttleTable(table, type, limit, time_unit, scope) + elsif args.key?(NAMESPACE) + namespace = args.delete(NAMESPACE) + raise(ArgumentError, 'Unexpected arguments: ' + args.inspect) unless args.empty? + settings = QuotaSettingsFactory.throttleNamespace(namespace, type, limit, time_unit, scope) + elsif args.key?(REGIONSERVER) + # TODO: Setting specified region server quota isn't supported currently and using 'all' for all RS + if scope == QuotaScope.valueOf(CLUSTER) + raise(ArgumentError, 'Invalid region server throttle scope, must be MACHINE') + end + settings = QuotaSettingsFactory.throttleRegionServer('all', type, limit, time_unit) + else + raise 'One of USER, TABLE, NAMESPACE or REGIONSERVER must be specified' + end + @admin.setQuota(settings) + end + + def unthrottle(args) + raise(ArgumentError, 'Arguments should be a Hash') unless args.is_a?(Hash) + + if args.key?(USER) then settings = unthrottle_user_table_namespace(args) + elsif args.key?(TABLE) then settings = unthrottle_table(args) + elsif args.key?(NAMESPACE) then settings = unthrottle_namespace(args) + elsif args.key?(REGIONSERVER) + settings = unthrottle_regionserver(args) + else + raise 'One of USER, TABLE, NAMESPACE or REGIONSERVER must be specified' + end + @admin.setQuota(settings) + end + + def _parse_throttle_type(type_cls, throttle_type) + type_cls.valueOf(throttle_type) + end + + def get_throttle_type(args) + throttle_type_str = args.delete(THROTTLE_TYPE) + throttle_type = _parse_throttle_type(ThrottleType, throttle_type_str) + throttle_type + end + + def unthrottle_user_table_namespace(args) + user = args.delete(USER) + settings = if args.key?(TABLE) + unthrottle_user_table(args, user) + elsif args.key?(NAMESPACE) + unthrottle_user_namespace(args, user) + else + unthrottle_user(args, user) + end + settings + end + + def args_empty(args) + return if args.empty? + + raise(ArgumentError, + 'Unexpected arguments: ' + args.inspect) + end + + def unthrottle_user_table(args, user) + table = TableName.valueOf(args.delete(TABLE)) + if args.key?(THROTTLE_TYPE) + settings = QuotaSettingsFactory + .unthrottleUserByThrottleType(user, + table, get_throttle_type(args)) + else + args_empty(args) + settings = QuotaSettingsFactory.unthrottleUser(user, table) + end + settings + end + + def unthrottle_user_namespace(args, user) + namespace = args.delete(NAMESPACE) + if args.key?(THROTTLE_TYPE) + throttle_type = get_throttle_type(args) + settings = QuotaSettingsFactory + .unthrottleUserByThrottleType(user, namespace, throttle_type) + else + args_empty(args) + settings = QuotaSettingsFactory.unthrottleUser(user, namespace) + end + settings + end + + def unthrottle_user(args, user) + if args.key?(THROTTLE_TYPE) + throttle_type = get_throttle_type(args) + settings = QuotaSettingsFactory + .unthrottleUserByThrottleType(user, throttle_type) + else + args_empty(args) + settings = QuotaSettingsFactory.unthrottleUser(user) + end + settings + end + + def unthrottle_table(args) + table = TableName.valueOf(args.delete(TABLE)) + if args.key?(THROTTLE_TYPE) + throttle_type = get_throttle_type(args) + settings = QuotaSettingsFactory + .unthrottleTableByThrottleType(table, throttle_type) + else + args_empty(args) + settings = QuotaSettingsFactory.unthrottleTable(table) + end + settings + end + + def unthrottle_namespace(args) + namespace = args.delete(NAMESPACE) + if args.key?(THROTTLE_TYPE) + throttle_type = get_throttle_type(args) + settings = QuotaSettingsFactory + .unthrottleNamespaceByThrottleType(namespace, throttle_type) + else + args_empty(args) + settings = QuotaSettingsFactory.unthrottleNamespace(namespace) + end + settings + end + + def unthrottle_regionserver(args) + _region_server = args.delete(REGIONSERVER) + if args.key?(THROTTLE_TYPE) + throttle_type = get_throttle_type(args) + settings = QuotaSettingsFactory + .unthrottleRegionServerByThrottleType('all', throttle_type) + else + args_empty(args) + settings = QuotaSettingsFactory.unthrottleRegionServer('all') + end + settings + end + + # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity + # rubocop:disable Metrics/MethodLength, Metrics/PerceivedComplexity + def limit_space(args) + raise(ArgumentError, 'Argument should be a Hash') unless !args.nil? && args.is_a?(Hash) + # Let the user provide a raw number + limit = if args[LIMIT].is_a?(Numeric) + args[LIMIT] + else + # Parse a string a 1K, 2G, etc. + _parse_size(args[LIMIT]) + end + if limit <= 0 + raise(ArgumentError, 'Invalid space limit, must be greater than 0') + end + + # Extract the policy, failing if something bogus was provided + policy = SpaceViolationPolicy.valueOf(args[POLICY]) + # Create a table or namespace quota + if args.key?(TABLE) + if args.key?(NAMESPACE) + raise(ArgumentError, 'Only one of TABLE or NAMESPACE can be specified.') + end + settings = QuotaSettingsFactory.limitTableSpace(TableName.valueOf(args.delete(TABLE)), limit, policy) + elsif args.key?(NAMESPACE) + if args.key?(TABLE) + raise(ArgumentError, 'Only one of TABLE or NAMESPACE can be specified.') + end + settings = QuotaSettingsFactory.limitNamespaceSpace(args.delete(NAMESPACE), limit, policy) + else + raise(ArgumentError, 'One of TABLE or NAMESPACE must be specified.') + end + # Apply the quota + @admin.setQuota(settings) + end + # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity + # rubocop:enable Metrics/MethodLength, Metrics/PerceivedComplexity + + def remove_space_limit(args) + raise(ArgumentError, 'Argument should be a Hash') unless !args.nil? && args.is_a?(Hash) + if args.key?(TABLE) + if args.key?(NAMESPACE) + raise(ArgumentError, 'Only one of TABLE or NAMESPACE can be specified.') + end + table = TableName.valueOf(args.delete(TABLE)) + settings = QuotaSettingsFactory.removeTableSpaceLimit(table) + elsif args.key?(NAMESPACE) + if args.key?(TABLE) + raise(ArgumentError, 'Only one of TABLE or NAMESPACE can be specified.') + end + settings = QuotaSettingsFactory.removeNamespaceSpaceLimit(args.delete(NAMESPACE)) + else + raise(ArgumentError, 'One of TABLE or NAMESPACE must be specified.') + end + @admin.setQuota(settings) + end + + def get_master_table_sizes + @admin.getSpaceQuotaTableSizes + end + + def get_quota_snapshots(regionserver = nil) + # Ask a regionserver if we were given one + return get_rs_quota_snapshots(regionserver) if regionserver + # Otherwise, read from the quota table + get_quota_snapshots_from_table + end + + def get_quota_snapshots_from_table + # Reads the snapshots from the hbase:quota table + QuotaTableUtil.getSnapshots(@admin.getConnection) + end + + def get_rs_quota_snapshots(rs) + # Reads the snapshots from a specific regionserver + @admin.getRegionServerSpaceQuotaSnapshots(ServerName.valueOf(rs)) + end + + def set_global_bypass(bypass, args) + raise(ArgumentError, 'Arguments should be a Hash') unless args.is_a?(Hash) + + if args.key?(USER) + user = args.delete(USER) + raise(ArgumentError, 'Unexpected arguments: ' + args.inspect) unless args.empty? + settings = QuotaSettingsFactory.bypassGlobals(user, bypass) + else + raise 'Expected USER' + end + @admin.setQuota(settings) + end + + def list_quotas(args = {}) + raise(ArgumentError, 'Arguments should be a Hash') unless args.is_a?(Hash) + + limit = args.delete('LIMIT') || -1 + count = 0 + + filter = QuotaFilter.new + filter.setUserFilter(args.delete(USER)) if args.key?(USER) + filter.setTableFilter(args.delete(TABLE)) if args.key?(TABLE) + filter.setNamespaceFilter(args.delete(NAMESPACE)) if args.key?(NAMESPACE) + raise(ArgumentError, 'Unexpected arguments: ' + args.inspect) unless args.empty? + + # Start the scanner + scanner = @admin.getQuotaRetriever(filter) + begin + iter = scanner.iterator + + # Iterate results + while iter.hasNext + break if limit > 0 && count >= limit + + settings = iter.next + owner = { + USER => settings.getUserName, + TABLE => settings.getTableName, + NAMESPACE => settings.getNamespace, + REGIONSERVER => settings.getRegionServer + }.delete_if { |_k, v| v.nil? }.map { |k, v| k.to_s + ' => ' + v.to_s } * ', ' + + yield owner, settings.to_s + + count += 1 + end + ensure + scanner.close + end + + count + end + + def list_snapshot_sizes + QuotaTableUtil.getObservedSnapshotSizes(@admin.getConnection) + end + + def list_snapshot_sizes() + QuotaTableUtil.getObservedSnapshotSizes(@admin.getConnection()) + end + + def switch_rpc_throttle(enabled) + @admin.switchRpcThrottle(java.lang.Boolean.valueOf(enabled)) + end + + def switch_exceed_throttle_quota(enabled) + @admin.exceedThrottleQuotaSwitch(java.lang.Boolean.valueOf(enabled)) + end + + def _parse_size(str_limit) + str_limit = str_limit.downcase + match = /^(\d+)([bkmgtp%]?)$/.match(str_limit) + if match + if match[2] == '%' + return match[1].to_i + else + return _size_from_str(match[1].to_i, match[2]) + end + else + raise(ArgumentError, 'Invalid size limit syntax') + end + end + + # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def _parse_limit(str_limit, type_cls, type) + str_limit = str_limit.downcase + match = /^(\d+)(req|cu|[bkmgtp])\/(sec|min|hour|day)$/.match(str_limit) + if match + limit = match[1].to_i + if match[2] == 'req' + type = type_cls.valueOf(type + '_NUMBER') + elsif match[2] == 'cu' + type = type_cls.valueOf(type + '_CAPACITY_UNIT') + else + limit = _size_from_str(limit, match[2]) + type = type_cls.valueOf(type + '_SIZE') + end + + if limit <= 0 + raise(ArgumentError, 'Invalid throttle limit, must be greater than 0') + end + + case match[3] + when 'sec' then time_unit = TimeUnit::SECONDS + when 'min' then time_unit = TimeUnit::MINUTES + when 'hour' then time_unit = TimeUnit::HOURS + when 'day' then time_unit = TimeUnit::DAYS + end + + return type, limit, time_unit + else + raise(ArgumentError, 'Invalid throttle limit syntax') + end + end + # rubocop:enable Metrics/AbcSize, Metrics/MethodLength + + def _size_from_str(value, suffix) + case suffix + when 'k' then value <<= 10 + when 'm' then value <<= 20 + when 'g' then value <<= 30 + when 't' then value <<= 40 + when 'p' then value <<= 50 + end + value + end + + def _parse_scope(scope_str) + scope_str = scope_str.upcase + return QuotaScope.valueOf(scope_str) if [CLUSTER, MACHINE].include?(scope_str) + unless raise(ArgumentError, 'Invalid throttle scope, must be either CLUSTER or MACHINE') + end + end + end + # rubocop:enable Metrics/ClassLength +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase/replication_admin.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase/replication_admin.rb new file mode 100644 index 0000000000..0ed1a56bdf --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase/replication_admin.rb @@ -0,0 +1,466 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include Java + +java_import org.apache.hadoop.hbase.client.replication.ReplicationPeerConfigUtil +java_import org.apache.hadoop.hbase.replication.ReplicationPeerConfig +java_import org.apache.hadoop.hbase.util.Bytes +java_import org.apache.hadoop.hbase.zookeeper.ZKConfig +java_import org.apache.hadoop.hbase.TableName +java_import com.hydraql.connection.HBaseConnectionManagerRuby + +# Used for replication administrative operations. + +module Hbase + class RepAdmin + include HBaseConstants + + def initialize(configuration) + @configuration = configuration + @admin = HBaseConnectionManagerRuby.getConnection(configuration).getAdmin + end + + #---------------------------------------------------------------------------------------------- + # Add a new peer cluster to replicate to + def add_peer(id, args = {}, peer_tableCFs = nil) + if args.is_a?(Hash) + unless peer_tableCFs.nil? + raise(ArgumentError, 'peer_tableCFs should be specified as TABLE_CFS in args') + end + + endpoint_classname = args.fetch(ENDPOINT_CLASSNAME, nil) + cluster_key = args.fetch(CLUSTER_KEY, nil) + + # Handle cases where custom replication endpoint and cluster key are either both provided + # or neither are provided + if endpoint_classname.nil? && cluster_key.nil? + raise(ArgumentError, 'Either ENDPOINT_CLASSNAME or CLUSTER_KEY must be specified.') + end + + # Cluster Key is required for ReplicationPeerConfig for a custom replication endpoint + if !endpoint_classname.nil? && cluster_key.nil? + cluster_key = ZKConfig.getZooKeeperClusterKey(@configuration) + end + + # Optional parameters + config = args.fetch(CONFIG, nil) + data = args.fetch(DATA, nil) + table_cfs = args.fetch(TABLE_CFS, nil) + namespaces = args.fetch(NAMESPACES, nil) + peer_state = args.fetch(STATE, nil) + serial = args.fetch(SERIAL, nil) + + # Create and populate a ReplicationPeerConfig + builder = org.apache.hadoop.hbase.replication.ReplicationPeerConfig + .newBuilder() + builder.set_cluster_key(cluster_key) + + unless endpoint_classname.nil? + builder.set_replication_endpoint_impl(endpoint_classname) + end + + unless serial.nil? + builder.setSerial(serial) + end + + unless config.nil? + builder.putAllConfiguration(config) + end + + unless data.nil? + # Convert Strings to Bytes for peer_data + data.each do |key, val| + builder.putPeerData(Bytes.to_bytes(key), Bytes.to_bytes(val)) + end + end + + unless namespaces.nil? + ns_set = java.util.HashSet.new + namespaces.each do |n| + ns_set.add(n) + end + builder.setReplicateAllUserTables(false) + builder.set_namespaces(ns_set) + end + + unless table_cfs.nil? + # convert table_cfs to TableName + map = java.util.HashMap.new + table_cfs.each do |key, val| + map.put(org.apache.hadoop.hbase.TableName.valueOf(key), val) + end + builder.setReplicateAllUserTables(false) + builder.set_table_cfs_map(map) + end + + enabled = true + unless peer_state.nil? + enabled = false if peer_state == 'DISABLED' + end + @admin.addReplicationPeer(id, builder.build, enabled) + else + raise(ArgumentError, 'args must be a Hash') + end + end + + #---------------------------------------------------------------------------------------------- + # Remove a peer cluster, stops the replication + def remove_peer(id) + @admin.removeReplicationPeer(id) + end + + #--------------------------------------------------------------------------------------------- + # Show replcated tables/column families, and their ReplicationType + def list_replicated_tables(regex = '.*') + pattern = java.util.regex.Pattern.compile(regex) + list = @admin.listReplicatedTableCFs + list.select { |t| pattern.match(t.getTable.getNameAsString) } + end + + #---------------------------------------------------------------------------------------------- + # List all peer clusters + def list_peers + @admin.listReplicationPeers + end + + #---------------------------------------------------------------------------------------------- + # Restart the replication stream to the specified peer + def enable_peer(id) + @admin.enableReplicationPeer(id) + end + + #---------------------------------------------------------------------------------------------- + # Stop the replication stream to the specified peer + def disable_peer(id) + @admin.disableReplicationPeer(id) + end + + #---------------------------------------------------------------------------------------------- + # Show the current tableCFs config for the specified peer + def show_peer_tableCFs(id) + rpc = @admin.getReplicationPeerConfig(id) + show_peer_tableCFs_by_config(rpc) + end + + def show_peer_tableCFs_by_config(peer_config) + ReplicationPeerConfigUtil.convertToString(peer_config.getTableCFsMap) + end + + #---------------------------------------------------------------------------------------------- + # Set new tableCFs config for the specified peer + def set_peer_tableCFs(id, tableCFs) + unless tableCFs.nil? + # convert tableCFs to TableName + map = java.util.HashMap.new + tableCFs.each do |key, val| + map.put(org.apache.hadoop.hbase.TableName.valueOf(key), val) + end + rpc = get_peer_config(id) + unless rpc.nil? + rpc.setTableCFsMap(map) + @admin.updateReplicationPeerConfig(id, rpc) + end + end + end + + #---------------------------------------------------------------------------------------------- + # Append a tableCFs config for the specified peer + def append_peer_tableCFs(id, tableCFs) + unless tableCFs.nil? + # convert tableCFs to TableName + map = java.util.HashMap.new + tableCFs.each do |key, val| + map.put(org.apache.hadoop.hbase.TableName.valueOf(key), val) + end + end + @admin.appendReplicationPeerTableCFs(id, map) + end + + #---------------------------------------------------------------------------------------------- + # Remove some tableCFs from the tableCFs config of the specified peer + def remove_peer_tableCFs(id, tableCFs) + unless tableCFs.nil? + # convert tableCFs to TableName + map = java.util.HashMap.new + tableCFs.each do |key, val| + map.put(org.apache.hadoop.hbase.TableName.valueOf(key), val) + end + end + @admin.removeReplicationPeerTableCFs(id, map) + end + + # Append exclude-tableCFs to the exclude-tableCFs config for the specified peer + def append_peer_exclude_tableCFs(id, excludeTableCFs) + unless excludeTableCFs.nil? + # convert tableCFs to TableName + map = java.util.HashMap.new + excludeTableCFs.each do |key, val| + map.put(org.apache.hadoop.hbase.TableName.valueOf(key), val) + end + rpc = get_peer_config(id) + unless rpc.nil? + rpc = ReplicationPeerConfigUtil.appendExcludeTableCFsToReplicationPeerConfig(map, rpc) + @admin.updateReplicationPeerConfig(id, rpc) + end + end + end + + # Remove some exclude-tableCFs from the exclude-tableCFs config for the specified peer + def remove_peer_exclude_tableCFs(id, excludeTableCFs) + unless excludeTableCFs.nil? + # convert tableCFs to TableName + map = java.util.HashMap.new + excludeTableCFs.each do |key, val| + map.put(org.apache.hadoop.hbase.TableName.valueOf(key), val) + end + rpc = get_peer_config(id) + unless rpc.nil? + rpc = ReplicationPeerConfigUtil.removeExcludeTableCFsFromReplicationPeerConfig(map, rpc, id) + @admin.updateReplicationPeerConfig(id, rpc) + end + end + end + + # Set new namespaces config for the specified peer + def set_peer_namespaces(id, namespaces) + unless namespaces.nil? + ns_set = java.util.HashSet.new + namespaces.each do |n| + ns_set.add(n) + end + rpc = get_peer_config(id) + unless rpc.nil? + rpc.setNamespaces(ns_set) + @admin.updateReplicationPeerConfig(id, rpc) + end + end + end + + # Add some namespaces for the specified peer + def add_peer_namespaces(id, namespaces) + unless namespaces.nil? + rpc = get_peer_config(id) + unless rpc.nil? + if rpc.getNamespaces.nil? + ns_set = java.util.HashSet.new + else + ns_set = java.util.HashSet.new(rpc.getNamespaces) + end + namespaces.each do |n| + ns_set.add(n) + end + builder = org.apache.hadoop.hbase.replication.ReplicationPeerConfig + .newBuilder(rpc) + builder.setNamespaces(ns_set) + @admin.updateReplicationPeerConfig(id, builder.build) + end + end + end + + # Remove some namespaces for the specified peer + def remove_peer_namespaces(id, namespaces) + unless namespaces.nil? + rpc = get_peer_config(id) + unless rpc.nil? + ns_set = rpc.getNamespaces + unless ns_set.nil? + ns_set = java.util.HashSet.new(ns_set) + namespaces.each do |n| + ns_set.remove(n) + end + end + builder = org.apache.hadoop.hbase.replication.ReplicationPeerConfig + .newBuilder(rpc) + builder.setNamespaces(ns_set) + @admin.updateReplicationPeerConfig(id, builder.build) + end + end + end + + # Show the current namespaces config for the specified peer + def show_peer_namespaces(peer_config) + namespaces = peer_config.get_namespaces + if !namespaces.nil? + namespaces = java.util.ArrayList.new(namespaces) + java.util.Collections.sort(namespaces) + return namespaces.join(';') + else + return nil + end + end + + # Set new bandwidth config for the specified peer + def set_peer_bandwidth(id, bandwidth) + rpc = get_peer_config(id) + unless rpc.nil? + rpc.setBandwidth(bandwidth) + @admin.updateReplicationPeerConfig(id, rpc) + end + end + + # Append exclude namespaces config for the specified peer + def append_peer_exclude_namespaces(id, namespaces) + unless namespaces.nil? + rpc = get_peer_config(id) + unless rpc.nil? + if rpc.getExcludeNamespaces.nil? + ns_set = java.util.HashSet.new + else + ns_set = java.util.HashSet.new(rpc.getExcludeNamespaces) + end + namespaces.each do |n| + ns_set.add(n) + end + builder = ReplicationPeerConfig.newBuilder(rpc) + builder.setExcludeNamespaces(ns_set) + @admin.updateReplicationPeerConfig(id, builder.build) + end + end + end + + # Remove exclude namespaces config for the specified peer + def remove_peer_exclude_namespaces(id, namespaces) + unless namespaces.nil? + rpc = get_peer_config(id) + unless rpc.nil? + ns_set = rpc.getExcludeNamespaces + unless ns_set.nil? + ns_set = java.util.HashSet.new(ns_set) + namespaces.each do |n| + ns_set.remove(n) + end + end + builder = ReplicationPeerConfig.newBuilder(rpc) + builder.setExcludeNamespaces(ns_set) + @admin.updateReplicationPeerConfig(id, builder.build) + end + end + end + + def set_peer_replicate_all(id, replicate_all) + rpc = get_peer_config(id) + return if rpc.nil? + rpc.setReplicateAllUserTables(replicate_all) + @admin.updateReplicationPeerConfig(id, rpc) + end + + def set_peer_serial(id, peer_serial) + rpc = get_peer_config(id) + return if rpc.nil? + rpc_builder = org.apache.hadoop.hbase.replication.ReplicationPeerConfig + .newBuilder(rpc) + new_rpc = rpc_builder.setSerial(peer_serial).build + @admin.updateReplicationPeerConfig(id, new_rpc) + end + + # Set exclude namespaces config for the specified peer + def set_peer_exclude_namespaces(id, exclude_namespaces) + return if exclude_namespaces.nil? + exclude_ns_set = java.util.HashSet.new + exclude_namespaces.each do |n| + exclude_ns_set.add(n) + end + rpc = get_peer_config(id) + return if rpc.nil? + rpc.setExcludeNamespaces(exclude_ns_set) + @admin.updateReplicationPeerConfig(id, rpc) + end + + # Show the exclude namespaces config for the specified peer + def show_peer_exclude_namespaces(peer_config) + namespaces = peer_config.getExcludeNamespaces + return nil if namespaces.nil? + namespaces = java.util.ArrayList.new(namespaces) + java.util.Collections.sort(namespaces) + '!' + namespaces.join(';') + end + + # Set exclude tableCFs config for the specified peer + def set_peer_exclude_tableCFs(id, exclude_tableCFs) + return if exclude_tableCFs.nil? + # convert tableCFs to TableName + map = java.util.HashMap.new + exclude_tableCFs.each do |key, val| + map.put(org.apache.hadoop.hbase.TableName.valueOf(key), val) + end + rpc = get_peer_config(id) + return if rpc.nil? + rpc.setExcludeTableCFsMap(map) + @admin.updateReplicationPeerConfig(id, rpc) + end + + # Show the exclude tableCFs config for the specified peer + def show_peer_exclude_tableCFs(peer_config) + tableCFs = peer_config.getExcludeTableCFsMap + return nil if tableCFs.nil? + '!' + ReplicationPeerConfigUtil.convertToString(tableCFs) + end + + #---------------------------------------------------------------------------------------------- + # Enables a table's replication switch + def enable_tablerep(table_name) + tableName = TableName.valueOf(table_name) + @admin.enableTableReplication(tableName) + end + + #---------------------------------------------------------------------------------------------- + # Disables a table's replication switch + def disable_tablerep(table_name) + tableName = TableName.valueOf(table_name) + @admin.disableTableReplication(tableName) + end + + def list_peer_configs + map = java.util.HashMap.new + peers = @admin.listReplicationPeers + peers.each do |peer| + map.put(peer.getPeerId, peer.getPeerConfig) + end + map + end + + def get_peer_config(id) + @admin.getReplicationPeerConfig(id) + end + + def update_peer_config(id, args = {}) + # Optional parameters + config = args.fetch(CONFIG, nil) + data = args.fetch(DATA, nil) + + # Create and populate a ReplicationPeerConfig + replication_peer_config = get_peer_config(id) + builder = org.apache.hadoop.hbase.replication.ReplicationPeerConfig + .newBuilder(replication_peer_config) + unless config.nil? + builder.putAllConfiguration(config) + end + + unless data.nil? + # Convert Strings to Bytes for peer_data + data.each do |key, val| + builder.putPeerData(Bytes.to_bytes(key), Bytes.to_bytes(val)) + end + end + + @admin.updateReplicationPeerConfig(id, builder.build) + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase/rsgroup_admin.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase/rsgroup_admin.rb new file mode 100644 index 0000000000..6d08dbcfab --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase/rsgroup_admin.rb @@ -0,0 +1,233 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include Java +java_import org.apache.hadoop.hbase.util.Pair + +require 'hbase/balancer_utils' + +# Wrapper for org.apache.hadoop.hbase.group.GroupAdminClient +# Which is an API to manage region server groups + +module Hbase + class RSGroupAdmin + def initialize(connection) + @connection = connection + @admin = org.apache.hadoop.hbase.rsgroup.RSGroupAdminClient.new(connection) + @hb_admin = @connection.getAdmin + end + + def close + @admin.close + end + + #-------------------------------------------------------------------------- + # Returns a list of groups in hbase + def list_rs_groups + @admin.listRSGroups + end + + #-------------------------------------------------------------------------- + # get a group's information + def get_rsgroup(group_name) + group = @admin.getRSGroupInfo(group_name) + raise(ArgumentError, 'Group does not exist: ' + group_name) if group.nil? + group + end + + #-------------------------------------------------------------------------- + # add a group + def add_rs_group(group_name) + @admin.addRSGroup(group_name) + end + + #-------------------------------------------------------------------------- + # remove a group + def remove_rs_group(group_name) + @admin.removeRSGroup(group_name) + end + + #-------------------------------------------------------------------------- + # balance a group + def balance_rs_group(group_name, *args) + request = ::Hbase::BalancerUtils.create_balance_request(args) + @admin.balanceRSGroup(group_name, request) + end + + #-------------------------------------------------------------------------- + # move server to a group + def move_servers(dest, *args) + servers = java.util.HashSet.new + args[0].each do |s| + servers.add(org.apache.hadoop.hbase.net.Address.fromString(s)) + end + @admin.moveServers(servers, dest) + end + + #-------------------------------------------------------------------------- + # move tables to a group + def move_tables(dest, *args) + tables = java.util.HashSet.new + args[0].each do |s| + tables.add(org.apache.hadoop.hbase.TableName.valueOf(s)) + end + @admin.moveTables(tables, dest) + end + + #-------------------------------------------------------------------------- + # move namespaces to a group + def move_namespaces(dest, *args) + tables = get_tables(args[0]) + @admin.moveTables(tables, dest) + end + + #-------------------------------------------------------------------------- + # get group of server + def get_rsgroup_of_server(server) + res = @admin.getRSGroupOfServer( + org.apache.hadoop.hbase.net.Address.fromString(server) + ) + raise(ArgumentError, 'Server has no group: ' + server) if res.nil? + res + end + + #-------------------------------------------------------------------------- + # get group of table + def get_rsgroup_of_table(table) + res = @admin.getRSGroupInfoOfTable( + org.apache.hadoop.hbase.TableName.valueOf(table) + ) + raise(ArgumentError, 'Table has no group: ' + table) if res.nil? + res + end + + #-------------------------------------------------------------------------- + # move server and table to a group + def move_servers_tables(dest, *args) + servers = get_servers(args[0]) + tables = java.util.HashSet.new + args[1].each do |t| + tables.add(org.apache.hadoop.hbase.TableName.valueOf(t)) + end + @admin.moveServersAndTables(servers, tables, dest) + end + + #-------------------------------------------------------------------------- + # move server and namespace to a group + def move_servers_namespaces(dest, *args) + servers = get_servers(args[0]) + tables = get_tables(args[1]) + @admin.moveServersAndTables(servers, tables, dest) + end + + def get_servers(servers) + server_set = java.util.HashSet.new + servers.each do |s| + server_set.add(org.apache.hadoop.hbase.net.Address.fromString(s)) + end + server_set + end + + def get_tables(namespaces) + table_set = java.util.HashSet.new + error = "Can't find a namespace: " + namespaces.each do |ns| + raise(ArgumentError, "#{error}#{ns}") unless namespace_exists?(ns) + table_set.addAll(get_tables_by_namespace(ns)) + end + table_set + end + + # Get tables by namespace + def get_tables_by_namespace(ns) + tables = java.util.HashSet.new + tablelist = @hb_admin.listTableNamesByNamespace(ns).map(&:getNameAsString) + tablelist.each do |table| + tables.add(org.apache.hadoop.hbase.TableName.valueOf(table)) + end + tables + end + + # Does Namespace exist + def namespace_exists?(ns) + return !@hb_admin.getNamespaceDescriptor(ns).nil? + rescue org.apache.hadoop.hbase.NamespaceNotFoundException + return false + end + + #-------------------------------------------------------------------------- + # remove decommissioned server from rsgroup + def remove_servers(*args) + # Flatten params array + args = args.flatten.compact + servers = java.util.HashSet.new + args.each do |s| + servers.add(org.apache.hadoop.hbase.net.Address.fromString(s)) + end + @admin.removeServers(servers) + end + + #-------------------------------------------------------------------------- + # rename rsgroup + def rename_rsgroup(oldname, newname) + @admin.renameRSGroup(oldname, newname) + end + + #---------------------------------------------------------------------------------------------- + # modify a rsgroup configuration + def alter_rsgroup_config(rsgroup_name, *args) + # Fail if table name is not a string + raise(ArgumentError, 'RSGroup name must be of type String') unless rsgroup_name.is_a?(String) + + group = @admin.getRSGroupInfo(rsgroup_name) + + raise(ArgumentError, 'RSGroup does not exist') unless group + + configuration = java.util.HashMap.new + configuration.putAll(group.getConfiguration) + + # Flatten params array + args = args.flatten.compact + + # Start defining the table + args.each do |arg| + unless arg.is_a?(Hash) + raise(ArgumentError, "#{arg.class} of #{arg.inspect} is not of Hash type") + end + method = arg[::HBaseConstants::METHOD] + if method == 'unset' + configuration.remove(arg[::HBaseConstants::NAME]) + elsif method == 'set' + arg.delete(::HBaseConstants::METHOD) + for k, v in arg + v = v.to_s unless v.nil? + configuration.put(k, v) + end + else + raise(ArgumentError, "Unknown method #{method}") + end + end + @admin.updateRSGroupConfig(rsgroup_name, configuration) + end + + #---------------------------------------------------------------------------------------------- + # Updates the configuration of all the regionservers in the rsgroup. + def update_rsgroup_config(groupName) + @admin.updateConfiguration(groupName) + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase/security.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase/security.rb new file mode 100644 index 0000000000..0bb0e8bdd2 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase/security.rb @@ -0,0 +1,214 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include Java + +# Wrapper for org.apache.hadoop.hbase.client.HBaseAdmin + +module Hbase + class SecurityAdmin + def initialize(admin) + @admin = admin + @connection = @admin.getConnection + end + + def close + @admin.close + end + + #---------------------------------------------------------------------------------------------- + def grant(user, permissions, table_name = nil, family = nil, qualifier = nil) + security_available? + + # TODO: need to validate user name + + begin + # Verify that the specified permission is valid + if permissions.nil? || permissions.empty? + raise(ArgumentError, 'Invalid permission: no actions associated with user') + end + + perm = org.apache.hadoop.hbase.security.access.Permission.new( + permissions.to_java_bytes + ) + + if !table_name.nil? + tablebytes = table_name.to_java_bytes + # check if the tablename passed is actually a namespace + if isNamespace?(table_name) + # Namespace should exist first. + namespace_name = table_name[1...table_name.length] + raise(ArgumentError, "Can't find a namespace: #{namespace_name}") unless + namespace_exists?(namespace_name) + + org.apache.hadoop.hbase.security.access.AccessControlClient.grant( + @connection, namespace_name, user, perm.getActions + ) + else + # Table should exist + raise(ArgumentError, "Can't find a table: #{table_name}") unless exists?(table_name) + + tableName = org.apache.hadoop.hbase.TableName.valueOf(table_name) + htd = @admin.getTableDescriptor(tableName) + + unless family.nil? + raise(ArgumentError, "Can't find a family: #{family}") unless htd.hasFamily(family.to_java_bytes) + end + + fambytes = family.to_java_bytes unless family.nil? + qualbytes = qualifier.to_java_bytes unless qualifier.nil? + + org.apache.hadoop.hbase.security.access.AccessControlClient.grant( + @connection, tableName, user, fambytes, qualbytes, perm.getActions + ) + end + else + # invoke cp endpoint to perform access controls + org.apache.hadoop.hbase.security.access.AccessControlClient.grant( + @connection, user, perm.getActions + ) + end + end + end + + #---------------------------------------------------------------------------------------------- + def revoke(user, table_name = nil, family = nil, qualifier = nil) + security_available? + + # TODO: need to validate user name + + begin + if !table_name.nil? + # check if the tablename passed is actually a namespace + if isNamespace?(table_name) + # Namespace should exist first. + namespace_name = table_name[1...table_name.length] + raise(ArgumentError, "Can't find a namespace: #{namespace_name}") unless namespace_exists?(namespace_name) + + tablebytes = table_name.to_java_bytes + org.apache.hadoop.hbase.security.access.AccessControlClient.revoke( + @connection, namespace_name, user + ) + else + # Table should exist + raise(ArgumentError, "Can't find a table: #{table_name}") unless exists?(table_name) + + tableName = org.apache.hadoop.hbase.TableName.valueOf(table_name) + htd = @admin.getTableDescriptor(tableName) + + unless family.nil? + raise(ArgumentError, "Can't find a family: #{family}") unless htd.hasFamily(family.to_java_bytes) + end + + fambytes = family.to_java_bytes unless family.nil? + qualbytes = qualifier.to_java_bytes unless qualifier.nil? + + org.apache.hadoop.hbase.security.access.AccessControlClient.revoke( + @connection, tableName, user, fambytes, qualbytes + ) + end + else + perm = org.apache.hadoop.hbase.security.access.Permission.new(''.to_java_bytes) + org.apache.hadoop.hbase.security.access.AccessControlClient.revoke( + @connection, user, perm.getActions + ) + end + end + end + + #---------------------------------------------------------------------------------------------- + def user_permission(table_regex = nil) + security_available? + all_perms = org.apache.hadoop.hbase.security.access.AccessControlClient.getUserPermissions( + @connection, table_regex + ) + res = {} + count = 0 + all_perms.each do |value| + user_name = value.getUser + permission = value.getPermission + table = '' + family = '' + qualifier = '' + if !table_regex.nil? && isNamespace?(table_regex) + nsPerm = permission.to_java(org.apache.hadoop.hbase.security.access.NamespacePermission) + namespace = nsPerm.getNamespace + elsif !table_regex.nil? && isTablePermission?(permission) + tblPerm = permission.to_java(org.apache.hadoop.hbase.security.access.TablePermission) + namespace = tblPerm.getNamespace + table = !tblPerm.getTableName.nil? ? tblPerm.getTableName.getNameAsString : '' + family = !tblPerm.getFamily.nil? ? + org.apache.hadoop.hbase.util.Bytes.toStringBinary(tblPerm.getFamily) : '' + qualifier = !tblPerm.getQualifier.nil? ? + org.apache.hadoop.hbase.util.Bytes.toStringBinary(tblPerm.getQualifier) : '' + end + + action = org.apache.hadoop.hbase.security.access.Permission.new permission.getActions + + if block_given? + yield(user_name, "#{namespace},#{table},#{family},#{qualifier}: #{action}") + else + res[user_name] ||= {} + res[user_name]["#{family}:#{qualifier}"] = action + end + count += 1 + end + + (block_given? ? count : res) + end + + # Does table exist? + def exists?(table_name) + @admin.tableExists(TableName.valueOf(table_name)) + end + + def isNamespace?(table_name) + table_name.start_with?('@') + end + + def isTablePermission?(permission) + permission.java_kind_of?(org.apache.hadoop.hbase.security.access.TablePermission) + end + + # Does Namespace exist + def namespace_exists?(namespace_name) + return !@admin.getNamespaceDescriptor(namespace_name).nil? + rescue org.apache.hadoop.hbase.NamespaceNotFoundException => e + return false + end + + # Make sure that security features are available + def security_available? + caps = [] + begin + # Try the getSecurityCapabilities API where supported. + # We only need to look at AUTHORIZATION, the AccessController doesn't support + # CELL_AUTHORIZATION without AUTHORIZATION also available. + caps = @admin.getSecurityCapabilities + rescue + # If we are unable to use getSecurityCapabilities, fall back with a check for + # deployment of the ACL table + raise(ArgumentError, 'DISABLED: Security features are not available') unless \ + exists?(org.apache.hadoop.hbase.security.access.PermissionStorage::ACL_TABLE_NAME.getNameAsString) + return + end + raise(ArgumentError, 'DISABLED: Security features are not available') unless \ + caps.include? org.apache.hadoop.hbase.client.security.SecurityCapability::AUTHORIZATION + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase/table.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase/table.rb new file mode 100644 index 0000000000..fc63785d68 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase/table.rb @@ -0,0 +1,907 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include Java + +java_import org.apache.hadoop.hbase.util.Bytes +java_import org.apache.hadoop.hbase.client.RegionReplicaUtil + +# Wrapper for org.apache.hadoop.hbase.client.Table + +module Hbase + # rubocop:disable Metrics/ClassLength + class Table + include HBaseConstants + @@thread_pool = nil + + # Add the command 'name' to table s.t. the shell command also called via 'name' + # and has an internal method also called 'name'. + # + # e.g. name = scan, adds table.scan which calls Scan.scan + def self.add_shell_command(name) + add_command(name, name, name) + end + + # add a named command to the table instance + # + # name - name of the command that should added to the table + # (eg. sending 'scan' here would allow you to do table.scan) + # shell_command - name of the command in the shell + # internal_method_name - name of the method in the shell command to forward the call + def self.add_command(name, shell_command, internal_method_name) + method = name.to_sym + class_eval do + define_method method do |*args| + @shell.internal_command(shell_command, internal_method_name, self, *args) + end + end + end + + # General help for the table + # class level so we can call it from anywhere + def self.help + <<-EOF +Help for table-reference commands. + +You can either create a table via 'create' and then manipulate the table via commands like 'put', 'get', etc. +See the standard help information for how to use each of these commands. + +However, as of 0.96, you can also get a reference to a table, on which you can invoke commands. +For instance, you can get create a table and keep around a reference to it via: + + hbase> t = create 't', 'cf' + +Or, if you have already created the table, you can get a reference to it: + + hbase> t = get_table 't' + +You can do things like call 'put' on the table: + + hbase> t.put 'r', 'cf:q', 'v' + +which puts a row 'r' with column family 'cf', qualifier 'q' and value 'v' into table t. + +To read the data out, you can scan the table: + + hbase> t.scan + +which will read all the rows in table 't'. + +Essentially, any command that takes a table name can also be done via table reference. +Other commands include things like: get, delete, deleteall, +get_all_columns, get_counter, count, incr. These functions, along with +the standard JRuby object methods are also available via tab completion. + +For more information on how to use each of these commands, you can also just type: + + hbase> t.help 'scan' + +which will output more information on how to use that command. + +You can also do general admin actions directly on a table; things like enable, disable, +flush and drop just by typing: + + hbase> t.enable + hbase> t.flush + hbase> t.disable + hbase> t.drop + +Note that after dropping a table, your reference to it becomes useless and further usage +is undefined (and not recommended). +EOF + end + + #--------------------------------------------------------------------------------------------- + + # let external objects read the underlying table object + attr_reader :table + # let external objects read the table name + attr_reader :name + + def initialize(table, shell) + @table = table + @name = @table.getName.getNameAsString + @shell = shell + @converters = {} + @timestamp_format_epoch = table.getConfiguration.getBoolean( + HConstants::SHELL_TIMESTAMP_FORMAT_EPOCH_KEY, + HConstants::DEFAULT_SHELL_TIMESTAMP_FORMAT_EPOCH) + end + + def close + @table.close + end + + # Note the below methods are prefixed with '_' to hide them from the average user, as + # they will be much less likely to tab complete to the 'dangerous' internal method + #---------------------------------------------------------------------------------------------- + + # Put a cell 'value' at specified table/row/column + def _put_internal(row, column, value, timestamp = nil, args = {}) + p = org.apache.hadoop.hbase.client.Put.new(row.to_s.to_java_bytes) + family, qualifier = parse_column_name(column) + if args.any? + attributes = args[ATTRIBUTES] + set_attributes(p, attributes) if attributes + visibility = args[VISIBILITY] + set_cell_visibility(p, visibility) if visibility + ttl = args[TTL] + set_op_ttl(p, ttl) if ttl + end + # Case where attributes are specified without timestamp + if timestamp.is_a?(Hash) + timestamp.each do |k, v| + if k == 'ATTRIBUTES' + set_attributes(p, v) + elsif k == 'VISIBILITY' + set_cell_visibility(p, v) + elsif k == 'TTL' + set_op_ttl(p, v) + end + end + timestamp = nil + end + if timestamp + p.addColumn(family, qualifier, timestamp, value.to_s.to_java_bytes) + else + p.addColumn(family, qualifier, value.to_s.to_java_bytes) + end + @table.put(p) + end + + #---------------------------------------------------------------------------------------------- + # Create a Delete mutation + def _createdelete_internal(row, column = nil, + timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP, + args = {}, all_version = true) + temptimestamp = timestamp + if temptimestamp.is_a?(Hash) + timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP + end + d = org.apache.hadoop.hbase.client.Delete.new(row.to_s.to_java_bytes, timestamp) + if temptimestamp.is_a?(Hash) + temptimestamp.each do |_k, v| + if v.is_a?(String) + set_cell_visibility(d, v) if v + end + end + end + if args.any? + visibility = args[VISIBILITY] + set_cell_visibility(d, visibility) if visibility + end + if column != "" + if column && all_version + family, qualifier = parse_column_name(column) + if qualifier + d.addColumns(family, qualifier, timestamp) + else + d.addFamily(family, timestamp) + end + elsif column && !all_version + family, qualifier = parse_column_name(column) + if qualifier + d.addColumn(family, qualifier, timestamp) + else + d.addFamilyVersion(family, timestamp) + end + end + end + d + end + + #---------------------------------------------------------------------------------------------- + # Delete rows using prefix + def _deleterows_internal(row, column = nil, + timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP, + args = {}, all_version = true) + cache = row['CACHE'] ? row['CACHE'] : 100 + prefix = row['ROWPREFIXFILTER'] + + # create scan to get table names using prefix + scan = org.apache.hadoop.hbase.client.Scan.new + scan.setStartStopRowForPrefixScan(prefix.to_java_bytes) + # Run the scanner to get all rowkeys + scanner = @table.getScanner(scan) + # Create a list to store all deletes + list = java.util.ArrayList.new + # Iterate results + iter = scanner.iterator + while iter.hasNext + row = iter.next + key = org.apache.hadoop.hbase.util.Bytes.toStringBinary(row.getRow) + d = _createdelete_internal(key, column, timestamp, args, all_version) + list.add(d) + if list.size >= cache + @table.delete(list) + list.clear + end + end + @table.delete(list) + end + + #---------------------------------------------------------------------------------------------- + # Delete a cell + def _delete_internal(row, column, + timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP, + args = {}, all_version = false) + _deleteall_internal(row, column, timestamp, args, all_version) + end + + #---------------------------------------------------------------------------------------------- + # Delete a row + def _deleteall_internal(row, column = nil, + timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP, + args = {}, all_version = true) + # delete operation doesn't need read permission. Retaining the read check for + # meta table as a part of HBASE-5837. + if is_meta_table? + if row.is_a?(Hash) and row.key?('ROWPREFIXFILTER') + raise ArgumentError, 'deleteall with ROWPREFIXFILTER in hbase:meta is not allowed.' + else + raise ArgumentError, 'Row Not Found' if _get_internal(row).nil? + end + end + if row.is_a?(Hash) + _deleterows_internal(row, column, timestamp, args, all_version) + else + d = _createdelete_internal(row, column, timestamp, args, all_version) + @table.delete(d) + end + end + + #---------------------------------------------------------------------------------------------- + # Increment a counter atomically + # rubocop:disable Metrics/AbcSize, CyclomaticComplexity, MethodLength + def _incr_internal(row, column, value = nil, args = {}) + value = 1 if value.is_a?(Hash) + value ||= 1 + incr = org.apache.hadoop.hbase.client.Increment.new(row.to_s.to_java_bytes) + family, qualifier = parse_column_name(column) + if args.any? + attributes = args[ATTRIBUTES] + visibility = args[VISIBILITY] + set_attributes(incr, attributes) if attributes + set_cell_visibility(incr, visibility) if visibility + ttl = args[TTL] + set_op_ttl(incr, ttl) if ttl + end + incr.addColumn(family, qualifier, value) + result = @table.increment(incr) + return nil if result.isEmpty + + # Fetch cell value + cell = result.listCells[0] + org.apache.hadoop.hbase.util.Bytes.toLong(cell.getValueArray, + cell.getValueOffset, cell.getValueLength) + end + + #---------------------------------------------------------------------------------------------- + # appends the value atomically + def _append_internal(row, column, value, args = {}) + append = org.apache.hadoop.hbase.client.Append.new(row.to_s.to_java_bytes) + family, qualifier = parse_column_name(column) + if args.any? + attributes = args[ATTRIBUTES] + visibility = args[VISIBILITY] + set_attributes(append, attributes) if attributes + set_cell_visibility(append, visibility) if visibility + ttl = args[TTL] + set_op_ttl(append, ttl) if ttl + end + append.add(family, qualifier, value.to_s.to_java_bytes) + result = @table.append(append) + return nil if result.isEmpty + + # Fetch cell value + cell = result.listCells[0] + org.apache.hadoop.hbase.util.Bytes.toStringBinary(cell.getValueArray, + cell.getValueOffset, cell.getValueLength) + end + # rubocop:enable Metrics/AbcSize, CyclomaticComplexity, MethodLength + + #---------------------------------------------------------------------------------------------- + # Count rows in a table + def _count_internal(interval = 1000, scan = nil, cacheBlocks=false) + raise(ArgumentError, 'Scan argument should be org.apache.hadoop.hbase.client.Scan') \ + unless scan.nil? || scan.is_a?(org.apache.hadoop.hbase.client.Scan) + # We can safely set scanner caching with the first key only filter + + if scan.nil? + scan = org.apache.hadoop.hbase.client.Scan.new + scan.setCacheBlocks(cacheBlocks) + scan.setCaching(10) + scan.setFilter(org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter.new) + else + scan.setCacheBlocks(cacheBlocks) + filter = scan.getFilter + firstKeyOnlyFilter = org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter.new + if filter.nil? + scan.setFilter(firstKeyOnlyFilter) + else + firstKeyOnlyFilter.setReversed(filter.isReversed) + scan.setFilter(org.apache.hadoop.hbase.filter.FilterList.new(filter, firstKeyOnlyFilter)) + end + end + + # Run the scanner + scanner = @table.getScanner(scan) + count = 0 + iter = scanner.iterator + + # Iterate results + while iter.hasNext + row = iter.next + count += 1 + next unless block_given? && count % interval == 0 + # Allow command modules to visualize counting process + yield(count, + org.apache.hadoop.hbase.util.Bytes.toStringBinary(row.getRow)) + end + + scanner.close + # Return the counter + count + end + + #---------------------------------------------------------------------------------------------- + # Get from table + def _get_internal(row, *args) + get = org.apache.hadoop.hbase.client.Get.new(row.to_s.to_java_bytes) + maxlength = -1 + count = 0 + @converters.clear + + # Normalize args + args = args.first if args.first.is_a?(Hash) + if args.is_a?(String) || args.is_a?(Array) + columns = [args].flatten.compact + args = { COLUMNS => columns } + end + + # + # Parse arguments + # + unless args.is_a?(Hash) + raise ArgumentError, "Failed parse of #{args.inspect}, #{args.class}" + end + + # Get maxlength parameter if passed + maxlength = args.delete(MAXLENGTH) if args[MAXLENGTH] + filter = args.delete(FILTER) if args[FILTER] + attributes = args[ATTRIBUTES] + authorizations = args[AUTHORIZATIONS] + consistency = args.delete(CONSISTENCY) if args[CONSISTENCY] + replicaId = args.delete(REGION_REPLICA_ID) if args[REGION_REPLICA_ID] + converter = args.delete(FORMATTER) || nil + converter_class = args.delete(FORMATTER_CLASS) || 'org.apache.hadoop.hbase.util.Bytes' + unless args.empty? + columns = args[COLUMN] || args[COLUMNS] + vers = if args[VERSIONS] + args[VERSIONS] + else + 1 + end + if columns + # Normalize types, convert string to an array of strings + columns = [columns] if columns.is_a?(String) + + # At this point it is either an array or some unsupported stuff + unless columns.is_a?(Array) + raise ArgumentError, "Failed parse column argument type #{args.inspect}, #{args.class}" + end + + # Get each column name and add it to the filter + columns.each do |column| + family, qualifier = parse_column_name(column.to_s) + if qualifier + get.addColumn(family, qualifier) + else + get.addFamily(family) + end + end + + # Additional params + get.setMaxVersions(vers) + get.setTimeStamp(args[TIMESTAMP]) if args[TIMESTAMP] + get.setTimeRange(args[TIMERANGE][0], args[TIMERANGE][1]) if args[TIMERANGE] + else + if attributes + set_attributes(get, attributes) + elsif authorizations + set_authorizations(get, authorizations) + else + # May have passed TIMESTAMP and row only; wants all columns from ts. + unless ts = args[TIMESTAMP] || tr = args[TIMERANGE] + raise ArgumentError, "Failed parse of #{args.inspect}, #{args.class}" + end + end + + get.setMaxVersions(vers) + # Set the timestamp/timerange + get.setTimeStamp(ts.to_i) if args[TIMESTAMP] + get.setTimeRange(args[TIMERANGE][0], args[TIMERANGE][1]) if args[TIMERANGE] + end + set_attributes(get, attributes) if attributes + set_authorizations(get, authorizations) if authorizations + end + + if filter.class == String + get.setFilter( + org.apache.hadoop.hbase.filter.ParseFilter.new.parseFilterString(filter.to_java_bytes) + ) + else + get.setFilter(filter) + end + + get.setConsistency(org.apache.hadoop.hbase.client.Consistency.valueOf(consistency)) if consistency + get.setReplicaId(replicaId) if replicaId + + # Call hbase for the results + result = @table.get(get) + return nil if result.isEmpty + + # Get stale info from results + is_stale = result.isStale + count += 1 + + # Print out results. Result can be Cell or RowResult. + res = {} + result.listCells.each do |c| + # Get the family and qualifier of the cell without escaping non-printable characters. It is crucial that + # column is constructed in this consistent way to that it can be used as a key. + family_bytes = org.apache.hadoop.hbase.util.Bytes.copy(c.getFamilyArray, c.getFamilyOffset, c.getFamilyLength) + qualifier_bytes = org.apache.hadoop.hbase.util.Bytes.copy(c.getQualifierArray, c.getQualifierOffset, c.getQualifierLength) + column = "#{family_bytes}:#{qualifier_bytes}" + + value = to_string(column, c, maxlength, converter_class, converter) + + # Use the FORMATTER to determine how column is printed + family = convert_bytes(family_bytes, converter_class, converter) + qualifier = convert_bytes(qualifier_bytes, converter_class, converter) + formatted_column = "#{family}:#{qualifier}" + + if block_given? + yield(formatted_column, value) + else + res[formatted_column] = value + end + end + + # If block given, we've yielded all the results, otherwise just return them + (block_given? ? [count, is_stale] : res) + end + + #---------------------------------------------------------------------------------------------- + # Fetches and decodes a counter value from hbase + def _get_counter_internal(row, column) + family, qualifier = parse_column_name(column.to_s) + # Format get request + get = org.apache.hadoop.hbase.client.Get.new(row.to_s.to_java_bytes) + get.addColumn(family, qualifier) + get.setMaxVersions(1) + + # Call hbase + result = @table.get(get) + return nil if result.isEmpty + + # Fetch cell value + cell = result.listCells[0] + org.apache.hadoop.hbase.util.Bytes.toLong(cell.getValueArray, + cell.getValueOffset, cell.getValueLength) + end + + def _hash_to_scan(args) + if args.any? + enablemetrics = args['ALL_METRICS'].nil? ? false : args['ALL_METRICS'] + enablemetrics ||= !args['METRICS'].nil? + filter = args['FILTER'] + startrow = args['STARTROW'] || '' + stoprow = args['STOPROW'] + rowprefixfilter = args['ROWPREFIXFILTER'] + timestamp = args['TIMESTAMP'] + columns = args['COLUMNS'] || args['COLUMN'] || [] + # If CACHE_BLOCKS not set, then default 'true'. + cache_blocks = args['CACHE_BLOCKS'].nil? ? true : args['CACHE_BLOCKS'] + cache = args['CACHE'] || 0 + reversed = args['REVERSED'] || false + versions = args['VERSIONS'] || 1 + timerange = args[TIMERANGE] + raw = args['RAW'] || false + attributes = args[ATTRIBUTES] + authorizations = args[AUTHORIZATIONS] + consistency = args[CONSISTENCY] + # Normalize column names + columns = [columns] if columns.class == String + limit = args['LIMIT'] || -1 + replica_id = args[REGION_REPLICA_ID] + isolation_level = args[ISOLATION_LEVEL] + read_type = args[READ_TYPE] + allow_partial_results = args[ALLOW_PARTIAL_RESULTS].nil? ? false : args[ALLOW_PARTIAL_RESULTS] + batch = args[BATCH] || -1 + max_result_size = args[MAX_RESULT_SIZE] || -1 + + unless columns.is_a?(Array) + raise ArgumentError, 'COLUMNS must be specified as a String or an Array' + end + + scan = if stoprow + org.apache.hadoop.hbase.client.Scan.new(startrow.to_java_bytes, stoprow.to_java_bytes) + else + org.apache.hadoop.hbase.client.Scan.new(startrow.to_java_bytes) + end + + # This will overwrite any startrow/stoprow settings + scan.setStartStopRowForPrefixScan(rowprefixfilter.to_java_bytes) if rowprefixfilter + + # Clear converters from last scan. + @converters.clear + + columns.each do |c| + family, qualifier = parse_column_name(c.to_s) + if qualifier + scan.addColumn(family, qualifier) + else + scan.addFamily(family) + end + end + + if filter.class == String + scan.setFilter( + org.apache.hadoop.hbase.filter.ParseFilter.new.parseFilterString(filter.to_java_bytes) + ) + else + scan.setFilter(filter) + end + + scan.setScanMetricsEnabled(enablemetrics) if enablemetrics + scan.setTimestamp(timestamp) if timestamp + scan.setCacheBlocks(cache_blocks) + scan.setReversed(reversed) + scan.setCaching(cache) if cache > 0 + scan.setMaxVersions(versions) if versions > 1 + scan.setTimeRange(timerange[0], timerange[1]) if timerange + scan.setRaw(raw) + scan.setLimit(limit) if limit > 0 + set_attributes(scan, attributes) if attributes + set_authorizations(scan, authorizations) if authorizations + scan.setConsistency(org.apache.hadoop.hbase.client.Consistency.valueOf(consistency)) if consistency + scan.setReplicaId(replica_id) if replica_id + scan.setIsolationLevel(org.apache.hadoop.hbase.client.IsolationLevel.valueOf(isolation_level)) if isolation_level + scan.setReadType(org.apache.hadoop.hbase.client::Scan::ReadType.valueOf(read_type)) if read_type + scan.setAllowPartialResults(allow_partial_results) if allow_partial_results + scan.setBatch(batch) if batch > 0 + scan.setMaxResultSize(max_result_size) if max_result_size > 0 + else + scan = org.apache.hadoop.hbase.client.Scan.new + end + + scan + end + + def _get_scanner(args) + @table.getScanner(_hash_to_scan(args)) + end + + #---------------------------------------------------------------------------------------------- + # Scans whole table or a range of keys and returns rows matching specific criteria + def _scan_internal(args = {}, scan = nil) + raise(ArgumentError, 'Args should be a Hash') unless args.is_a?(Hash) + raise(ArgumentError, 'Scan argument should be org.apache.hadoop.hbase.client.Scan') \ + unless scan.nil? || scan.is_a?(org.apache.hadoop.hbase.client.Scan) + + maxlength = args.delete('MAXLENGTH') || -1 + converter = args.delete(FORMATTER) || nil + converter_class = args.delete(FORMATTER_CLASS) || 'org.apache.hadoop.hbase.util.Bytes' + count = 0 + res = {} + + # Start the scanner + scan = scan.nil? ? _hash_to_scan(args) : scan + scanner = @table.getScanner(scan) + iter = scanner.iterator + + # Iterate results + while iter.hasNext + row = iter.next + key = convert_bytes(row.getRow, nil, converter) + is_stale |= row.isStale + + row.listCells.each do |c| + # Get the family and qualifier of the cell without escaping non-printable characters. It is crucial that + # column is constructed in this consistent way to that it can be used as a key. + family_bytes = org.apache.hadoop.hbase.util.Bytes.copy(c.getFamilyArray, c.getFamilyOffset, c.getFamilyLength) + qualifier_bytes = org.apache.hadoop.hbase.util.Bytes.copy(c.getQualifierArray, c.getQualifierOffset, c.getQualifierLength) + column = "#{family_bytes}:#{qualifier_bytes}" + + cell = to_string(column, c, maxlength, converter_class, converter) + + # Use the FORMATTER to determine how column is printed + family = convert_bytes(family_bytes, converter_class, converter) + qualifier = convert_bytes(qualifier_bytes, converter_class, converter) + formatted_column = "#{family}:#{qualifier}" + + if block_given? + yield(key, "column=#{formatted_column}, #{cell}") + else + res[key] ||= {} + res[key][formatted_column] = cell + end + end + # One more row processed + count += 1 + end + + scanner.close + (block_given? ? [count, is_stale] : res) + end + + # Apply OperationAttributes to puts/scans/gets + def set_attributes(oprattr, attributes) + raise(ArgumentError, 'Attributes must be a Hash type') unless attributes.is_a?(Hash) + for k, v in attributes + v = v.to_s unless v.nil? + oprattr.setAttribute(k.to_s, v.to_java_bytes) + end + end + + def set_cell_permissions(op, permissions) + raise(ArgumentError, 'Permissions must be a Hash type') unless permissions.is_a?(Hash) + map = java.util.HashMap.new + permissions.each do |user, perms| + map.put(user.to_s, org.apache.hadoop.hbase.security.access.Permission.new( + perms.to_java_bytes + )) + end + op.setACL(map) + end + + def set_cell_visibility(oprattr, visibility) + oprattr.setCellVisibility( + org.apache.hadoop.hbase.security.visibility.CellVisibility.new( + visibility.to_s + ) + ) + end + + def set_authorizations(oprattr, authorizations) + raise(ArgumentError, 'Authorizations must be a Array type') unless authorizations.is_a?(Array) + auths = [authorizations].flatten.compact + oprattr.setAuthorizations( + org.apache.hadoop.hbase.security.visibility.Authorizations.new( + auths.to_java(:string) + ) + ) + end + + def set_op_ttl(op, ttl) + op.setTTL(ttl.to_java(:long)) + end + + #---------------------------- + # Add general administration utilities to the shell + # each of the names below adds this method name to the table + # by callling the corresponding method in the shell + # Add single method utilities to the current class + # Generally used for admin functions which just have one name and take the table name + def self.add_admin_utils(*args) + args.each do |method| + define_method method do |*method_args| + @shell.command(method, @name, *method_args) + end + end + end + + # Add the following admin utilities to the table + add_admin_utils :enable, :disable, :flush, :drop, :describe, :snapshot + + #---------------------------- + # give the general help for the table + # or the named command + def help(command = nil) + # if there is a command, get the per-command help from the shell + if command + begin + return @shell.help_command(command) + rescue NoMethodError + puts "Command \'#{command}\' does not exist. Please see general table help." + return nil + end + end + @shell.help('table_help') + end + + # Table to string + def to_s + cl = self.class + "#{cl} - #{@name}" + end + + # Standard ruby call to get the return value for an object + # overriden here so we get sane semantics for printing a table on return + def inspect + to_s + end + + #---------------------------------------------------------------------------------------- + # Helper methods + + # Returns a list of column names in the table + def get_all_columns + @table.table_descriptor.getFamilies.map do |family| + "#{family.getNameAsString}:" + end + end + + # Checks if current table is one of the 'meta' tables + def is_meta_table? + org.apache.hadoop.hbase.TableName::META_TABLE_NAME.equals(@table.getName) + end + + # Given a column specification in the format FAMILY[:QUALIFIER[:CONVERTER]] + # 1. Save the converter for the given column + # 2. Return a 2-element Array with [family, qualifier or nil], discarding the converter if provided + # + # @param [String] column specification + def parse_column_name(column) + spec = parse_column_format_spec(column) + set_column_converter(spec.family, spec.qualifier, spec.converter) unless spec.converter.nil? + [spec.family, spec.qualifier] + end + + def toLocalDateTime(millis) + if @timestamp_format_epoch + return millis + else + instant = java.time.Instant.ofEpochMilli(millis) + return java.time.LocalDateTime.ofInstant(instant, java.time.ZoneId.systemDefault()).toString + end + end + + # Make a String of the passed kv + # Intercept cells whose format we know such as the info:regioninfo in hbase:meta + def to_string(column, kv, maxlength = -1, converter_class = nil, converter = nil) + if is_meta_table? + if column == 'info:regioninfo' || column == 'info:splitA' || column == 'info:splitB' || \ + column.start_with?('info:merge') + hri = org.apache.hadoop.hbase.HRegionInfo.parseFromOrNull(kv.getValueArray, + kv.getValueOffset, kv.getValueLength) + return format('timestamp=%s, value=%s', toLocalDateTime(kv.getTimestamp), + hri.nil? ? '' : hri.toString) + end + if column == 'info:serverstartcode' + if kv.getValueLength > 0 + str_val = org.apache.hadoop.hbase.util.Bytes.toLong(kv.getValueArray, + kv.getValueOffset, kv.getValueLength) + else + str_val = org.apache.hadoop.hbase.util.Bytes.toStringBinary(kv.getValueArray, + kv.getValueOffset, kv.getValueLength) + end + return format('timestamp=%s, value=%s', toLocalDateTime(kv.getTimestamp), str_val) + end + end + + if org.apache.hadoop.hbase.CellUtil.isDelete(kv) + val = "timestamp=#{toLocalDateTime(kv.getTimestamp)}, type=#{org.apache.hadoop.hbase.KeyValue::Type.codeToType(kv.getTypeByte)}" + else + val = "timestamp=#{toLocalDateTime(kv.getTimestamp)}, value=#{convert(column, kv, converter_class, converter)}" + end + maxlength != -1 ? val[0, maxlength] : val + end + + def convert(column, kv, converter_class = 'org.apache.hadoop.hbase.util.Bytes', converter = 'toStringBinary') + # use org.apache.hadoop.hbase.util.Bytes as the default class + converter_class = 'org.apache.hadoop.hbase.util.Bytes' unless converter_class + # use org.apache.hadoop.hbase.util.Bytes::toStringBinary as the default convertor + converter = 'toStringBinary' unless converter + if @converters.key?(column) + # lookup the CONVERTER for certain column - "cf:qualifier" + matches = /c\((.+)\)\.(.+)/.match(@converters[column]) + if matches.nil? + # cannot match the pattern of 'c(className).functionname' + # use the default klazz_name + converter = @converters[column] + else + klazz_name = matches[1] + converter = matches[2] + end + end + # apply the converter + convert_bytes(org.apache.hadoop.hbase.CellUtil.cloneValue(kv), klazz_name, converter) + end + + def convert_bytes(bytes, converter_class = nil, converter_method = nil) + # Avoid nil + converter_class ||= 'org.apache.hadoop.hbase.util.Bytes' + converter_method ||= 'toStringBinary' + eval(converter_class).method(converter_method).call(bytes) + end + + def convert_bytes_with_position(bytes, offset, len, converter_class, converter_method) + # Avoid nil + converter_class ||= 'org.apache.hadoop.hbase.util.Bytes' + converter_method ||= 'toStringBinary' + eval(converter_class).method(converter_method).call(bytes, offset, len) + end + + # store the information designating what part of a column should be printed, and how + ColumnFormatSpec = Struct.new(:family, :qualifier, :converter) + + ## + # Parse the column specification for formatting used by shell commands like :scan + # + # Strings should be structured as follows: + # FAMILY:QUALIFIER[:CONVERTER] + # Where: + # - FAMILY is the column family + # - QUALIFIER is the column qualifier. Non-printable characters should be left AS-IS and should NOT BE escaped. + # - CONVERTER is optional and is the name of a converter (like toLong) to apply + # + # @param [String] column + # @return [ColumnFormatSpec] family, qualifier, and converter as Java bytes + private def parse_column_format_spec(column) + split = org.apache.hadoop.hbase.CellUtil.parseColumn(column.to_java_bytes) + family = split[0] + qualifier = nil + converter = nil + if split.length > 1 + parts = org.apache.hadoop.hbase.CellUtil.parseColumn(split[1]) + qualifier = parts[0] + if parts.length > 1 + converter = parts[1] + end + end + + ColumnFormatSpec.new(family, qualifier, converter) + end + + private def set_column_converter(family, qualifier, converter) + @converters["#{String.from_java_bytes(family)}:#{String.from_java_bytes(qualifier)}"] = String.from_java_bytes(converter) + end + + # if the column spec contains CONVERTER information, to get rid of :CONVERTER info from column pair. + # 1. return back normal column pair as usual, i.e., "cf:qualifier[:CONVERTER]" to "cf" and "qualifier" only + # 2. register the CONVERTER information based on column spec - "cf:qualifier" + # + # Deprecated for removal in 4.0.0 + def set_converter(column) + family = String.from_java_bytes(column[0]) + parts = org.apache.hadoop.hbase.CellUtil.parseColumn(column[1]) + if parts.length > 1 + @converters["#{family}:#{String.from_java_bytes(parts[0])}"] = String.from_java_bytes(parts[1]) + column[1] = parts[0] + end + end + extend Gem::Deprecate + deprecate :set_converter, "4.0.0", nil, nil + + #---------------------------------------------------------------------------------------------- + # Get the split points for the table + def _get_splits_internal + locator = @table.getRegionLocator + locator.getAllRegionLocations + .select { |s| RegionReplicaUtil.isDefaultReplica(s.getRegion) } + .map { |i| Bytes.toStringBinary(i.getRegionInfo.getStartKey) } + .delete_if { |k| k == '' } + ensure + locator.close + end + end + # rubocop:enable Metrics/ClassLength +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase/taskmonitor.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase/taskmonitor.rb new file mode 100644 index 0000000000..2f4e428606 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase/taskmonitor.rb @@ -0,0 +1,194 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include Java + +# Add the $HBASE_HOME/lib directory to the ruby load_path to load jackson +if File.exist?(File.join(File.dirname(__FILE__), '..', 'lib')) + $LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..', 'lib') +end + +module Hbase + class TaskMonitor + #--------------------------------------------------------------------------------------------- + # Represents information reported by a server on a single MonitoredTask + class Task + def initialize(taskMap, host) + taskMap.entrySet.each do |entry| + k = entry.getKey + v = entry.getValue + case k + when 'statustimems' + @statustime = Time.at(v.getAsLong / 1000) + when 'status' + @status = v.getAsString + when 'starttimems' + @starttime = Time.at(v.getAsLong / 1000) + when 'description' + @description = v.getAsString + when 'state' + @state = v.getAsString + end + end + + @host = host + end + + def statustime + # waiting IPC handlers often have statustime = -1, in this case return starttime + return @statustime if @statustime > Time.at(-1) + @starttime + end + + attr_reader :host + attr_reader :status + attr_reader :starttime + attr_reader :description + attr_reader :state + end + + def initialize(configuration) + @conf = configuration + @conn = org.apache.linkis.manager.engineplugin.hbase.HBaseConnectionManager.getInstance().getConnection(@conf) + @admin = @conn.getAdmin + end + + #--------------------------------------------------------------------------------------------------- + # Returns a filtered list of tasks on the given host + def tasksOnHost(filter, host) + java_import 'java.net.URL' + java_import 'java.net.SocketException' + java_import 'java.io.InputStreamReader' + java_import 'org.apache.hbase.thirdparty.com.google.gson.JsonParser' + + infoport = @admin.getClusterStatus.getLoad(host).getInfoServerPort.to_s + + begin + schema = "http://" + url = schema + host.hostname + ':' + infoport + '/rs-status?format=json&filter=' + filter + json = URL.new(url).openStream + rescue SocketException => e + # Let's try with https when SocketException occur + schema = "https://" + url = schema + host.hostname + ':' + infoport + '/rs-status?format=json&filter=' + filter + json = URL.new(url).openStream + end + + parser = JsonParser.new + + # read and parse JSON + begin + tasks_array_list = parser.parse(InputStreamReader.new(json, 'UTF-8')).getAsJsonArray + ensure + json.close + end + # convert to an array of TaskMonitor::Task instances + tasks = [] + tasks_array_list.each do |t| + tasks.unshift Task.new(t.getAsJsonObject, host) + end + + tasks + end + + #--------------------------------------------------------------------------------------------------- + # Prints a table of filtered tasks on requested hosts + def tasks(filter, hosts) + # put all tasks on all requested hosts in the same list + tasks = [] + hosts.each do |host| + tasks.concat(tasksOnHost(filter, host)) + end + + puts(format('%d tasks as of: %s', tasks.size, Time.now.strftime('%Y-%m-%d %H:%M:%S'))) + + if tasks.empty? + puts('No ' + filter + ' tasks currently running.') + else + + # determine table width + longestStatusWidth = 0 + longestDescriptionWidth = 0 + tasks.each do |t| + longestStatusWidth = [longestStatusWidth, t.status.length].max + longestDescriptionWidth = [longestDescriptionWidth, t.description.length].max + end + + # set the maximum character width of each column, without padding + hostWidth = 15 + startTimeWidth = 19 + stateWidth = 8 + descriptionWidth = [32, longestDescriptionWidth].min + statusWidth = [36, longestStatusWidth + 27].min + + rowSeparator = '+' + '-' * (hostWidth + 2) + + '+' + '-' * (startTimeWidth + 2) + + '+' + '-' * (stateWidth + 2) + + '+' + '-' * (descriptionWidth + 2) + + '+' + '-' * (statusWidth + 2) + '+' + + # print table header + cells = [setCellWidth('Host', hostWidth), + setCellWidth('Start Time', startTimeWidth), + setCellWidth('State', stateWidth), + setCellWidth('Description', descriptionWidth), + setCellWidth('Status', statusWidth)] + + line = format('| %s | %s | %s | %s | %s |', *cells) + + puts(rowSeparator) + puts(line) + + # print table content + tasks.each do |t| + cells = [setCellWidth(t.host.hostname, hostWidth), + setCellWidth(t.starttime.strftime('%Y-%m-%d %H:%M:%S'), startTimeWidth), + setCellWidth(t.state, stateWidth), + setCellWidth(t.description, descriptionWidth), + setCellWidth(format('%s (since %d seconds ago)', t.status, Time.now - t.statustime), statusWidth)] + + line = format('| %s | %s | %s | %s | %s |', *cells) + + puts(rowSeparator) + puts(line) + end + puts(rowSeparator) + + end + end + + #--------------------------------------------------------------------------------------------------- + # + # Helper methods + # + + # right-pad with spaces or truncate with ellipses to match passed width + def setCellWidth(cellContent, width) + numCharsTooShort = width - cellContent.length + if numCharsTooShort < 0 + # cellContent is too long, so truncate + return cellContent[0, [width - 3, 0].max] + '.' * [3, width].min + else + # cellContent is requested width or too short, so right-pad with zero or more spaces + return cellContent + ' ' * numCharsTooShort + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase/visibility_labels.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase/visibility_labels.rb new file mode 100644 index 0000000000..e2ae2cad35 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase/visibility_labels.rb @@ -0,0 +1,143 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include Java +java_import org.apache.hadoop.hbase.security.visibility.VisibilityClient +java_import org.apache.hadoop.hbase.security.visibility.VisibilityConstants +java_import org.apache.hadoop.hbase.util.Bytes + +module Hbase + class VisibilityLabelsAdmin + def initialize(admin) + @admin = admin + @connection = @admin.getConnection + end + + def close + @admin.close + end + + def add_labels(*args) + visibility_feature_available? + # Normalize args + labels = [args].flatten.compact if args.is_a?(Array) + raise(ArgumentError, 'Arguments cannot be null') if labels.empty? + + begin + response = VisibilityClient.addLabels(@connection, labels.to_java(:string)) + if response.nil? + raise(ArgumentError, 'DISABLED: Visibility labels feature is not available') + end + labelsWithException = '' + list = response.getResultList + list.each do |result| + if result.hasException + labelsWithException += Bytes.toString(result.getException.getValue.toByteArray) + end + end + raise(ArgumentError, labelsWithException) unless labelsWithException.empty? + end + end + + def set_auths(user, *args) + visibility_feature_available? + # Normalize args + auths = [args].flatten.compact if args.is_a?(Array) + + begin + response = VisibilityClient.setAuths(@connection, auths.to_java(:string), user) + if response.nil? + raise(ArgumentError, 'DISABLED: Visibility labels feature is not available') + end + labelsWithException = '' + list = response.getResultList + list.each do |result| + if result.hasException + labelsWithException += Bytes.toString(result.getException.getValue.toByteArray) + end + end + raise(ArgumentError, labelsWithException) unless labelsWithException.empty? + end + end + + def get_auths(user) + visibility_feature_available? + begin + response = VisibilityClient.getAuths(@connection, user) + if response.nil? + raise(ArgumentError, 'DISABLED: Visibility labels feature is not available') + end + return response.getAuthList + end + end + + def list_labels(regex = '.*') + visibility_feature_available? + begin + response = VisibilityClient.listLabels(@connection, regex) + if response.nil? + raise(ArgumentError, 'DISABLED: Visibility labels feature is not available') + end + return response.getLabelList + end + end + + def clear_auths(user, *args) + visibility_feature_available? + # Normalize args + auths = [args].flatten.compact if args.is_a?(Array) + + begin + response = VisibilityClient.clearAuths(@connection, auths.to_java(:string), user) + if response.nil? + raise(ArgumentError, 'DISABLED: Visibility labels feature is not available') + end + labelsWithException = '' + list = response.getResultList + list.each do |result| + if result.hasException + labelsWithException += Bytes.toString(result.getException.getValue.toByteArray) + end + end + raise(ArgumentError, labelsWithException) unless labelsWithException.empty? + end + end + + # Make sure that lables table is available + def visibility_feature_available? + caps = [] + begin + # Try the getSecurityCapabilities API where supported. + caps = @admin.getSecurityCapabilities + rescue + # If we are unable to use getSecurityCapabilities, fall back with a check for + # deployment of the labels table + raise(ArgumentError, 'DISABLED: Visibility labels feature is not available') unless \ + exists?(VisibilityConstants::LABELS_TABLE_NAME) + return + end + raise(ArgumentError, 'DISABLED: Visibility labels feature is not available') unless \ + caps.include? org.apache.hadoop.hbase.client.security.SecurityCapability::CELL_VISIBILITY + end + + # Does table exist? + def exists?(table_name) + @admin.tableExists(TableName.valueOf(table_name)) + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase_constants.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase_constants.rb new file mode 100644 index 0000000000..76631c91cf --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase_constants.rb @@ -0,0 +1,131 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# HBase ruby classes. +# Has wrapper classes for org.apache.hadoop.hbase.client.Admin +# and for org.apache.hadoop.hbase.client.Table. Classes take +# Formatters on construction and outputs any results using +# Formatter methods. These classes are only really for use by +# the hirb.rb HBase Shell script; they don't make much sense elsewhere. +# For example, the exists method on Admin class prints to the formatter +# whether the table exists and returns nil regardless. +include Java + +java_import('java.lang.Integer') { |_package, name| "J#{name}" } +java_import('java.lang.Long') { |_package, name| "J#{name}" } +java_import('java.lang.Boolean') { |_package, name| "J#{name}" } + +module HBaseConstants + ALLOW_PARTIAL_RESULTS = 'ALLOW_PARTIAL_RESULTS'.freeze + ALL_METRICS = 'ALL_METRICS'.freeze + ATTRIBUTES = 'ATTRIBUTES'.freeze + AUTHORIZATIONS = 'AUTHORIZATIONS'.freeze + BATCH = 'BATCH'.freeze + CACHE = 'CACHE'.freeze + CACHE_BLOCKS = 'CACHE_BLOCKS'.freeze + CLASSNAME = 'CLASSNAME'.freeze + CLONE_SFT = 'CLONE_SFT'.freeze + CLUSTER_KEY = 'CLUSTER_KEY'.freeze + COLUMN = 'COLUMN'.freeze + COLUMNS = 'COLUMNS'.freeze + CONFIG = 'CONFIG'.freeze + CONFIGURATION = org.apache.hadoop.hbase.HConstants::CONFIGURATION + CONSISTENCY = 'CONSISTENCY'.freeze + DATA = 'DATA'.freeze + ENDPOINT_CLASSNAME = 'ENDPOINT_CLASSNAME'.freeze + FILTER = 'FILTER'.freeze + FORMATTER = 'FORMATTER'.freeze + FORMATTER_CLASS = 'FORMATTER_CLASS'.freeze + INTERVAL = 'INTERVAL'.freeze + IN_MEMORY = org.apache.hadoop.hbase.HConstants::IN_MEMORY + IN_MEMORY_COMPACTION = org.apache.hadoop.hbase.HColumnDescriptor::IN_MEMORY_COMPACTION + ISOLATION_LEVEL = 'ISOLATION_LEVEL'.freeze + LIMIT = 'LIMIT'.freeze + LOCALITY_THRESHOLD = 'LOCALITY_THRESHOLD'.freeze + MAXLENGTH = 'MAXLENGTH'.freeze + MAX_RESULT_SIZE = 'MAX_RESULT_SIZE'.freeze + METADATA = org.apache.hadoop.hbase.HConstants::METADATA + METHOD = 'METHOD'.freeze + METRICS = 'METRICS'.freeze + NAME = org.apache.hadoop.hbase.HConstants::NAME + NAMESPACE = 'NAMESPACE'.freeze + NAMESPACES = 'NAMESPACES'.freeze + NONE = 'NONE'.freeze + NUMREGIONS = 'NUMREGIONS'.freeze + POLICY = 'POLICY'.freeze + RAW = 'RAW'.freeze + READ_TYPE = 'READ_TYPE'.freeze + REGEX = 'REGEX'.freeze + REGIONSERVER = 'REGIONSERVER'.freeze + REGION_REPLICATION = 'REGION_REPLICATION'.freeze + REGION_REPLICA_ID = 'REGION_REPLICA_ID'.freeze + REPLICATION_SCOPE = 'REPLICATION_SCOPE'.freeze + RESTORE_ACL = 'RESTORE_ACL'.freeze + REVERSED = 'REVERSED'.freeze + ROWPREFIXFILTER = 'ROWPREFIXFILTER'.freeze + SERIAL = 'SERIAL'.freeze + SERVER_NAME = 'SERVER_NAME'.freeze + SKIP_FLUSH = 'SKIP_FLUSH'.freeze + SPLITALGO = 'SPLITALGO'.freeze + SPLITS = 'SPLITS'.freeze + SPLITS_FILE = 'SPLITS_FILE'.freeze + STARTROW = 'STARTROW'.freeze + STATE = 'STATE'.freeze + STOPROW = 'STOPROW'.freeze + TABLE = 'TABLE'.freeze + TABLE_CFS = 'TABLE_CFS'.freeze + TABLE_NAME = 'TABLE_NAME'.freeze + TABLE_NAMES = 'TABLE_NAMES'.freeze + TIMERANGE = 'TIMERANGE'.freeze + TIMESTAMP = 'TIMESTAMP'.freeze + TYPE = 'TYPE'.freeze + USER = 'USER'.freeze + VALUE = 'VALUE'.freeze + VERSIONS = org.apache.hadoop.hbase.HConstants::VERSIONS + VISIBILITY = 'VISIBILITY'.freeze + + # aliases + ENDKEY = STOPROW + ENDROW = STOPROW + STARTKEY = STARTROW + + # Load constants from hbase java API + def self.promote_constants(constants) + # The constants to import are all in uppercase + constants.each do |c| + next if c =~ /DEFAULT_.*/ || c != c.upcase + next if eval("defined?(#{c})") + eval("#{c} = '#{c}'") + end + end + + promote_constants(org.apache.hadoop.hbase.HColumnDescriptor.constants) + promote_constants(org.apache.hadoop.hbase.HTableDescriptor.constants) +end + +# Ensure that hbase class definitions are imported +require 'hbase/hbase' +require 'hbase/admin' +require 'hbase/taskmonitor' +require 'hbase/table' +require 'hbase/quotas' +require 'hbase/replication_admin' +require 'hbase/security' +require 'hbase/visibility_labels' +require 'hbase/rsgroup_admin' diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase_shell.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase_shell.rb new file mode 100644 index 0000000000..08a1dece17 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hbase_shell.rb @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Ruby has a stdlib named 'shell' so using "require 'shell'" does not +# work if our shell implementation is not on the local filesystem. +# this is the absolute path to our shell implementation when packaged +# in a jar. The level of indirection provided by this file lets things +# still behave the same as in earlier releases if folks unpackage the +# jar contents onto the local filesystem if they need that for some +# other reason. +require 'uri:classloader:/hbase-ruby/shell.rb' diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hirb.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hirb.rb new file mode 100644 index 0000000000..cdc6ab46f5 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/hirb.rb @@ -0,0 +1,272 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# File passed to org.jruby.Main by bin/hbase. Pollutes jirb with hbase imports +# and hbase commands and then loads jirb. Outputs a banner that tells user +# where to find help, shell version, and loads up a custom hirb. +# +# In noninteractive mode, runs commands from stdin until completion or an error. +# On success will exit with status 0, on any problem will exit non-zero. Callers +# should only rely on "not equal to 0", because the current error exit code of 1 +# will likely be updated to diffentiate e.g. invalid commands, incorrect args, +# permissions, etc. + +# TODO: Interrupt a table creation or a connection to a bad master. Currently +# has to time out. Below we've set down the retries for rpc and hbase but +# still can be annoying (And there seem to be times when we'll retry for +# ever regardless) +# TODO: Add support for listing and manipulating catalog tables, etc. +# TODO: Encoding; need to know how to go from ruby String to UTF-8 bytes + +# Run the java magic include and import basic HBase types that will help ease +# hbase hacking. +include Java + +# Some goodies for hirb. Should these be left up to the user's discretion? +require 'irb/completion' +require 'pathname' + +# Add the directory names in hbase.jruby.sources commandline option +# to the ruby load path so I can load up my HBase ruby modules +# sources = java.lang.System.getProperty('hbase.ruby.sources') +# $LOAD_PATH.unshift Pathname.new(sources) +$LOAD_PATH.unshift 'uri:classloader:/hbase-ruby' +args_input_by_env = java.lang.System.getProperty('hbase.ruby.args') +if args_input_by_env.nil? + args = Array.new +else + args = args_input_by_env.split("$") +end + +# +# FIXME: Switch args processing to getopt +# +# See if there are args for this shell. If any, read and then strip from ARGV +# so they don't go through to irb. Output shell 'usage' if user types '--help' +cmdline_help = < exception + message = exception.to_s + # exception unwrapping in shell means we'll have to handle Java exceptions + # as a special case in order to format them properly. + if exception.is_a? java.lang.Exception + $stderr.puts 'java exception' + message = exception.get_message + end + # Include the 'ERROR' string to try to make transition easier for scripts that + # may have already been relying on grepping output. + puts "ERROR #{exception.class}: #{message}" + if $fullBacktrace + # re-raising the will include a backtrace and exit. + raise exception + else + exit 1 + end + end +end \ No newline at end of file diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/irb/hirb.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/irb/hirb.rb new file mode 100644 index 0000000000..7fc4091248 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/irb/hirb.rb @@ -0,0 +1,206 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +require 'rbconfig' + +module IRB + WINDOZE = RbConfig::CONFIG['host_os'] =~ /mswin|mingw/ + + # Subclass of IRB so can intercept methods + class HIRB < Irb + def initialize(workspace = nil, interactive = true, input_method = nil) + # This is ugly. Our 'help' method above provokes the following message + # on irb construction: 'irb: warn: can't alias help from irb_help.' + # Below, we reset the output so its pointed at /dev/null during irb + # construction just so this message does not come out after we emit + # the banner. Other attempts at playing with the hash of methods + # down in IRB didn't seem to work. I think the worst thing that can + # happen is the shell exiting because of failed IRB construction with + # no error (though we're not blanking STDERR) + + # Map the '/dev/null' according to the running platform + # Under Windows platform the 'dev/null' is not fully compliant with unix, + # and the 'NUL' object need to be use instead. + devnull = '/dev/null' + devnull = 'NUL' if WINDOZE + f = File.open(devnull, 'w') + $stdout = f + # This is a workaround for the jruby issue 1372. + # The stderr is an input to stty to re-adjust the terminal for the error('stdin isnt a terminal') + # in case the command is piped with hbase shell(eg - >echo 'list' | bin/hbase shell) + if $stdin.tty? + `stty icrnl <&2` + end + @interactive = interactive + super(workspace, input_method) + ensure + f.close + $stdout = STDOUT + end + + def output_value + # Suppress output if last_value is 'nil' + # Otherwise, when user types help, get ugly 'nil' + # after all output. + super unless @context.last_value.nil? + end + + # Copied from irb.rb and overrides the rescue Exception block so the + # Shell::exception_handler can deal with the exceptions. + def eval_input + @scanner.set_prompt do + |ltype, indent, continue, line_no| + if ltype + f = @context.prompt_s + elsif continue + f = @context.prompt_c + elsif indent > 0 + f = @context.prompt_n + else + f = @context.prompt_i + end + f = "" unless f + if @context.prompting? + @context.io.prompt = p = prompt(f, ltype, indent, line_no) + else + @context.io.prompt = p = "" + end + if @context.auto_indent_mode + unless ltype + ind = prompt(@context.prompt_i, ltype, indent, line_no)[/.*\z/].size + + indent * 2 - p.size + ind += 2 if continue + @context.io.prompt = p + " " * ind if ind > 0 + end + end + end + + @scanner.set_input(@context.io) do + signal_status(:IN_INPUT) do + if l = @context.io.gets + print l if @context.verbose? + else + if @context.ignore_eof? and @context.io.readable_after_eof? + l = "\n" + if @context.verbose? + printf "Use \"exit\" to leave %s\n", @context.ap_name + end + else + print "\n" + end + end + l + end + end + + @scanner.each_top_level_statement do |line, line_no| + signal_status(:IN_EVAL) do + begin + line.untaint + @context.evaluate(line, line_no) + output_value if @context.echo? + exc = nil + rescue Interrupt => exc + rescue SystemExit, SignalException + raise + rescue NameError => exc + raise exc unless @interactive + # HBASE-26880: Ignore NameError to prevent exiting Shell on mistyped commands. + rescue Exception => exc + # HBASE-26741: Raise exception so Shell::exception_handler can catch it. + # This modifies this copied method from JRuby so that the HBase shell can + # manage the exception and set a proper exit code on the process. + raise exc + end + if exc + if exc.backtrace && exc.backtrace[0] =~ /irb(2)?(\/.*|-.*|\.rb)?:/ && exc.class.to_s !~ /^IRB/ && + !(SyntaxError === exc) + irb_bug = true + else + irb_bug = false + end + + messages = [] + lasts = [] + levels = 0 + if exc.backtrace + count = 0 + exc.backtrace.each do |m| + m = @context.workspace.filter_backtrace(m) or next unless irb_bug + m = sprintf("%9d: from %s", (count += 1), m) + if messages.size < @context.back_trace_limit + messages.push(m) + elsif lasts.size < @context.back_trace_limit + lasts.push(m).shift + levels += 1 + end + end + end + attr = STDOUT.tty? ? ATTR_TTY : ATTR_PLAIN + print "#{attr[1]}Traceback#{attr[]} (most recent call last):\n" + unless lasts.empty? + puts lasts.reverse + printf "... %d levels...\n", levels if levels > 0 + end + puts messages.reverse + messages = exc.to_s.split(/\n/) + print "#{attr[1]}#{exc.class} (#{attr[4]}#{messages.shift}#{attr[0, 1]})#{attr[]}\n" + puts messages.map {|s| "#{attr[1]}#{s}#{attr[]}\n"} + print "Maybe IRB bug!\n" if irb_bug + end + end + end + end + end + + ## + # HBaseLoader serves a similar purpose to IRB::IrbLoader, but with a different separation of + # concerns. This loader allows us to directly get the path for a filename in ruby's load path, + # and then use that in IRB::Irb + module HBaseLoader + ## + # Determine the loadable path for a given filename by searching through $LOAD_PATH + # + # This serves a similar purpose to IRB::IrbLoader#search_file_from_ruby_path, but uses JRuby's + # loader, which allows us to find special paths like "uri:classloader" inside of a Jar. + # + # @param [String] filename + # @return [String] path + def self.path_for_load(filename) + return File.absolute_path(filename) if File.exist? filename + + # Get JRuby's LoadService from the global (singleton) instance of the runtime + # (org.jruby.Ruby), which allows us to use JRuby's tools for searching the load path. + runtime = org.jruby.Ruby.getGlobalRuntime + loader = runtime.getLoadService + search_state = loader.findFileForLoad filename + raise LoadError, "no such file to load -- #{filename}" if search_state.library.nil? + + search_state.loadName + end + + ## + # Return a file handle for the given file found in the load path + # + # @param [String] filename + # @return [FileInputMethod] InputMethod for passing to IRB session + def self.file_for_load(filename) + FileInputMethod.new(File.new(path_for_load(filename))) + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell.rb new file mode 100644 index 0000000000..f6b3efd945 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell.rb @@ -0,0 +1,638 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +require 'irb' +require 'irb/workspace' + +# +# Simple class to act as the main receiver for an IRB Workspace (and its respective ruby Binding) +# in our HBase shell. This will hold all the commands we want in our shell. +# +class HBaseReceiver < Object + def get_binding + binding + end +end + +## +# HBaseIOExtensions is a module to be "mixed-in" (ie. included) to Ruby's IO class. It is required +# if you want to use RubyLex with an IO object. RubyLex claims to take an IO but really wants an +# InputMethod. +module HBaseIOExtensions + def encoding + external_encoding + end +end + + +# Shell commands module +module Shell + @@commands = {} + def self.commands + @@commands + end + + @@command_groups = {} + def self.command_groups + @@command_groups + end + + def self.load_command(name, group, aliases = []) + return if commands[name] + + # Register command in the group + raise ArgumentError, "Unknown group: #{group}" unless command_groups[group] + command_groups[group][:commands] << name + + # Load command + begin + require "shell/commands/#{name}" + klass_name = name.to_s.gsub(/(?:^|_)(.)/) { Regexp.last_match(1).upcase } # camelize + commands[name] = eval("Commands::#{klass_name}") + aliases.each do |an_alias| + commands[an_alias] = commands[name] + end + rescue => e + raise "Can't load hbase shell command: #{name}. Error: #{e}\n#{e.backtrace.join("\n")}" + end + end + + def self.load_command_group(group, opts) + raise ArgumentError, "No :commands for group #{group}" unless opts[:commands] + + command_groups[group] = { + commands: [], + command_names: opts[:commands], + full_name: opts[:full_name] || group, + comment: opts[:comment] + } + + all_aliases = opts[:aliases] || {} + + opts[:commands].each do |command| + aliases = all_aliases[command] || [] + load_command(command, group, aliases) + end + end + + #---------------------------------------------------------------------- + # rubocop:disable Metrics/ClassLength + class Shell + attr_accessor :hbase + attr_accessor :interactive + alias interactive? interactive + + @debug = false + attr_accessor :debug + + # keep track of the passed exit code. nil means never called. + @exit_code = nil + attr_accessor :exit_code + + alias __exit__ exit + # exit the interactive shell and save that this + # happend via a call to exit + def exit(ret = 0) + @exit_code = ret + IRB.irb_exit(IRB.CurrentContext.irb, ret) + end + + def initialize(hbase, interactive = true) + self.hbase = hbase + self.interactive = interactive + end + + # Returns Admin class from admin.rb + def admin + @admin ||= hbase.admin + end + + def hbase_taskmonitor + @hbase_taskmonitor ||= hbase.taskmonitor + end + + def hbase_table(name) + hbase.table(name, self) + end + + def hbase_replication_admin + @hbase_replication_admin ||= hbase.replication_admin + end + + def hbase_security_admin + @hbase_security_admin ||= hbase.security_admin + end + + def hbase_visibility_labels_admin + @hbase_visibility_labels_admin ||= hbase.visibility_labels_admin + end + + def hbase_quotas_admin + @hbase_quotas_admin ||= hbase.quotas_admin + end + + def hbase_rsgroup_admin + @rsgroup_admin ||= hbase.rsgroup_admin + end + + ## + # Create singleton methods on the target receiver object for all the loaded commands + # + # Therefore, export_commands will create "class methods" if passed a Module/Class and if passed + # an instance the methods will not exist on any other instances of the instantiated class. + def export_commands(target) + # We need to store a reference to this Shell instance in the scope of this method so that it + # can be accessed later in the scope of the target object. + shell_inst = self + # Define each method as a lambda. We need to use a lambda (rather than a Proc or block) for + # its properties: preservation of local variables and return + ::Shell.commands.keys.each do |cmd| + target.send :define_singleton_method, cmd.to_sym, lambda { |*args| + ret = shell_inst.command(cmd.to_s, *args) + puts + ret + } + end + # Export help method + target.send :define_singleton_method, :help, lambda { |command = nil| + shell_inst.help(command) + nil + } + # Export tools method for backwards compatibility + target.send :define_singleton_method, :tools, lambda { + shell_inst.help_group('tools') + nil + } + end + + # Export HBase commands, constants, and variables to target receiver + def export_all(target) + raise ArgumentError, 'target should not be a module' if target.is_a? Module + + # add constants to class of target + target.class.include ::HBaseConstants + target.class.include ::HBaseQuotasConstants + # add instance variables @hbase and @shell for backwards compatibility + target.instance_variable_set :'@hbase', @hbase + target.instance_variable_set :'@shell', self + # add commands to target + export_commands(target) + end + + def command_instance(command) + ::Shell.commands[command.to_s].new(self) + end + + # call the method 'command' on the specified command + def command(command, *args) + internal_command(command, :command, *args) + end + + # call a specific internal method in the command instance + # command - name of the command to call + # method_name - name of the method on the command to call. Defaults to just 'command' + # args - to be passed to the named method + def internal_command(command, method_name = :command, *args) + command_instance(command).command_safe(debug, method_name, *args) + end + + def print_banner + puts 'HBase Shell' + puts 'Use "help" to get list of supported commands.' + puts 'Use "exit" to quit this interactive shell.' + puts 'For Reference, please visit: http://hbase.apache.org/2.0/book.html#shell' + print 'Version ' + command('version') + puts + end + + def help_multi_command(command) + puts "Command: #{command}" + puts command_instance(command).help + puts + nil + end + + def help_command(command) + puts command_instance(command).help + nil + end + + def help_group(group_name) + group = ::Shell.command_groups[group_name.to_s] + group[:commands].sort.each { |cmd| help_multi_command(cmd) } + if group[:comment] + puts '-' * 80 + puts + puts group[:comment] + puts + end + nil + end + + def help(command = nil) + if command + return help_command(command) if ::Shell.commands[command.to_s] + return help_group(command) if ::Shell.command_groups[command.to_s] + puts "ERROR: Invalid command or command group name: #{command}" + puts + end + + puts help_header + puts + puts 'COMMAND GROUPS:' + ::Shell.command_groups.each do |name, group| + puts ' Group name: ' + name + puts ' Commands: ' + group[:command_names].sort.join(', ') + puts + end + unless command + puts 'SHELL USAGE:' + help_footer + end + nil + end + + def help_header + "HBase Shell, version #{org.apache.hadoop.hbase.util.VersionInfo.getVersion}, " \ + "r#{org.apache.hadoop.hbase.util.VersionInfo.getRevision}, " \ + "#{org.apache.hadoop.hbase.util.VersionInfo.getDate}" + "\n" \ + "Type 'help \"COMMAND\"', (e.g. 'help \"get\"' -- the quotes are necessary) for help on a specific command.\n" \ + "Commands are grouped. Type 'help \"COMMAND_GROUP\"', (e.g. 'help \"general\"') for help on a command group." + end + + def help_footer + puts <<-HERE +Quote all names in HBase Shell such as table and column names. Commas delimit +command parameters. Type after entering a command to run it. +Dictionaries of configuration used in the creation and alteration of tables are +Ruby Hashes. They look like this: + + {'key1' => 'value1', 'key2' => 'value2', ...} + +and are opened and closed with curley-braces. Key/values are delimited by the +'=>' character combination. Usually keys are predefined constants such as +NAME, VERSIONS, COMPRESSION, etc. Constants do not need to be quoted. Type +'Object.constants' to see a (messy) list of all constants in the environment. + +If you are using binary keys or values and need to enter them in the shell, use +double-quote'd hexadecimal representation. For example: + + hbase> get 't1', "key\\x03\\x3f\\xcd" + hbase> get 't1', "key\\003\\023\\011" + hbase> put 't1', "test\\xef\\xff", 'f1:', "\\x01\\x33\\x40" + +The HBase shell is the (J)Ruby IRB with the above HBase-specific commands added. +For more on the HBase Shell, see http://hbase.apache.org/book.html + HERE + end + + @irb_workspace = nil + ## + # Returns an IRB Workspace for this shell instance with all the IRB and HBase commands installed + def get_workspace + return @irb_workspace unless @irb_workspace.nil? + + hbase_receiver = HBaseReceiver.new + # install all the IRB commands onto our receiver + IRB::ExtendCommandBundle.extend_object(hbase_receiver) + # Install all the hbase commands, constants, and instance variables @shell and @hbase. This + # will override names that conflict with IRB methods like "help". + export_all(hbase_receiver) + # make it so calling exit will hit our pass-through rather than going directly to IRB + hbase_receiver.send :define_singleton_method, :exit, lambda { |rc = 0| + @shell.exit(rc) + } + ::IRB::WorkSpace.new(hbase_receiver.get_binding) + end + + ## + # Runs a block and logs exception from both Ruby and Java, optionally discarding the traceback + # + # @param [Boolean] hide_traceback if true, Exceptions will be converted to + # a SystemExit so that the traceback is not printed + def self.exception_handler(hide_traceback) + begin + yield + rescue Exception => e + message = e.to_s + # exception unwrapping in shell means we'll have to handle Java exceptions + # as a special case in order to format them properly. + if e.is_a? java.lang.Exception + warn 'java exception' + message = e.get_message + end + # Include the 'ERROR' string to try to make transition easier for scripts that + # may have already been relying on grepping output. + puts "ERROR #{e.class}: #{message}" + raise e unless hide_traceback + + exit 1 + end + nil + end + end + # rubocop:enable Metrics/ClassLength +end + +# Load commands base class +require 'shell/commands' + +# Load all commands +Shell.load_command_group( + 'general', + full_name: 'GENERAL HBASE SHELL COMMANDS', + commands: %w[ + status + version + table_help + whoami + processlist + ] +) + +Shell.load_command_group( + 'ddl', + full_name: 'TABLES MANAGEMENT COMMANDS', + commands: %w[ + alter + create + describe + disable + disable_all + is_disabled + drop + drop_all + enable + enable_all + is_enabled + exists + list + show_filters + alter_status + alter_async + get_table + locate_region + list_regions + clone_table_schema + ], + aliases: { + 'describe' => ['desc'] + } +) + +Shell.load_command_group( + 'namespace', + full_name: 'NAMESPACE MANAGEMENT COMMANDS', + commands: %w[ + create_namespace + drop_namespace + alter_namespace + describe_namespace + list_namespace + list_namespace_tables + ] +) + +Shell.load_command_group( + 'dml', + full_name: 'DATA MANIPULATION COMMANDS', + commands: %w[ + count + delete + deleteall + get + get_counter + incr + put + scan + truncate + truncate_preserve + append + get_splits + ] +) + +Shell.load_command_group( + 'tools', + full_name: 'HBASE SURGERY TOOLS', + comment: "WARNING: Above commands are for 'experts'-only as misuse can damage an install", + commands: %w[ + assign + balancer + balance_switch + balancer_enabled + normalize + normalizer_switch + normalizer_enabled + is_in_maintenance_mode + clear_slowlog_responses + close_region + compact + compaction_switch + flush + flush_master_store + get_balancer_decisions + get_balancer_rejections + get_slowlog_responses + get_largelog_responses + major_compact + move + split + merge_region + unassign + zk_dump + wal_roll + hbck_chore_run + catalogjanitor_run + catalogjanitor_switch + catalogjanitor_enabled + cleaner_chore_run + cleaner_chore_switch + cleaner_chore_enabled + compact_rs + compaction_state + trace + snapshot_cleanup_switch + snapshot_cleanup_enabled + splitormerge_switch + splitormerge_enabled + clear_compaction_queues + list_deadservers + list_liveservers + list_unknownservers + clear_deadservers + clear_block_cache + stop_master + stop_regionserver + regioninfo + rit + list_decommissioned_regionservers + decommission_regionservers + recommission_regionserver + ], + # TODO: remove older hlog_roll command + aliases: { + 'wal_roll' => ['hlog_roll'] + } +) + +Shell.load_command_group( + 'replication', + full_name: 'CLUSTER REPLICATION TOOLS', + commands: %w[ + add_peer + remove_peer + list_peers + enable_peer + disable_peer + set_peer_replicate_all + set_peer_serial + set_peer_namespaces + append_peer_namespaces + remove_peer_namespaces + set_peer_exclude_namespaces + append_peer_exclude_namespaces + remove_peer_exclude_namespaces + show_peer_tableCFs + set_peer_tableCFs + set_peer_exclude_tableCFs + append_peer_exclude_tableCFs + remove_peer_exclude_tableCFs + set_peer_bandwidth + list_replicated_tables + append_peer_tableCFs + remove_peer_tableCFs + enable_table_replication + disable_table_replication + get_peer_config + list_peer_configs + update_peer_config + ] +) + +Shell.load_command_group( + 'snapshots', + full_name: 'CLUSTER SNAPSHOT TOOLS', + commands: %w[ + snapshot + clone_snapshot + restore_snapshot + delete_snapshot + delete_all_snapshot + delete_table_snapshots + list_snapshots + list_table_snapshots + ] +) + +Shell.load_command_group( + 'configuration', + full_name: 'ONLINE CONFIGURATION TOOLS', + commands: %w[ + update_config + update_all_config + update_rsgroup_config + ] +) + +Shell.load_command_group( + 'quotas', + full_name: 'CLUSTER QUOTAS TOOLS', + commands: %w[ + set_quota + list_quotas + list_quota_table_sizes + list_quota_snapshots + list_snapshot_sizes + enable_rpc_throttle + disable_rpc_throttle + enable_exceed_throttle_quota + disable_exceed_throttle_quota + ] +) + +Shell.load_command_group( + 'security', + full_name: 'SECURITY TOOLS', + comment: 'NOTE: Above commands are only applicable if running with the AccessController coprocessor', + commands: %w[ + list_security_capabilities + grant + revoke + user_permission + ] +) + +Shell.load_command_group( + 'procedures', + full_name: 'PROCEDURES & LOCKS MANAGEMENT', + commands: %w[ + list_procedures + list_locks + ] +) + +Shell.load_command_group( + 'visibility labels', + full_name: 'VISIBILITY LABEL TOOLS', + comment: 'NOTE: Above commands are only applicable if running with the VisibilityController coprocessor', + commands: %w[ + add_labels + list_labels + set_auths + get_auths + clear_auths + set_visibility + ] +) + +Shell.load_command_group( + 'rsgroup', + full_name: 'RSGroups', + comment: "NOTE: The rsgroup Coprocessor Endpoint must be enabled on the Master else commands fail with: + UnknownProtocolException: No registered Master Coprocessor Endpoint found for RSGroupAdminService", + commands: %w[ + list_rsgroups + get_rsgroup + add_rsgroup + remove_rsgroup + balance_rsgroup + move_servers_rsgroup + move_tables_rsgroup + move_namespaces_rsgroup + move_servers_tables_rsgroup + move_servers_namespaces_rsgroup + get_server_rsgroup + get_table_rsgroup + remove_servers_rsgroup + rename_rsgroup + alter_rsgroup_config + show_rsgroup_config + get_namespace_rsgroup + ] +) + +Shell.load_command_group( + 'storefiletracker', + full_name: 'StoreFileTracker', + commands: %w[ + change_sft + change_sft_all + ] +) diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands.rb new file mode 100644 index 0000000000..a40f737e79 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands.rb @@ -0,0 +1,201 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'shell/formatter' + +module Shell + module Commands + # rubocop:disable Metrics/ClassLength + class Command + def initialize(shell) + @shell = shell + end + + # gets the name that an operator would type into the shell + + def command_name + klass_name = self.class.name.split('::').last + command = klass_name.gsub(/([^\^])([A-Z])/, '\1_\2').downcase + command + end + + # wrap an execution of cmd to catch hbase exceptions + # cmd - command name to execute + # args - arguments to pass to the command + + # rubocop:disable Metrics/AbcSize + def command_safe(debug, cmd = :command, *args) + # Commands can overwrite start_time to skip time used in some kind of setup. + # See count.rb for example. + @start_time = Time.now + # send is internal ruby method to call 'cmd' with *args + # (everything is a message, so this is just the formal semantics to support that idiom) + translate_hbase_exceptions(*args) { send(cmd, *args) } + rescue => e + rootCause = e + + # JRuby9000 made RubyException respond to cause, ignore it for back compat + while !rootCause.is_a?(Exception) && rootCause.respond_to?(:cause) && !rootCause.cause.nil? + rootCause = rootCause.cause + end + if @shell.interactive? + puts + puts "ERROR: #{rootCause}" + puts "Backtrace: #{rootCause.backtrace.join("\n ")}" if debug + puts + puts "For usage try 'help \"#{command_name}\"'" + puts + else + raise rootCause + end + ensure + # If end_time is not already set by the command, use current time. + @end_time ||= Time.now + formatter.output_str(format('Took %.4f seconds', @end_time - @start_time)) + end + # rubocop:enable Metrics/AbcSize + + # Convenience functions to get different admins + # Returns HBase::Admin ruby class. + def admin + @shell.admin + end + + def taskmonitor + @shell.hbase_taskmonitor + end + + def table(name) + @shell.hbase_table(name) + end + + def replication_admin + @shell.hbase_replication_admin + end + + def security_admin + @shell.hbase_security_admin + end + + def visibility_labels_admin + @shell.hbase_visibility_labels_admin + end + + def quotas_admin + @shell.hbase_quotas_admin + end + + def rsgroup_admin + @shell.hbase_rsgroup_admin + end + + #---------------------------------------------------------------------- + # Creates formatter instance first time and then reuses it. + def formatter + @formatter ||= ::Shell::Formatter::Console.new + end + + # for testing purposes to catch the output of the commands + def set_formatter(formatter) + @formatter = formatter + end + + # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity + # rubocop:disable Metrics/MethodLength, Metrics/PerceivedComplexity + def translate_hbase_exceptions(*args) + yield + rescue => cause + # let individual command handle exceptions first + cause = cause.getCause if cause.is_a? java.io.UncheckedIOException + handle_exceptions(cause, *args) if respond_to?(:handle_exceptions) + # Global HBase exception handling below if not handled by respective command above + if cause.is_a?(org.apache.hadoop.hbase.TableNotFoundException) + strs = cause.to_s.split(' ') + raise "Unknown table #{strs[0]}!" if strs.size == 1 + raise "Unknown table #{args.first}!" + end + if cause.is_a?(org.apache.hadoop.hbase.TableNotEnabledException) + raise "Table #{args.first} is disabled!" + end + if cause.is_a?(org.apache.hadoop.hbase.TableNotDisabledException) + raise "Table #{cause.message} should be disabled!" + end + if cause.is_a?(org.apache.hadoop.hbase.UnknownRegionException) + raise cause.message + end + if cause.is_a?(org.apache.hadoop.hbase.exceptions.MergeRegionException) + strs = cause.message.split("\n") + raise(strs[0]).to_s unless strs.empty? + end + if cause.is_a?(org.apache.hadoop.hbase.NamespaceNotFoundException) + s = /.*NamespaceNotFoundException: (?[^\n]+).*/.match(cause.message) + raise "Unknown namespace #{s['namespace']}!" + end + if cause.is_a?(org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException) + raise "Unknown snapshot #{args.first}!" + end + if cause.is_a?(org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException) + exceptions = cause.getCauses + exceptions.each do |exception| + if exception.is_a?(org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException) + valid_cols = table(args.first).get_all_columns.map { |c| c + '*' } + raise "Unknown column family! Valid column names: #{valid_cols.join(', ')}" + end + end + end + if cause.is_a?(org.apache.hadoop.hbase.TableExistsException) + strs = cause.to_s.split(' ') + raise "Table already exists: #{strs[0]}!" if strs.size == 1 + raise "Table already exists: #{args.first}!" + end + # To be safe, here only AccessDeniedException is considered. In future + # we might support more in more generic approach when possible. + if cause.is_a?(org.apache.hadoop.hbase.security.AccessDeniedException) + str = java.lang.String.new(cause.to_s) + # Error message is merged with stack trace, reference StringUtils.stringifyException + # This is to parse and get the error message from the whole. + strs = str.split("\n") + raise (strs[0]).to_s unless strs.empty? + end + if cause.is_a?(org.apache.hadoop.hbase.quotas.SpaceLimitingException) + strs = cause.message.split("\n") + raise(strs[0]).to_s unless strs.empty? + end + if cause.is_a?(org.apache.hadoop.hbase.client.RetriesExhaustedException) + str = cause.cause.to_s + regex = /.*RpcThrottlingException: (?[^\n]+).*/ + error = regex.match(str) + raise error[:message].capitalize unless error.nil? + end + if cause.is_a?(org.apache.hadoop.hbase.DoNotRetryIOException) + regex = /.*UnsupportedOperationException: quota support disabled.*/ + error = regex.match(cause.message) + error_msg = 'Quota Support disabled. Please enable in configuration.' + raise error_msg unless error.nil? + end + + # Throw the other exception which hasn't been handled above + raise cause + end + # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity + # rubocop:enable Metrics/MethodLength, Metrics/PerceivedComplexity + end + # rubocop:enable Metrics/ClassLength + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/add_labels.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/add_labels.rb new file mode 100644 index 0000000000..b021b92785 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/add_labels.rb @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class AddLabels < Command + def help + <<-EOF +Add a set of visibility labels. +Syntax : add_labels [label1, label2] + +For example: + + hbase> add_labels ['SECRET','PRIVATE'] +EOF + end + + def command(*args) + visibility_labels_admin.add_labels(args) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/add_peer.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/add_peer.rb new file mode 100644 index 0000000000..bfa50a62bf --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/add_peer.rb @@ -0,0 +1,89 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class AddPeer < Command + def help + <<-EOF +A peer can either be another HBase cluster or a custom replication endpoint. In either case an id +must be specified to identify the peer. + +For a HBase cluster peer, a cluster key must be provided and is composed like this: +hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent +This gives a full path for HBase to connect to another HBase cluster. +An optional parameter for state identifies the replication peer's state is enabled or disabled. +And the default state is enabled. +An optional parameter for namespaces identifies which namespace's tables will be replicated +to the peer cluster. +An optional parameter for table column families identifies which tables and/or column families +will be replicated to the peer cluster. +An optional parameter for serial flag identifies whether or not the replication peer is a serial +replication peer. The default serial flag is false. + +Notice: Set a namespace in the peer config means that all tables in this namespace +will be replicated to the peer cluster. So if you already have set a namespace in peer config, +then you can't set this namespace's tables in the peer config again. + +Examples: + + hbase> add_peer '1', CLUSTER_KEY => "server1.cie.com:2181:/hbase" + hbase> add_peer '1', CLUSTER_KEY => "server1.cie.com:2181:/hbase", STATE => "ENABLED" + hbase> add_peer '1', CLUSTER_KEY => "server1.cie.com:2181:/hbase", STATE => "DISABLED" + hbase> add_peer '2', CLUSTER_KEY => "zk1,zk2,zk3:2182:/hbase-prod", + TABLE_CFS => { "table1" => [], "table2" => ["cf1"], "table3" => ["cf1", "cf2"] } + hbase> add_peer '2', CLUSTER_KEY => "zk1,zk2,zk3:2182:/hbase-prod", + NAMESPACES => ["ns1", "ns2", "ns3"] + hbase> add_peer '2', CLUSTER_KEY => "zk1,zk2,zk3:2182:/hbase-prod", + NAMESPACES => ["ns1", "ns2"], TABLE_CFS => { "ns3:table1" => [], "ns3:table2" => ["cf1"] } + hbase> add_peer '3', CLUSTER_KEY => "zk1,zk2,zk3:2182:/hbase-prod", + NAMESPACES => ["ns1", "ns2", "ns3"], SERIAL => true + +For a custom replication endpoint, the ENDPOINT_CLASSNAME can be provided. Two optional arguments +are DATA and CONFIG which can be specified to set different either the peer_data or configuration +for the custom replication endpoint. Table column families is optional and can be specified with +the key TABLE_CFS. + + hbase> add_peer '6', ENDPOINT_CLASSNAME => 'org.apache.hadoop.hbase.MyReplicationEndpoint' + hbase> add_peer '7', ENDPOINT_CLASSNAME => 'org.apache.hadoop.hbase.MyReplicationEndpoint', + DATA => { "key1" => 1 } + hbase> add_peer '8', ENDPOINT_CLASSNAME => 'org.apache.hadoop.hbase.MyReplicationEndpoint', + CONFIG => { "config1" => "value1", "config2" => "value2" } + hbase> add_peer '9', ENDPOINT_CLASSNAME => 'org.apache.hadoop.hbase.MyReplicationEndpoint', + DATA => { "key1" => 1 }, CONFIG => { "config1" => "value1", "config2" => "value2" }, + hbase> add_peer '10', ENDPOINT_CLASSNAME => 'org.apache.hadoop.hbase.MyReplicationEndpoint', + TABLE_CFS => { "table1" => [], "ns2:table2" => ["cf1"], "ns3:table3" => ["cf1", "cf2"] } + hbase> add_peer '11', ENDPOINT_CLASSNAME => 'org.apache.hadoop.hbase.MyReplicationEndpoint', + DATA => { "key1" => 1 }, CONFIG => { "config1" => "value1", "config2" => "value2" }, + TABLE_CFS => { "table1" => [], "ns2:table2" => ["cf1"], "ns3:table3" => ["cf1", "cf2"] } + hbase> add_peer '12', ENDPOINT_CLASSNAME => 'org.apache.hadoop.hbase.MyReplicationEndpoint', + CLUSTER_KEY => "server2.cie.com:2181:/hbase" + +Note: Either CLUSTER_KEY or ENDPOINT_CLASSNAME must be specified. If ENDPOINT_CLASSNAME is specified, CLUSTER_KEY is +optional and should only be specified if a particular custom endpoint requires it. + +EOF + end + + def command(id, args = {}, peer_tableCFs = nil) + replication_admin.add_peer(id, args, peer_tableCFs) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/add_rsgroup.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/add_rsgroup.rb new file mode 100644 index 0000000000..ab32d05691 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/add_rsgroup.rb @@ -0,0 +1,40 @@ +# +# Copyright The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class AddRsgroup < Command + def help + <<-EOF +Create a new RegionServer group. + +Example: + + hbase> add_rsgroup 'my_group' + +EOF + end + + def command(group_name) + rsgroup_admin.add_rs_group(group_name) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/alter.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/alter.rb new file mode 100644 index 0000000000..9d95bffbdd --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/alter.rb @@ -0,0 +1,119 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Alter < Command + def help + <<-EOF +Alter a table. Tables can be altered without disabling them first. +Altering enabled tables has caused problems +in the past, so use caution and test it before using in production. + +You can use the alter command to add, +modify or delete column families or change table configuration options. +Column families work in a similar way as the 'create' command. The column family +specification can either be a name string, or a dictionary with the NAME attribute. +Dictionaries are described in the output of the 'help' command, with no arguments. + +For example, to change or add the 'f1' column family in table 't1' from +current value to keep a maximum of 5 cell VERSIONS, do: + + hbase> alter 't1', NAME => 'f1', VERSIONS => 5 + +You can operate on several column families: + + hbase> alter 't1', 'f1', {NAME => 'f2', IN_MEMORY => true}, {NAME => 'f3', VERSIONS => 5} + +To delete the 'f1' column family in table 'ns1:t1', use one of: + + hbase> alter 'ns1:t1', NAME => 'f1', METHOD => 'delete' + hbase> alter 'ns1:t1', 'delete' => 'f1' + +You can also change table-scope attributes like MAX_FILESIZE, READONLY, +MEMSTORE_FLUSHSIZE, NORMALIZATION_ENABLED, NORMALIZER_TARGET_REGION_COUNT, +NORMALIZER_TARGET_REGION_SIZE_MB, DURABILITY, etc. These can be put at the end; +for example, to change the max size of a region to 128MB, do: + + hbase> alter 't1', MAX_FILESIZE => '134217728' + +You can add a table coprocessor by setting a table coprocessor attribute: + + hbase> alter 't1', + 'coprocessor'=>'hdfs:///foo.jar|com.foo.FooRegionObserver|1001|arg1=1,arg2=2' + +Since you can have multiple coprocessors configured for a table, a +sequence number will be automatically appended to the attribute name +to uniquely identify it. + +The coprocessor attribute must match the pattern below in order for +the framework to understand how to load the coprocessor classes: + + [coprocessor jar file location] | class name | [priority] | [arguments] + +You can also set configuration settings specific to this table or column family: + + hbase> alter 't1', CONFIGURATION => {'hbase.hregion.scan.loadColumnFamiliesOnDemand' => 'true'} + hbase> alter 't1', {NAME => 'f2', CONFIGURATION => {'hbase.hstore.blockingStoreFiles' => '10'}} + +You can also unset configuration settings specific to this table: + + hbase> alter 't1', METHOD => 'table_conf_unset', NAME => 'hbase.hregion.majorcompaction' + +You can also remove a table-scope attribute: + + hbase> alter 't1', METHOD => 'table_att_unset', NAME => 'MAX_FILESIZE' + + hbase> alter 't1', METHOD => 'table_att_unset', NAME => 'coprocessor$1' + +Other than removing coprocessor from the table-scope attribute via 'table_att_unset', you can also +use 'table_remove_coprocessor' by specifying the class name: + + hbase> alter 't1', METHOD => 'table_remove_coprocessor', CLASSNAME => + 'org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver' + +You can also remove multiple coprocessors at once: + + hbase> alter 't1', METHOD => 'table_remove_coprocessor', CLASSNAME => + ['org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver', + 'org.apache.hadoop.hbase.coprocessor.Export'] + +You can also set REGION_REPLICATION: + + hbase> alter 't1', {REGION_REPLICATION => 2} + +You can disable/enable table split and/or merge: + + hbase> alter 't1', {SPLIT_ENABLED => false} + hbase> alter 't1', {MERGE_ENABLED => false} + +There could be more than one alteration in one command: + + hbase> alter 't1', { NAME => 'f1', VERSIONS => 3 }, + { MAX_FILESIZE => '134217728' }, { METHOD => 'delete', NAME => 'f2' }, + OWNER => 'johndoe', METADATA => { 'mykey' => 'myvalue' } +EOF + end + + def command(table, *args) + admin.alter(table, true, *args) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/alter_async.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/alter_async.rb new file mode 100644 index 0000000000..824de1082b --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/alter_async.rb @@ -0,0 +1,63 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class AlterAsync < Command + def help + <<-EOF +Alter column family schema, does not wait for all regions to receive the +schema changes. Pass table name and a dictionary specifying new column +family schema. Dictionaries are described on the main help command output. +Dictionary must include name of column family to alter. For example, + +To change or add the 'f1' column family in table 't1' from defaults +to instead keep a maximum of 5 cell VERSIONS, do: + + hbase> alter_async 't1', NAME => 'f1', VERSIONS => 5 + +To delete the 'f1' column family in table 'ns1:t1', do: + + hbase> alter_async 'ns1:t1', NAME => 'f1', METHOD => 'delete' + +or a shorter version: + + hbase> alter_async 'ns1:t1', 'delete' => 'f1' + +You can also change table-scope attributes like MAX_FILESIZE, +MEMSTORE_FLUSHSIZE, and READONLY. + +For example, to change the max size of a family to 128MB, do: + + hbase> alter 't1', METHOD => 'table_att', MAX_FILESIZE => '134217728' + +There could be more than one alteration in one command: + + hbase> alter 't1', {NAME => 'f1'}, {NAME => 'f2', METHOD => 'delete'} + +To check if all the regions have been updated, use alter_status +EOF + end + + def command(table, *args) + admin.alter(table, false, *args) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/alter_namespace.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/alter_namespace.rb new file mode 100644 index 0000000000..602ac6fb38 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/alter_namespace.rb @@ -0,0 +1,42 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class AlterNamespace < Command + def help + <<-EOF +Alter namespace properties. + +To add/modify a property: + + hbase> alter_namespace 'ns1', {METHOD => 'set', 'PROPERTY_NAME' => 'PROPERTY_VALUE'} + +To delete a property: + + hbase> alter_namespace 'ns1', {METHOD => 'unset', NAME=>'PROPERTY_NAME'} +EOF + end + + def command(namespace, *args) + admin.alter_namespace(namespace, *args) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/alter_rsgroup_config.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/alter_rsgroup_config.rb new file mode 100644 index 0000000000..a8489b873c --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/alter_rsgroup_config.rb @@ -0,0 +1,38 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class AlterRsgroupConfig < Command + def help + <<-EOF +Alter RSGroup configuration. +Example: + hbase> alter_rsgroup_config 'grp1', {METHOD => 'set', 'PROPERTY_NAME' => 'PROPERTY_VALUE'} +To delete a property: + hbase> alter_rsgroup_config 'grp1', {METHOD => 'unset', NAME=>'PROPERTY_NAME'} +EOF + end + + def command(group, args) + rsgroup_admin.alter_rsgroup_config(group, args) + end + end + end +end \ No newline at end of file diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/alter_status.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/alter_status.rb new file mode 100644 index 0000000000..828482585c --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/alter_status.rb @@ -0,0 +1,39 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class AlterStatus < Command + def help + <<-EOF +Get the status of the alter command. Indicates the number of regions of the +table that have received the updated schema +Pass table name. + +hbase> alter_status 't1' +hbase> alter_status 'ns1:t1' +EOF + end + + def command(table) + admin.alter_status(table) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/append.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/append.rb new file mode 100644 index 0000000000..b469f0bda6 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/append.rb @@ -0,0 +1,54 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Append < Command + def help + <<-EOF +Appends a cell 'value' at specified table/row/column coordinates. + + hbase> append 't1', 'r1', 'c1', 'value', ATTRIBUTES=>{'mykey'=>'myvalue'} + hbase> append 't1', 'r1', 'c1', 'value', {VISIBILITY=>'PRIVATE|SECRET'} + +The same commands also can be run on a table reference. Suppose you had a reference +t to table 't1', the corresponding command would be: + + hbase> t.append 'r1', 'c1', 'value', ATTRIBUTES=>{'mykey'=>'myvalue'} + hbase> t.append 'r1', 'c1', 'value', {VISIBILITY=>'PRIVATE|SECRET'} +EOF + end + + def command(table_name, row, column, value, args = {}) + table = table(table_name) + @start_time = Time.now + append(table, row, column, value, args) + end + + def append(table, row, column, value, args = {}) + if current_value = table._append_internal(row, column, value, args) + puts "CURRENT VALUE = #{current_value}" + end + end + end + end +end + +# add append command to Table +::Hbase::Table.add_shell_command('append') diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/append_peer_exclude_namespaces.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/append_peer_exclude_namespaces.rb new file mode 100644 index 0000000000..4f500c8dce --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/append_peer_exclude_namespaces.rb @@ -0,0 +1,47 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class AppendPeerExcludeNamespaces < Command + def help + <<-EOF +Append the namespaces which not replicated for the specified peer. + +Note: + 1. The replicate_all flag need to be true when append exclude namespaces. + 2. Append a exclude namespace in the peer config means that all tables in this + namespace will not be replicated to the peer cluster. If peer config + already has a exclude table, then not allow append this table's namespace + as a exclude namespace. + +Examples: + + # append ns1,ns2 to be not replicable for peer '2'. + hbase> append_peer_exclude_namespaces '2', ["ns1", "ns2"] + + EOF + end + + def command(id, namespaces) + replication_admin.append_peer_exclude_namespaces(id, namespaces) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/append_peer_exclude_tableCFs.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/append_peer_exclude_tableCFs.rb new file mode 100644 index 0000000000..c229c9425c --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/append_peer_exclude_tableCFs.rb @@ -0,0 +1,42 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class AppendPeerExcludeTableCFs < Command + def help + <<-EOF +Append table-cfs config to the specified peer' exclude table-cfs to make them non-replicable +Examples: + + # append tables / table-cfs to peers' exclude table-cfs + hbase> append_peer_exclude_tableCFs '2', { "table1" => [], "ns2:table2" => ["cfA", "cfB"]} + EOF + end + + def command(id, table_cfs) + replication_admin.append_peer_exclude_tableCFs(id, table_cfs) + end + + def command_name + 'append_peer_exclude_tableCFs' + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/append_peer_namespaces.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/append_peer_namespaces.rb new file mode 100644 index 0000000000..59657d8f3c --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/append_peer_namespaces.rb @@ -0,0 +1,44 @@ +# +# Copyright The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class AppendPeerNamespaces < Command + def help + <<~EOF + Append some namespaces to be replicable for the specified peer. + + Set a namespace in the peer config means that all tables in this + namespace (with replication_scope != 0 ) will be replicated. + + Examples: + + # append ns1,ns2 to be replicable for peer '2'. + hbase> append_peer_namespaces '2', ["ns1", "ns2"] + + EOF + end + + def command(id, namespaces) + replication_admin.add_peer_namespaces(id, namespaces) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/append_peer_tableCFs.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/append_peer_tableCFs.rb new file mode 100644 index 0000000000..16b1288329 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/append_peer_tableCFs.rb @@ -0,0 +1,43 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class AppendPeerTableCFs < Command + def help + <<-EOF +Append a replicable table-cf config for the specified peer +Examples: + + # append a table / table-cf to be replicable for a peer + hbase> append_peer_tableCFs '2', { "ns1:table4" => ["cfA", "cfB"]} + +EOF + end + + def command(id, table_cfs) + replication_admin.append_peer_tableCFs(id, table_cfs) + end + + def command_name + 'append_peer_tableCFs' + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/assign.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/assign.rb new file mode 100644 index 0000000000..769ed7801f --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/assign.rb @@ -0,0 +1,41 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Assign < Command + def help + <<-EOF +Assign a region. It could be executed only when region in expected state(CLOSED, OFFLINE). +In addition, you can use "assigns" command available on HBCK2 tool to skip the state check. +(For more info on HBCK2: https://github.com/apache/hbase-operator-tools/blob/master/hbase-hbck2/README.md) +Use with caution. For experts only. +Examples: + + hbase> assign 'REGIONNAME' + hbase> assign 'ENCODED_REGIONNAME' +EOF + end + + def command(region_name) + admin.assign(region_name) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/balance_rsgroup.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/balance_rsgroup.rb new file mode 100644 index 0000000000..aff9fa7ce9 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/balance_rsgroup.rb @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class BalanceRsgroup < Command + def help + <<-EOF +Balance a RegionServer group + +Parameter can be "force" or "dry_run": + - "dry_run" will run the balancer to generate a plan, but will not actually execute that plan. + This is useful for testing out new balance configurations. See the active HMaster logs for the results of the dry_run. + - "ignore_rit" tells master whether we should force the balancer to run even if there is region in transition. + WARNING: For experts only. Forcing a balance may do more damage than repair when assignment is confused + +Example: + + hbase> balance_rsgroup 'my_group' + hbase> balance_rsgroup 'my_group', 'ignore_rit' + hbase> balance_rsgroup 'my_group', 'dry_run' + hbase> balance_rsgroup 'my_group', 'dry_run', 'ignore_rit' + +EOF + end + + def command(group_name, *args) + # Returns true if balancer was run, otherwise false. + resp = rsgroup_admin.balance_rs_group(group_name, args) + if resp.isBalancerRan + formatter.row(["Balancer ran"]) + formatter.row(["Moves calculated: #{resp.getMovesCalculated}, moves executed: #{resp.getMovesExecuted}"]) + else + formatter.row(["Balancer did not run. See logs for details."]) + end + resp.isBalancerRan + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/balance_switch.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/balance_switch.rb new file mode 100644 index 0000000000..16c272bd59 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/balance_switch.rb @@ -0,0 +1,40 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class BalanceSwitch < Command + def help + <<-EOF +Enable/Disable balancer. Returns previous balancer state. +Examples: + + hbase> balance_switch true + hbase> balance_switch false +EOF + end + + def command(enableDisable) + prev_state = !!admin.balance_switch(enableDisable) + formatter.row(["Previous balancer state : #{prev_state}"]) + prev_state + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/balancer.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/balancer.rb new file mode 100644 index 0000000000..d5304e0007 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/balancer.rb @@ -0,0 +1,54 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Balancer < Command + def help + <<-EOF +Trigger the cluster balancer. Returns true if balancer ran, otherwise false (Will not run if regions in transition). + +Parameter can be "force" or "dry_run": + - "dry_run" will run the balancer to generate a plan, but will not actually execute that plan. + This is useful for testing out new balance configurations. See the active HMaster logs for the results of the dry_run. + - "ignore_rit" tells master whether we should force the balancer to run even if there is region in transition. + WARNING: For experts only. Forcing a balance may do more damage than repair when assignment is confused + +Examples: + + hbase> balancer + hbase> balancer "ignore_rit" + hbase> balancer "dry_run" + hbase> balancer "dry_run", "ignore_rit" +EOF + end + + def command(*args) + resp = admin.balancer(args) + if resp.isBalancerRan + formatter.row(["Balancer ran"]) + formatter.row(["Moves calculated: #{resp.getMovesCalculated}, moves executed: #{resp.getMovesExecuted}"]) + else + formatter.row(["Balancer did not run. See logs for details."]) + end + resp.isBalancerRan + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/balancer_enabled.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/balancer_enabled.rb new file mode 100644 index 0000000000..a2884c4c5f --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/balancer_enabled.rb @@ -0,0 +1,39 @@ +#!/usr/bin/env hbase-jruby +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Prints the current balancer status + +module Shell + module Commands + class BalancerEnabled < Command + def help + <<-EOF +Query the balancer's state. +Examples: + + hbase> balancer_enabled +EOF + end + + def command + state = admin.balancer_enabled? + formatter.row([state.to_s]) + state + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/catalogjanitor_enabled.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/catalogjanitor_enabled.rb new file mode 100644 index 0000000000..5ff3898381 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/catalogjanitor_enabled.rb @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class CatalogjanitorEnabled < Command + def help + <<-EOF +Query for the CatalogJanitor state (enabled/disabled?) +Examples: + + hbase> catalogjanitor_enabled +EOF + end + + def command + current_state = !!admin.catalogjanitor_enabled + formatter.row([current_state.to_s]) + current_state + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/catalogjanitor_run.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/catalogjanitor_run.rb new file mode 100644 index 0000000000..32f9f6a620 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/catalogjanitor_run.rb @@ -0,0 +1,36 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class CatalogjanitorRun < Command + def help + <<-EOF +Catalog janitor command to run the (garbage collection) scan from command line. + + hbase> catalogjanitor_run + +EOF + end + + def command + admin.catalogjanitor_run + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/catalogjanitor_switch.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/catalogjanitor_switch.rb new file mode 100644 index 0000000000..76718d2704 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/catalogjanitor_switch.rb @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class CatalogjanitorSwitch < Command + def help + <<-EOF +Enable/Disable CatalogJanitor. Returns previous CatalogJanitor state. +Examples: + + hbase> catalogjanitor_switch true + hbase> catalogjanitor_switch false +EOF + end + + def command(enableDisable) + previous_state = !!admin.catalogjanitor_switch(enableDisable) + formatter.row([previous_state.to_s]) + previous_state + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/change_sft.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/change_sft.rb new file mode 100644 index 0000000000..eb96f426a6 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/change_sft.rb @@ -0,0 +1,50 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ChangeSft < Command + def help + <<-EOF +Change table's or table column family's sft. Examples: + + hbase> change_sft 't1','FILE' + hbase> change_sft 't2','cf1','FILE' +EOF + end + + def command(*args) + arg_length = args.length + if arg_length == 2 + tableName = TableName.valueOf(args[0]) + sft = args[1] + admin.modify_table_sft(tableName, sft) + elsif arg_length == 3 + tableName = TableName.valueOf(args[0]) + family = args[1] + family_bytes = family.to_java_bytes + sft = args[2] + admin.modify_table_family_sft(tableName, family_bytes, sft) + else + raise(ArgumentError, 'Argument length should be two or three.') + end + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/change_sft_all.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/change_sft_all.rb new file mode 100644 index 0000000000..6e348195ba --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/change_sft_all.rb @@ -0,0 +1,58 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ChangeSftAll < Command + def help + <<-EOF +Change all of the tables's sft matching the given regex: + + hbase> change_sft_all 't.*','FILE' + hbase> change_sft_all 'ns:.*','FILE' + hbase> change_sft_all 'ns:t.*','FILE' +EOF + end + + def command(*args) + arg_length = args.length + if arg_length == 2 + tableRegex = args[0] + tableList = admin.list(tableRegex) + count = tableList.size + sft = args[1] + tableList.each do |table| + formatter.row([table]) + end + puts "\nChange the above #{count} tables's sft (y/n)?" unless count == 0 + answer = 'n' + answer = gets.chomp unless count == 0 + puts "No tables matched the regex #{tableRegex}" if count == 0 + return unless answer =~ /y.*/i + tableList.each do |table| + tableName = TableName.valueOf(table) + admin.modify_table_sft(tableName, sft) + end + else + raise(ArgumentError, 'Argument length should be two.') + end + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/cleaner_chore_enabled.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/cleaner_chore_enabled.rb new file mode 100644 index 0000000000..b7038b4959 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/cleaner_chore_enabled.rb @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class CleanerChoreEnabled < Command + def help + <<-EOF +Query for the Cleaner chore state (enabled/disabled?). +Examples: + + hbase> cleaner_chore_enabled +EOF + end + + def command + current_state = !!admin.cleaner_chore_enabled + formatter.row([current_state.to_s]) + current_state + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/cleaner_chore_run.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/cleaner_chore_run.rb new file mode 100644 index 0000000000..210f3889bb --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/cleaner_chore_run.rb @@ -0,0 +1,36 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class CleanerChoreRun < Command + def help + <<-EOF +Cleaner chore command for garbage collection of HFiles and WAL files. + + hbase> cleaner_chore_run + +EOF + end + + def command + admin.cleaner_chore_run + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/cleaner_chore_switch.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/cleaner_chore_switch.rb new file mode 100644 index 0000000000..62a46721b4 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/cleaner_chore_switch.rb @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class CleanerChoreSwitch < Command + def help + <<-EOF +Enable/Disable Cleaner chore. Returns previous Cleaner chore state. +Examples: + + hbase> cleaner_chore_switch true + hbase> cleaner_chore_switch false +EOF + end + + def command(enableDisable) + previous_state = !!admin.cleaner_chore_switch(enableDisable) + formatter.row([previous_state.to_s]) + previous_state + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/clear_auths.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/clear_auths.rb new file mode 100644 index 0000000000..9cd0a2b328 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/clear_auths.rb @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ClearAuths < Command + def help + <<-EOF +Clear visibility labels from a user or group +Syntax : clear_auths 'user',[label1, label2] + +For example: + + hbase> clear_auths 'user1', ['SECRET','PRIVATE'] + hbase> clear_auths '@group1', ['SECRET','PRIVATE'] +EOF + end + + def command(user, *args) + visibility_labels_admin.clear_auths(user, args) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/clear_block_cache.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/clear_block_cache.rb new file mode 100644 index 0000000000..6ad3d708e7 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/clear_block_cache.rb @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ClearBlockCache < Command + def help + <<-EOF +Clear all the blocks corresponding to this table from BlockCache. For expert-admins. +Calling this API will drop all the cached blocks specific to a table from BlockCache. +This can significantly impact the query performance as the subsequent queries will +have to retrieve the blocks from underlying filesystem. +For example: + + hbase> clear_block_cache 'TABLENAME' +EOF + end + + def command(table_name) + formatter.row([admin.clear_block_cache(table_name)]) + nil + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/clear_compaction_queues.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/clear_compaction_queues.rb new file mode 100644 index 0000000000..f5f94a975f --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/clear_compaction_queues.rb @@ -0,0 +1,41 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ClearCompactionQueues < Command + def help + <<~EOF + Clear compacting queues on a regionserver. + The queue_name contains short and long. + short is shortCompactions's queue,long is longCompactions's queue. + + Examples: + hbase> clear_compaction_queues 'host187.example.com,60020' + hbase> clear_compaction_queues 'host187.example.com,60020','long' + hbase> clear_compaction_queues 'host187.example.com,60020', ['long','short'] + EOF + end + + def command(server_name, queue_name = nil) + admin.clear_compaction_queues(server_name, queue_name) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/clear_deadservers.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/clear_deadservers.rb new file mode 100644 index 0000000000..e7edeb685e --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/clear_deadservers.rb @@ -0,0 +1,60 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ClearDeadservers < Command + def help + <<~EOF + Clear the dead region servers that are never used. Returns an array containing any + deadservers that could not be cleared. + + Examples: + Clear all dead region servers: + hbase> clear_deadservers + Clear the specified dead region servers: + hbase> clear_deadservers 'host187.example.com,60020,1289493121758' + or + hbase> clear_deadservers 'host187.example.com,60020,1289493121758', + 'host188.example.com,60020,1289493121758' + EOF + end + + # rubocop:disable Metrics/AbcSize + def command(*dead_servers) + servers = admin.clear_deadservers(dead_servers) + if servers.size <= 0 + formatter.row(['true']) + [] + else + formatter.row(['Some dead server clear failed']) + formatter.row(['SERVERNAME']) + server_names = servers.map { |server| server.toString } + server_names.each do |server| + formatter.row([server]) + end + formatter.footer(servers.size) + server_names + end + end + # rubocop:enable Metrics/AbcSize + # rubocop:enable Metrics/MethodLength + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/clear_slowlog_responses.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/clear_slowlog_responses.rb new file mode 100644 index 0000000000..ea96de3c26 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/clear_slowlog_responses.rb @@ -0,0 +1,47 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Clear slowlog responses maintained in memory by RegionServers + +module Shell + module Commands + # Clear slowlog responses + class ClearSlowlogResponses < Command + def help + <<-EOF +Clears SlowLog Responses maintained by each or specific RegionServers. +Specify array of server names for specific RS. A server name is +the host, port plus startcode of a RegionServer. +e.g.: host187.example.com,60020,1289493121758 (find servername in +master ui or when you do detailed status in shell) + +Examples: + + hbase> clear_slowlog_responses => clears slowlog responses from all RS + hbase> clear_slowlog_responses ['SERVER_NAME1', 'SERVER_NAME2'] => clears slowlog responses from SERVER_NAME1, + SERVER_NAME2 + + + EOF + end + + def command(server_names = nil) + admin.clear_slowlog_responses(server_names) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/clone_snapshot.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/clone_snapshot.rb new file mode 100644 index 0000000000..3edd16d326 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/clone_snapshot.rb @@ -0,0 +1,61 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class CloneSnapshot < Command + def help + <<-EOF +Create a new table by cloning the snapshot content. +There're no copies of data involved. +And writing on the newly created table will not influence the snapshot data. + +Examples: + hbase> clone_snapshot 'snapshotName', 'tableName' + hbase> clone_snapshot 'snapshotName', 'namespace:tableName' + +Following command will restore all acl from origin snapshot table into the +newly created table. + + hbase> clone_snapshot 'snapshotName', 'namespace:tableName', {RESTORE_ACL=>true} + +StoreFileTracker implementation used after restore can be set by the following command. + hbase> clone_snapshot 'snapshotName', 'namespace:tableName', {CLONE_SFT=>'FILE'} +EOF + end + + def command(snapshot_name, table, args = {}) + raise(ArgumentError, 'Arguments should be a Hash') unless args.is_a?(Hash) + restore_acl = args.delete(::HBaseConstants::RESTORE_ACL) || false + clone_sft = args.delete(::HBaseConstants::CLONE_SFT) || nil + admin.clone_snapshot(snapshot_name, table, restore_acl, clone_sft) + end + + def handle_exceptions(cause, *args) + if cause.is_a?(org.apache.hadoop.hbase.TableExistsException) + tableName = args[1] + raise "Table already exists: #{tableName}!" + end + if cause.is_a?(org.apache.hadoop.hbase.NamespaceNotFoundException) + namespace_name = args[1].split(':')[0] + raise "Unknown namespace: #{namespace_name}!" + end + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/clone_table_schema.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/clone_table_schema.rb new file mode 100644 index 0000000000..12dc77794c --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/clone_table_schema.rb @@ -0,0 +1,42 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + # create a new table by cloning the existent table schema. + class CloneTableSchema < Command + def help + <<~HELP + Create a new table by cloning the existent table schema. + There're no copies of data involved. + Just copy the table descriptor and split keys. + + Passing 'false' as the optional third parameter will + not preserve split keys. + Examples: + hbase> clone_table_schema 'table_name', 'new_table_name' + hbase> clone_table_schema 'table_name', 'new_table_name', false + HELP + end + + def command(table_name, new_table_name, preserve_splits = true) + admin.clone_table_schema(table_name, new_table_name, preserve_splits) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/close_region.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/close_region.rb new file mode 100644 index 0000000000..96450a7a23 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/close_region.rb @@ -0,0 +1,36 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class CloseRegion < Command + def help + <<-EOF +--------------------------------------------- +DEPRECATED!!! Use 'unassign' command instead. +--------------------------------------------- +EOF + end + + def command(region_name, server = nil) + puts "DEPRECATED!!! Use 'unassign' command instead." + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/compact.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/compact.rb new file mode 100644 index 0000000000..16edeb37fe --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/compact.rb @@ -0,0 +1,51 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Compact < Command + def help + <<~EOF + Compact all regions in passed table or pass a region row + to compact an individual region. You can also compact a single column + family within a region. + You can also set compact type, "NORMAL" or "MOB", and default is "NORMAL" + Examples: + Compact all regions in a table: + hbase> compact 'ns1:t1' + hbase> compact 't1' + Compact an entire region: + hbase> compact 'r1' + Compact only a column family within a region: + hbase> compact 'r1', 'c1' + Compact a column family within a table: + hbase> compact 't1', 'c1' + Compact table with type "MOB" + hbase> compact 't1', nil, 'MOB' + Compact a column family using "MOB" type within a table + hbase> compact 't1', 'c1', 'MOB' + EOF + end + + def command(table_or_region_name, family = nil, type = 'NORMAL') + admin.compact(table_or_region_name, family, type) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/compact_rs.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/compact_rs.rb new file mode 100644 index 0000000000..2dc62c166c --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/compact_rs.rb @@ -0,0 +1,41 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class CompactRs < Command + def help + <<~EOF + Compact all regions on passed regionserver. + Examples: + Compact all regions on a regionserver: + hbase> compact_rs 'host187.example.com,60020' + or + hbase> compact_rs 'host187.example.com,60020,1289493121758' + Major compact all regions on a regionserver: + hbase> compact_rs 'host187.example.com,60020,1289493121758', true + EOF + end + + def command(regionserver, major = false) + admin.compact_regionserver(regionserver, major) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/compaction_state.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/compaction_state.rb new file mode 100644 index 0000000000..4df7f07987 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/compaction_state.rb @@ -0,0 +1,38 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class CompactionState < Command + def help + <<~EOF + Gets compaction status (MAJOR, MAJOR_AND_MINOR, MINOR, NONE) for a table: + hbase> compaction_state 'ns1:t1' + hbase> compaction_state 't1' + EOF + end + + def command(table_name) + rv = admin.getCompactionState(table_name) + formatter.row([rv]) + rv + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/compaction_switch.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/compaction_switch.rb new file mode 100644 index 0000000000..d26323312f --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/compaction_switch.rb @@ -0,0 +1,52 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + # Switch compaction for a region server + class CompactionSwitch < Command + def help + <<~EOF + Turn the compaction on or off on regionservers. Disabling compactions will also interrupt + any currently ongoing compactions. This state is ephemeral. The setting will be lost on + restart of the server. Compaction can also be enabled/disabled by modifying configuration + hbase.regionserver.compaction.enabled in hbase-site.xml. + Examples: + To enable compactions on all region servers + hbase> compaction_switch true + To disable compactions on all region servers + hbase> compaction_switch false + To enable compactions on specific region servers + hbase> compaction_switch true, 'server2','server1' + To disable compactions on specific region servers + hbase> compaction_switch false, 'server2','server1' + NOTE: A server name is its host, port plus startcode. For example: + host187.example.com,60020,1289493121758 + EOF + end + + def command(enable_disable, *server) + formatter.header(%w(['SERVER' 'PREV_STATE'])) + prev_state = admin.compaction_switch(enable_disable, server) + prev_state.each { |k, v| formatter.row([k.getServerName, java.lang.String.valueOf(v)]) } + formatter.footer(prev_state.size) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/count.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/count.rb new file mode 100644 index 0000000000..7052358823 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/count.rb @@ -0,0 +1,107 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Count < Command + def help + <<-EOF +Count the number of rows in a table. Return value is the number of rows. +This operation may take a LONG time (Run '$HADOOP_HOME/bin/hadoop jar +hbase.jar rowcount' to run a counting mapreduce job). Current count is shown +every 1000 rows by default. Count interval may be optionally specified. Scan +caching is enabled on count scans by default. Default cache size is 10 rows. +If your rows are small in size, you may want to increase this +parameter. Examples: + + hbase> count 'ns1:t1' + hbase> count 't1' + hbase> count 't1', INTERVAL => 100000 + hbase> count 't1', CACHE => 1000 + hbase> count 't1', INTERVAL => 10, CACHE => 1000 + hbase> count 't1', FILTER => " + (QualifierFilter (>=, 'binary:xyz')) AND (TimestampsFilter ( 123, 456))" + hbase> count 't1', COLUMNS => ['c1', 'c2'], STARTROW => 'abc', STOPROW => 'xyz' + +The same commands also can be run on a table reference. Suppose you had a reference +t to table 't1', the corresponding commands would be: + + hbase> t.count + hbase> t.count INTERVAL => 100000 + hbase> t.count CACHE => 1000 + hbase> t.count INTERVAL => 10, CACHE => 1000 + hbase> t.count FILTER => " + (QualifierFilter (>=, 'binary:xyz')) AND (TimestampsFilter ( 123, 456))" + hbase> t.count COLUMNS => ['c1', 'c2'], STARTROW => 'abc', STOPROW => 'xyz' + +By default, this operation does not cause any new blocks to be read into +the RegionServer block cache. This is typically the desired action; however, +if you want to force all blocks for a table to be loaded into the block cache +on-demand, you can pass the 'CACHE_BLOCKS' option with a value of 'true'. A value +of 'false' is the default and will result in no blocks being cached. This +command can be used in conjunction with all other options. + +hbase> count 'ns1:t1', CACHE_BLOCKS => true +hbase> count 'ns1:t1', CACHE_BLOCKS => 'true' +hbase> count 'ns1:t1', INTERVAL => 100000, CACHE_BLOCKS => false +EOF + end + + def command(table, params = {}) + count(table(table), params) + end + + def count(table, params = {}) + # If the second parameter is an integer, then it is the old command syntax + params = { 'INTERVAL' => params } if params.is_a?(Integer) + + # Try to be nice and convert a string to a bool + if params.include?('CACHE_BLOCKS') and params['CACHE_BLOCKS'].is_a?(String) + if params['CACHE_BLOCKS'].downcase == 'true' + params['CACHE_BLOCKS'] = true + elsif params['CACHE_BLOCKS'].downcase == 'false' + params['CACHE_BLOCKS'] = false + else + raise(ArgumentError, "Expected CACHE_BLOCKS value to be a boolean or the string 'true' or 'false'") + end + end + + # Merge params with defaults + params = { + 'INTERVAL' => 1000, + 'CACHE' => 10, + 'CACHE_BLOCKS' => false + }.merge(params) + + scan = table._hash_to_scan(params) + # Call the counter method + @start_time = Time.now + formatter.header + count = table._count_internal(params['INTERVAL'].to_i, scan, params['CACHE_BLOCKS']) do |cnt, row| + formatter.row(["Current count: #{cnt}, row: #{row}"]) + end + formatter.footer(count) + count + end + end + end +end + +# Add the method table.count that calls count.count +::Hbase::Table.add_shell_command('count') diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/create.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/create.rb new file mode 100644 index 0000000000..b82b2bfc34 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/create.rb @@ -0,0 +1,76 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Create < Command + def help + <<-EOF +Creates a table. Pass a table name, and a set of column family +specifications (at least one), and, optionally, table configuration. +Column specification can be a simple string (name), or a dictionary +(dictionaries are described below in main help output), necessarily +including NAME attribute. +Examples: + +Create a table with namespace=ns1 and table qualifier=t1 + hbase> create 'ns1:t1', {NAME => 'f1', VERSIONS => 5} + +Create a table with namespace=default and table qualifier=t1 + hbase> create 't1', {NAME => 'f1'}, {NAME => 'f2'}, {NAME => 'f3'} + hbase> # The above in shorthand would be the following: + hbase> create 't1', 'f1', 'f2', 'f3' + hbase> create 't1', {NAME => 'f1', VERSIONS => 1, TTL => 2592000, BLOCKCACHE => true} + hbase> create 't1', {NAME => 'f1', CONFIGURATION => {'hbase.hstore.blockingStoreFiles' => '10'}} + hbase> create 't1', {NAME => 'f1', IS_MOB => true, MOB_THRESHOLD => 1000000, MOB_COMPACT_PARTITION_POLICY => 'weekly'} + +Table configuration options can be put at the end. +Examples: + + hbase> create 'ns1:t1', 'f1', SPLITS => ['10', '20', '30', '40'] + hbase> create 't1', 'f1', SPLITS => ['10', '20', '30', '40'] + hbase> create 't1', 'f1', SPLITS_FILE => 'splits.txt', OWNER => 'johndoe' + hbase> create 't1', {NAME => 'f1', VERSIONS => 5}, METADATA => { 'mykey' => 'myvalue' } + hbase> # Optionally pre-split the table into NUMREGIONS, using + hbase> # SPLITALGO ("HexStringSplit", "UniformSplit" or classname) + hbase> create 't1', 'f1', {NUMREGIONS => 15, SPLITALGO => 'HexStringSplit'} + hbase> create 't1', 'f1', {NUMREGIONS => 15, SPLITALGO => 'HexStringSplit', REGION_REPLICATION => 2, CONFIGURATION => {'hbase.hregion.scan.loadColumnFamiliesOnDemand' => 'true'}} + hbase> create 't1', 'f1', {SPLIT_ENABLED => false, MERGE_ENABLED => false} + hbase> create 't1', {NAME => 'f1', DFS_REPLICATION => 1} + +You can also keep around a reference to the created table: + + hbase> t1 = create 't1', 'f1' + +Which gives you a reference to the table named 't1', on which you can then +call methods. +EOF + end + + def command(table, *args) + admin.create(table, *args) + @end_time = Time.now + puts 'Created table ' + table.to_s + + # and then return the table just created + table(table) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/create_namespace.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/create_namespace.rb new file mode 100644 index 0000000000..62a3bc21c2 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/create_namespace.rb @@ -0,0 +1,39 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class CreateNamespace < Command + def help + <<-EOF +Create namespace; pass namespace name, +and optionally a dictionary of namespace configuration. +Examples: + + hbase> create_namespace 'ns1' + hbase> create_namespace 'ns1', {'PROPERTY_NAME'=>'PROPERTY_VALUE'} +EOF + end + + def command(namespace, *args) + admin.create_namespace(namespace, *args) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/decommission_regionservers.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/decommission_regionservers.rb new file mode 100644 index 0000000000..3fcb1838e6 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/decommission_regionservers.rb @@ -0,0 +1,49 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + # Decommission a list of region servers, optionally offload corresponding regions + class DecommissionRegionservers < Command + def help + <<~EOF + Mark region server(s) as decommissioned to prevent additional regions from + getting assigned to them. + + Optionally, offload the regions on the servers by passing true. + NOTE: Region offloading is asynchronous. + + If there are multiple servers to be decommissioned, decommissioning them + at the same time can prevent wasteful region movements. + + Examples: + hbase> decommission_regionservers 'server' + hbase> decommission_regionservers 'server,port' + hbase> decommission_regionservers 'server,port,starttime' + hbase> decommission_regionservers 'server', false + hbase> decommission_regionservers ['server1','server2'], true +EOF + end + + def command(server_names, should_offload = false) + admin.decommission_regionservers(server_names, should_offload) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/delete.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/delete.rb new file mode 100644 index 0000000000..923d3498a0 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/delete.rb @@ -0,0 +1,58 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Delete < Command + def help + <<-EOF +Put a delete cell value at specified table/row/column and optionally +timestamp coordinates. Deletes must match the deleted cell's +coordinates exactly. When scanning, a delete cell suppresses older +versions. To delete a cell from 't1' at row 'r1' under column 'c1' +marked with the time 'ts1', do: + + hbase> delete 'ns1:t1', 'r1', 'c1', ts1 + hbase> delete 't1', 'r1', 'c1', ts1 + hbase> delete 't1', 'r1', 'c1', ts1, {VISIBILITY=>'PRIVATE|SECRET'} + +The same command can also be run on a table reference. Suppose you had a reference +t to table 't1', the corresponding command would be: + + hbase> t.delete 'r1', 'c1', ts1 + hbase> t.delete 'r1', 'c1', ts1, {VISIBILITY=>'PRIVATE|SECRET'} +EOF + end + + def command(table, row, column, + timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP, args = {}) + delete(table(table), row, column, timestamp, args) + end + + def delete(table, row, column, + timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP, args = {}) + @start_time = Time.now + table._delete_internal(row, column, timestamp, args, false) + end + end + end +end + +# Add the method table.delete that calls delete.delete +::Hbase::Table.add_shell_command('delete') diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/delete_all_snapshot.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/delete_all_snapshot.rb new file mode 100644 index 0000000000..52276fc9c7 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/delete_all_snapshot.rb @@ -0,0 +1,61 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class DeleteAllSnapshot < Command + def help + <<-EOF +Delete all of the snapshots matching the given regex. Examples: + + hbase> delete_all_snapshot 's.*' + +EOF + end + + def command(regex) + formatter.header(['SNAPSHOT', 'TABLE + CREATION TIME']) + list = admin.list_snapshot(regex) + count = list.size + list.each do |snapshot| + creation_time = Time.at(snapshot.getCreationTime / 1000).to_s + formatter.row([snapshot.getName, snapshot.getTable + ' (' + creation_time + ')']) + end + puts "\nDelete the above #{count} snapshots (y/n)?" unless count == 0 + answer = 'n' + answer = gets.chomp unless count == 0 + puts "No snapshots matched the regex #{regex}" if count == 0 + return unless answer =~ /y.*/i + @start_time = Time.now + admin.delete_all_snapshot(regex) + @end_time = Time.now + list = admin.list_snapshot(regex) + leftOverSnapshotCount = list.size + successfullyDeleted = count - leftOverSnapshotCount + puts "#{successfullyDeleted} snapshots successfully deleted." unless successfullyDeleted == 0 + return if leftOverSnapshotCount == 0 + puts "\nFailed to delete the below #{leftOverSnapshotCount} snapshots." + formatter.header(['SNAPSHOT', 'TABLE + CREATION TIME']) + list.each do |snapshot| + creation_time = Time.at(snapshot.getCreationTime / 1000).to_s + formatter.row([snapshot.getName, snapshot.getTable + ' (' + creation_time + ')']) + end + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/delete_snapshot.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/delete_snapshot.rb new file mode 100644 index 0000000000..85c73fb3f7 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/delete_snapshot.rb @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class DeleteSnapshot < Command + def help + <<-EOF +Delete a specified snapshot. Examples: + + hbase> delete_snapshot 'snapshotName', +EOF + end + + def command(snapshot_name) + admin.delete_snapshot(snapshot_name) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/delete_table_snapshots.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/delete_table_snapshots.rb new file mode 100644 index 0000000000..a7875f1023 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/delete_table_snapshots.rb @@ -0,0 +1,68 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class DeleteTableSnapshots < Command + def help + <<-EOF +Delete all of the snapshots matching the given table name regular expression +and snapshot name regular expression. +By default snapshot name regular expression will delete all the snapshots of the +matching table name regular expression. + +Examples: + hbase> delete_table_snapshots 'tableName' + hbase> delete_table_snapshots 'tableName.*' + hbase> delete_table_snapshots 'tableName', 'snapshotName' + hbase> delete_table_snapshots 'tableName', 'snapshotName.*' + hbase> delete_table_snapshots 'tableName.*', 'snapshotName.*' + hbase> delete_table_snapshots 'ns:tableName.*', 'snapshotName.*' + +EOF + end + + def command(tableNameregex, snapshotNameRegex = '.*') + formatter.header(['SNAPSHOT', 'TABLE + CREATION TIME']) + list = admin.list_table_snapshots(tableNameregex, snapshotNameRegex) + count = list.size + list.each do |snapshot| + creation_time = Time.at(snapshot.getCreationTime / 1000).to_s + formatter.row([snapshot.getName, snapshot.getTable + ' (' + creation_time + ')']) + end + puts "\nDelete the above #{count} snapshots (y/n)?" unless count == 0 + answer = 'n' + answer = gets.chomp unless count == 0 + puts "No snapshots matched the table name regular expression #{tableNameregex} and the snapshot name regular expression #{snapshotNameRegex}" if count == 0 + return unless answer =~ /y.*/i + + @start_time = Time.now + list.each do |deleteSnapshot| + begin + admin.delete_snapshot(deleteSnapshot.getName) + puts "Successfully deleted snapshot: #{deleteSnapshot.getName}" + puts "\n" + rescue RuntimeError + puts "Failed to delete snapshot: #{deleteSnapshot.getName}, due to below exception,\n" + $ERROR_INFO + puts "\n" + end + end + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/deleteall.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/deleteall.rb new file mode 100644 index 0000000000..56f6d5549a --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/deleteall.rb @@ -0,0 +1,72 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Deleteall < Command + def help + <<-EOF +Delete all cells in a given row; pass a table name, row, and optionally +a column and timestamp. Deleteall also support deleting a row range using a +row key prefix. Examples: + + hbase> deleteall 'ns1:t1', 'r1' + hbase> deleteall 't1', 'r1' + hbase> deleteall 't1', 'r1', 'c1' + hbase> deleteall 't1', 'r1', 'c1', ts1 + //'' means no specific column, will delete all cells in the row which timestamp is lower than + //the one specified in the command + hbase> deleteall 't1', 'r1', '', ts1 + hbase> deleteall 't1', 'r1', 'c1', ts1, {VISIBILITY=>'PRIVATE|SECRET'} + +ROWPREFIXFILTER can be used to delete row ranges + hbase> deleteall 't1', {ROWPREFIXFILTER => 'prefix'} + hbase> deleteall 't1', {ROWPREFIXFILTER => 'prefix'}, 'c1' //delete certain column family in the row ranges + hbase> deleteall 't1', {ROWPREFIXFILTER => 'prefix'}, 'c1', ts1 + hbase> deleteall 't1', {ROWPREFIXFILTER => 'prefix'}, '', ts1 + hbase> deleteall 't1', {ROWPREFIXFILTER => 'prefix'}, 'c1', ts1, {VISIBILITY=>'PRIVATE|SECRET'} + +CACHE can be used to specify how many deletes batched to be sent to server at one time, default is 100 + hbase> deleteall 't1', {ROWPREFIXFILTER => 'prefix', CACHE => 100} + + +The same commands also can be run on a table reference. Suppose you had a reference +t to table 't1', the corresponding command would be: + + hbase> t.deleteall 'r1', 'c1', ts1, {VISIBILITY=>'PRIVATE|SECRET'} + hbase> t.deleteall {ROWPREFIXFILTER => 'prefix', CACHE => 100}, 'c1', ts1, {VISIBILITY=>'PRIVATE|SECRET'} +EOF + end + + def command(table, row, column = nil, + timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP, args = {}) + deleteall(table(table), row, column, timestamp, args) + end + + def deleteall(table, row, column = nil, + timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP, args = {}) + @start_time = Time.now + table._deleteall_internal(row, column, timestamp, args, true) + end + end + end +end + +# Add the method table.deleteall that calls deleteall.deleteall +::Hbase::Table.add_shell_command('deleteall') diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/describe.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/describe.rb new file mode 100644 index 0000000000..9865924fd7 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/describe.rb @@ -0,0 +1,64 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Describe < Command + def help + <<-EOF +Describe the named table. For example: + hbase> describe 't1' + hbase> describe 'ns1:t1' + +Alternatively, you can use the abbreviated 'desc' for the same thing. + hbase> desc 't1' + hbase> desc 'ns1:t1' +EOF + end + + # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def command(table) + column_families = admin.get_column_families(table) + + formatter.header(['Table ' + table.to_s + ' is ' + (admin.enabled?(table) ? 'ENABLED' : 'DISABLED')]) + formatter.row([table.to_s + admin.get_table_attributes(table)], true) + formatter.header(['COLUMN FAMILIES DESCRIPTION']) + column_families.each do |column_family| + formatter.row([column_family.to_s], true) + puts + end + formatter.footer + if admin.exists?(::HBaseQuotasConstants::QUOTA_TABLE_NAME.to_s) + if table.to_s != 'hbase:meta' + # No QUOTAS if hbase:meta table + puts + formatter.header(%w[QUOTAS]) + count = quotas_admin.list_quotas(::HBaseConstants::TABLE => table.to_s) do |_, quota| + formatter.row([quota]) + end + formatter.footer(count) + end + else + puts 'Quota is disabled' + end + end + # rubocop:enable Metrics/AbcSize, Metrics/MethodLength + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/describe_namespace.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/describe_namespace.rb new file mode 100644 index 0000000000..a929887dda --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/describe_namespace.rb @@ -0,0 +1,50 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class DescribeNamespace < Command + def help + <<-EOF +Describe the named namespace. For example: + hbase> describe_namespace 'ns1' +EOF + end + + # rubocop:disable Metrics/AbcSize + def command(namespace) + desc = admin.describe_namespace(namespace) + + formatter.header(['DESCRIPTION'], [64]) + formatter.row([desc], true, [64]) + ns = namespace.to_s + if admin.exists?(::HBaseQuotasConstants::QUOTA_TABLE_NAME.to_s) + puts formatter.header(%w[QUOTAS]) + count = quotas_admin.list_quotas(::HBaseConstants::NAMESPACE => ns) do |_, quota| + formatter.row([quota]) + end + formatter.footer(count) + else + puts 'Quota is disabled' + end + end + # rubocop:enable Metrics/AbcSize + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/disable.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/disable.rb new file mode 100644 index 0000000000..79be0a21df --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/disable.rb @@ -0,0 +1,36 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Disable < Command + def help + <<-EOF +Start disable of named table: + hbase> disable 't1' + hbase> disable 'ns1:t1' +EOF + end + + def command(table) + admin.disable(table) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/disable_all.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/disable_all.rb new file mode 100644 index 0000000000..b4c5a39f40 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/disable_all.rb @@ -0,0 +1,50 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class DisableAll < Command + def help + <<-EOF +Disable all of tables matching the given regex: + +hbase> disable_all 't.*' +hbase> disable_all 'ns:t.*' +hbase> disable_all 'ns:.*' +EOF + end + + def command(regex) + list = admin.list(regex) + count = list.size + list.each do |table| + formatter.row([table]) + end + puts "\nDisable the above #{count} tables (y/n)?" unless count == 0 + answer = 'n' + answer = gets.chomp unless count == 0 + puts "No tables matched the regex #{regex}" if count == 0 + return unless answer =~ /y.*/i + failed = admin.disable_all(regex) + puts "#{count - failed.size} tables successfully disabled" + puts "#{failed.size} tables not disabled due to an exception: #{failed.join ','}" unless failed.empty? + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/disable_exceed_throttle_quota.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/disable_exceed_throttle_quota.rb new file mode 100644 index 0000000000..8be46ee174 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/disable_exceed_throttle_quota.rb @@ -0,0 +1,40 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class DisableExceedThrottleQuota < Command + def help + <<-EOF +Disable exceed throttle quota. Returns previous exceed throttle quota enabled value. +NOTE: if quota is not enabled, this will not work and always return false. + +Examples: + hbase> disable_exceed_throttle_quota + EOF + end + + def command + prev_state = !!quotas_admin.switch_exceed_throttle_quota(false) + formatter.row(["Previous exceed throttle quota enabled : #{prev_state}"]) + prev_state + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/disable_peer.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/disable_peer.rb new file mode 100644 index 0000000000..102731667c --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/disable_peer.rb @@ -0,0 +1,39 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class DisablePeer < Command + def help + <<-EOF +Stops the replication stream to the specified cluster, but still +keeps track of new edits to replicate. + +Examples: + + hbase> disable_peer '1' +EOF + end + + def command(id) + replication_admin.disable_peer(id) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/disable_rpc_throttle.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/disable_rpc_throttle.rb new file mode 100644 index 0000000000..03652afeb9 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/disable_rpc_throttle.rb @@ -0,0 +1,40 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class DisableRpcThrottle < Command + def help + <<-EOF +Disable quota rpc throttle. Returns previous rpc throttle enabled value. +NOTE: if quota is not enabled, this will not work and always return false. + +Examples: + hbase> disable_rpc_throttle + EOF + end + + def command + prev_state = !!quotas_admin.switch_rpc_throttle(false) + formatter.row(["Previous rpc throttle state : #{prev_state}"]) + prev_state + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/disable_table_replication.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/disable_table_replication.rb new file mode 100644 index 0000000000..0e903ac8f7 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/disable_table_replication.rb @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class DisableTableReplication < Command + def help + <<-EOF +Disable a table's replication switch. + +Examples: + + hbase> disable_table_replication 'table_name' +EOF + end + + def command(table_name) + replication_admin.disable_tablerep(table_name) + puts "Replication of table '#{table_name}' successfully disabled." + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/drop.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/drop.rb new file mode 100644 index 0000000000..063eb32773 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/drop.rb @@ -0,0 +1,36 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Drop < Command + def help + <<-EOF +Drop the named table. Table must first be disabled: + hbase> drop 't1' + hbase> drop 'ns1:t1' +EOF + end + + def command(table) + admin.drop(table) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/drop_all.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/drop_all.rb new file mode 100644 index 0000000000..5c40dfd134 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/drop_all.rb @@ -0,0 +1,50 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class DropAll < Command + def help + <<-EOF +Drop all of the tables matching the given regex: + +hbase> drop_all 't.*' +hbase> drop_all 'ns:t.*' +hbase> drop_all 'ns:.*' +EOF + end + + def command(regex) + list = admin.list(regex) + count = list.size + list.each do |table| + formatter.row([table]) + end + puts "\nDrop the above #{count} tables (y/n)?" unless count == 0 + answer = 'n' + answer = gets.chomp unless count == 0 + puts "No tables matched the regex #{regex}" if count == 0 + return unless answer =~ /y.*/i + failed = admin.drop_all(regex) + puts "#{count - failed.size} tables successfully dropped" + puts "#{failed.size} tables not dropped due to an exception: #{failed.join ','}" unless failed.empty? + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/drop_namespace.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/drop_namespace.rb new file mode 100644 index 0000000000..e667c2ae21 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/drop_namespace.rb @@ -0,0 +1,34 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class DropNamespace < Command + def help + <<-EOF +Drop the named namespace. The namespace must be empty. +EOF + end + + def command(namespace) + admin.drop_namespace(namespace) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/enable.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/enable.rb new file mode 100644 index 0000000000..13598ecdf5 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/enable.rb @@ -0,0 +1,36 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Enable < Command + def help + <<-EOF +Start enable of named table: + hbase> enable 't1' + hbase> enable 'ns1:t1' +EOF + end + + def command(table) + admin.enable(table) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/enable_all.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/enable_all.rb new file mode 100644 index 0000000000..a58f1187a5 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/enable_all.rb @@ -0,0 +1,50 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class EnableAll < Command + def help + <<-EOF +Enable all of the tables matching the given regex: + +hbase> enable_all 't.*' +hbase> enable_all 'ns:t.*' +hbase> enable_all 'ns:.*' +EOF + end + + def command(regex) + list = admin.list(regex) + count = list.size + list.each do |table| + formatter.row([table]) + end + puts "\nEnable the above #{count} tables (y/n)?" unless count == 0 + answer = 'n' + answer = gets.chomp unless count == 0 + puts "No tables matched the regex #{regex}" if count == 0 + return unless answer =~ /y.*/i + failed = admin.enable_all(regex) + puts "#{count - failed.size} tables successfully enabled" + puts "#{failed.size} tables not enabled due to an exception: #{failed.join ','}" unless failed.empty? + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/enable_exceed_throttle_quota.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/enable_exceed_throttle_quota.rb new file mode 100644 index 0000000000..26b2b86d33 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/enable_exceed_throttle_quota.rb @@ -0,0 +1,50 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class EnableExceedThrottleQuota < Command + def help + <<-EOF +Enable exceed throttle quota. Returns previous exceed throttle quota enabled value. +NOTE: if quota is not enabled, this will not work and always return false. + +If enabled, allow requests exceed user/table/namespace throttle quotas when region +server has available quota. + +There are two limits if enable exceed throttle quota. First, please set region server +quota. Second, please make sure that all region server throttle quotas are in seconds +time unit, because once previous requests exceed their quota and consume region server +quota, quota in other time units may be refilled in a long time, which may affect later +requests. + + +Examples: + hbase> enable_exceed_throttle_quota + EOF + end + + def command + prev_state = !!quotas_admin.switch_exceed_throttle_quota(true) + formatter.row(["Previous exceed throttle quota enabled : #{prev_state}"]) + prev_state + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/enable_peer.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/enable_peer.rb new file mode 100644 index 0000000000..19cf7af26f --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/enable_peer.rb @@ -0,0 +1,39 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class EnablePeer < Command + def help + <<-EOF +Restarts the replication to the specified peer cluster, +continuing from where it was disabled. + +Examples: + + hbase> enable_peer '1' +EOF + end + + def command(id) + replication_admin.enable_peer(id) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/enable_rpc_throttle.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/enable_rpc_throttle.rb new file mode 100644 index 0000000000..f992b7ceeb --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/enable_rpc_throttle.rb @@ -0,0 +1,40 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class EnableRpcThrottle < Command + def help + <<-EOF +Enable quota rpc throttle. Returns previous rpc throttle enabled value. +NOTE: if quota is not enabled, this will not work and always return false. + +Examples: + hbase> enable_rpc_throttle + EOF + end + + def command + prev_state = !!quotas_admin.switch_rpc_throttle(true) + formatter.row(["Previous rpc throttle state : #{prev_state}"]) + prev_state + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/enable_table_replication.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/enable_table_replication.rb new file mode 100644 index 0000000000..47eb649de8 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/enable_table_replication.rb @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class EnableTableReplication < Command + def help + <<-EOF +Enable a table's replication switch. + +Examples: + + hbase> enable_table_replication 'table_name' +EOF + end + + def command(table_name) + replication_admin.enable_tablerep(table_name) + puts "The replication of table '#{table_name}' successfully enabled" + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/exists.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/exists.rb new file mode 100644 index 0000000000..1471d4aa65 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/exists.rb @@ -0,0 +1,40 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Exists < Command + def help + <<-EOF +Does the named table exist? + hbase> exists 't1' + hbase> exists 'ns1:t1' +EOF + end + + def command(table) + exists = admin.exists?(table.to_s) + formatter.row([ + "Table #{table} " + (exists ? 'does exist' : 'does not exist') + ]) + exists + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/flush.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/flush.rb new file mode 100644 index 0000000000..69bcf13467 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/flush.rb @@ -0,0 +1,47 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Flush < Command + def help + <<-EOF +Flush all regions in passed table or pass a region row to +flush an individual region or a region server name whose format +is 'host,port,startcode', to flush all its regions. +You can also flush a single column family for all regions within a table, +or for an specific region only. +For example: + + hbase> flush 'TABLENAME' + hbase> flush 'TABLENAME','FAMILYNAME' + hbase> flush 'REGIONNAME' + hbase> flush 'REGIONNAME','FAMILYNAME' + hbase> flush 'ENCODED_REGIONNAME' + hbase> flush 'ENCODED_REGIONNAME','FAMILYNAME' + hbase> flush 'REGION_SERVER_NAME' +EOF + end + + def command(table_or_region_name, family = nil) + admin.flush(table_or_region_name, family) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/flush_master_store.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/flush_master_store.rb new file mode 100644 index 0000000000..91b8961fff --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/flush_master_store.rb @@ -0,0 +1,37 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class FlushMasterStore < Command + def help + <<-EOF +Flush master local region. +For example: + + hbase> flush_master_store +EOF + end + + def command() + admin.flush_master_store() + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get.rb new file mode 100644 index 0000000000..cdf429996b --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get.rb @@ -0,0 +1,103 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Get < Command + def help + <<-EOF +Get row or cell contents; pass table name, row, and optionally +a dictionary of column(s), timestamp, timerange and versions. Examples: + + hbase> get 'ns1:t1', 'r1' + hbase> get 't1', 'r1' + hbase> get 't1', 'r1', {TIMERANGE => [ts1, ts2]} + hbase> get 't1', 'r1', {COLUMN => 'c1'} + hbase> get 't1', 'r1', {COLUMN => ['c1', 'c2', 'c3']} + hbase> get 't1', 'r1', {COLUMN => 'c1', TIMESTAMP => ts1} + hbase> get 't1', 'r1', {COLUMN => 'c1', TIMERANGE => [ts1, ts2], VERSIONS => 4} + hbase> get 't1', 'r1', {COLUMN => 'c1', TIMESTAMP => ts1, VERSIONS => 4} + hbase> get 't1', 'r1', {FILTER => "ValueFilter(=, 'binary:abc')"} + hbase> get 't1', 'r1', 'c1' + hbase> get 't1', 'r1', 'c1', 'c2' + hbase> get 't1', 'r1', ['c1', 'c2'] + hbase> get 't1', 'r1', {COLUMN => 'c1', ATTRIBUTES => {'mykey'=>'myvalue'}} + hbase> get 't1', 'r1', {COLUMN => 'c1', AUTHORIZATIONS => ['PRIVATE','SECRET']} + hbase> get 't1', 'r1', {CONSISTENCY => 'TIMELINE'} + hbase> get 't1', 'r1', {CONSISTENCY => 'TIMELINE', REGION_REPLICA_ID => 1} + +Besides the default 'toStringBinary' format, 'get' also supports custom formatting by +column. A user can define a FORMATTER by adding it to the column name in the get +specification. The FORMATTER can be stipulated: + + 1. either as a org.apache.hadoop.hbase.util.Bytes method name (e.g, toInt, toString) + 2. or as a custom class followed by method name: e.g. 'c(MyFormatterClass).format'. + +Example formatting cf:qualifier1 and cf:qualifier2 both as Integers: + hbase> get 't1', 'r1' {COLUMN => ['cf:qualifier1:toInt', + 'cf:qualifier2:c(org.apache.hadoop.hbase.util.Bytes).toInt'] } + +Note that you can specify a FORMATTER by column only (cf:qualifier). You can set a +formatter for all columns (including, all key parts) using the "FORMATTER" +and "FORMATTER_CLASS" options. The default "FORMATTER_CLASS" is +"org.apache.hadoop.hbase.util.Bytes". + + hbase> get 't1', 'r1', {FORMATTER => 'toString'} + hbase> get 't1', 'r1', {FORMATTER_CLASS => 'org.apache.hadoop.hbase.util.Bytes', FORMATTER => 'toString'} + +The same commands also can be run on a reference to a table (obtained via get_table or +create_table). Suppose you had a reference t to table 't1', the corresponding commands +would be: + + hbase> t.get 'r1' + hbase> t.get 'r1', {TIMERANGE => [ts1, ts2]} + hbase> t.get 'r1', {COLUMN => 'c1'} + hbase> t.get 'r1', {COLUMN => ['c1', 'c2', 'c3']} + hbase> t.get 'r1', {COLUMN => 'c1', TIMESTAMP => ts1} + hbase> t.get 'r1', {COLUMN => 'c1', TIMERANGE => [ts1, ts2], VERSIONS => 4} + hbase> t.get 'r1', {COLUMN => 'c1', TIMESTAMP => ts1, VERSIONS => 4} + hbase> t.get 'r1', {FILTER => "ValueFilter(=, 'binary:abc')"} + hbase> t.get 'r1', 'c1' + hbase> t.get 'r1', 'c1', 'c2' + hbase> t.get 'r1', ['c1', 'c2'] + hbase> t.get 'r1', {CONSISTENCY => 'TIMELINE'} + hbase> t.get 'r1', {CONSISTENCY => 'TIMELINE', REGION_REPLICA_ID => 1} +EOF + end + + def command(table, row, *args) + get(table(table), row, *args) + end + + def get(table, row, *args) + @start_time = Time.now + formatter.header(%w[COLUMN CELL]) + + count, is_stale = table._get_internal(row, *args) do |column, value| + formatter.row([column, value]) + end + + formatter.footer(count, is_stale) + end + end + end +end + +# add get command to table +::Hbase::Table.add_shell_command('get') diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_auths.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_auths.rb new file mode 100644 index 0000000000..80e7aeb128 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_auths.rb @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class GetAuths < Command + def help + <<-EOF +Get the visibility labels set for a particular user or group +Syntax : get_auths 'user' + +For example: + + hbase> get_auths 'user1' + hbase> get_auths '@group1' +EOF + end + + def command(user) + list = visibility_labels_admin.get_auths(user) + list.each do |auths| + formatter.row([org.apache.hadoop.hbase.util.Bytes.toStringBinary(auths.toByteArray)]) + end + list + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_balancer_decisions.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_balancer_decisions.rb new file mode 100644 index 0000000000..801166eb20 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_balancer_decisions.rb @@ -0,0 +1,49 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Retrieve latest balancer decisions maintained in memory by HMaster + +module Shell + module Commands + # Retrieve latest large log responses + class GetBalancerDecisions < Command + def help + <<-EOF +Retrieve latest balancer decisions made by LoadBalancers. + +Examples: + + hbase> get_balancer_decisions => Retrieve recent balancer decisions with + region plans + hbase> get_balancer_decisions LIMIT => 10 => Retrieve 10 most recent balancer decisions + with region plans + + EOF + end + + def command(args = {}) + unless args.is_a? Hash + raise 'Filter parameters are not Hash' + end + + balancer_decisions_resp_arr = admin.get_balancer_decisions(args) + puts 'Retrieved BalancerDecision Responses' + puts balancer_decisions_resp_arr + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_balancer_rejections.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_balancer_rejections.rb new file mode 100644 index 0000000000..df04a2cdfe --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_balancer_rejections.rb @@ -0,0 +1,49 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Retrieve latest balancer rejections maintained in memory by HMaster + +module Shell + module Commands + # Retrieve latest large log responses + class GetBalancerRejections < Command + def help + <<-EOF +Retrieve latest balancer rejections made by LoadBalancers. + +Examples: + + hbase> get_balancer_rejections => Retrieve recent balancer rejections with + region plans + hbase> get_balancer_rejections LIMIT => 10 => Retrieve 10 most recent balancer rejections + with region plans + + EOF + end + + def command(args = {}) + unless args.is_a? Hash + raise 'Filter parameters are not Hash' + end + + balancer_rejections_resp_arr = admin.get_balancer_rejections(args) + puts 'Retrieved BalancerRejection Responses' + puts balancer_rejections_resp_arr + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_counter.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_counter.rb new file mode 100644 index 0000000000..12ab4b2f2b --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_counter.rb @@ -0,0 +1,54 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class GetCounter < Command + def help + <<-EOF +Return a counter cell value at specified table/row/column coordinates. +A counter cell should be managed with atomic increment functions on HBase +and the data should be binary encoded (as long value). Example: + + hbase> get_counter 'ns1:t1', 'r1', 'c1' + hbase> get_counter 't1', 'r1', 'c1' + +The same commands also can be run on a table reference. Suppose you had a reference +t to table 't1', the corresponding command would be: + + hbase> t.get_counter 'r1', 'c1' +EOF + end + + def command(table, row, column) + get_counter(table(table), row, column) + end + + def get_counter(table, row, column) + if cnt = table._get_counter_internal(row, column) + puts "COUNTER VALUE = #{cnt}" + else + puts 'No counter found at specified coordinates' + end + end + end + end +end + +::Hbase::Table.add_shell_command('get_counter') diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_largelog_responses.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_largelog_responses.rb new file mode 100644 index 0000000000..8ed55abfc1 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_largelog_responses.rb @@ -0,0 +1,99 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Retrieve latest large log responses maintained in memory by RegionServers + +module Shell + module Commands + # Retrieve latest large log responses + class GetLargelogResponses < Command + def help + <<-EOF +Retrieve latest LargeLog Responses maintained by each or specific RegionServers. +Specify '*' to include all RS otherwise array of server names for specific +RS. A server name is the host, port plus startcode of a RegionServer. +e.g.: host187.example.com,60020,1289493121758 (find servername in +master ui or when you do detailed status in shell) + +Provide optional filter parameters as Hash. +Default Limit of each server for providing no of large log records is 10. User can specify +more limit by 'LIMIT' param in case more than 10 records should be retrieved. + +Examples: + + hbase> get_largelog_responses '*' => get largelog responses from all RS + hbase> get_largelog_responses '*', {'LIMIT' => 50} => get largelog responses from all RS + with 50 records limit (default limit: 10) + hbase> get_largelog_responses ['SERVER_NAME1', 'SERVER_NAME2'] => get largelog responses from SERVER_NAME1, + SERVER_NAME2 + hbase> get_largelog_responses '*', {'REGION_NAME' => 'hbase:meta,,1'} + => get largelog responses only related to meta + region + hbase> get_largelog_responses '*', {'TABLE_NAME' => 't1'} => get largelog responses only related to t1 table + hbase> get_largelog_responses '*', {'CLIENT_IP' => '192.162.1.40:60225', 'LIMIT' => 100} + => get largelog responses with given client + IP address and get 100 records limit + (default limit: 10) + hbase> get_largelog_responses '*', {'REGION_NAME' => 'hbase:meta,,1', 'TABLE_NAME' => 't1'} + => get largelog responses with given region name + or table name + hbase> get_largelog_responses '*', {'USER' => 'user_name', 'CLIENT_IP' => '192.162.1.40:60225'} + => get largelog responses that match either + provided client IP address or user name + +All of above queries with filters have default OR operation applied i.e. all +records with any of the provided filters applied will be returned. However, +we can also apply AND operator i.e. all records that match all (not any) of +the provided filters should be returned. + + hbase> get_largelog_responses '*', {'REGION_NAME' => 'hbase:meta,,1', 'TABLE_NAME' => 't1', 'FILTER_BY_OP' => 'AND'} + => get largelog responses with given region name + and table name, both should match + + hbase> get_largelog_responses '*', {'REGION_NAME' => 'hbase:meta,,1', 'TABLE_NAME' => 't1', 'FILTER_BY_OP' => 'OR'} + => get largelog responses with given region name + or table name, any one can match + + hbase> get_largelog_responses '*', {'TABLE_NAME' => 't1', 'CLIENT_IP' => '192.163.41.53:52781', 'FILTER_BY_OP' => 'AND'} + => get largelog responses with given region name + and client IP address, both should match + +Since OR is the default filter operator, without providing 'FILTER_BY_OP', query will have +same result as providing 'FILTER_BY_OP' => 'OR'. + +Sometimes output can be long pretty printed json for user to scroll in +a single screen and hence user might prefer +redirecting output of get_largelog_responses to a file. + +Example: + +echo "get_largelog_responses '*'" | hbase shell > xyz.out 2>&1 + + EOF + end + + def command(server_names, args = {}) + unless args.is_a? Hash + raise 'Filter parameters are not Hash' + end + large_log_responses_arr = admin.get_slowlog_responses(server_names, args, true) + puts 'Retrieved LargeLog Responses from RegionServers' + puts large_log_responses_arr + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_namespace_rsgroup.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_namespace_rsgroup.rb new file mode 100644 index 0000000000..a4991d1694 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_namespace_rsgroup.rb @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class GetNamespaceRsgroup < Command + def help + <<-EOF +Get the group name the given NameSpace is a member of. + +Example: + + hbase> get_namespace_rsgroup 'namespace_name' + +EOF + end + + def command(namespace_name) + group_name = admin.get_namespace_rsgroup(namespace_name) + unless group_name.nil? + formatter.row([group_name]) + end + formatter.footer(1) + end + end + end +end \ No newline at end of file diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_peer_config.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_peer_config.rb new file mode 100644 index 0000000000..3f4a9492c9 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_peer_config.rb @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class GetPeerConfig < Command + def help + <<~EOF + Outputs the cluster key, replication endpoint class (if present), and any replication configuration parameters + EOF + end + + def command(id) + peer_config = replication_admin.get_peer_config(id) + @start_time = Time.now + format_peer_config(peer_config) + peer_config + end + + def format_peer_config(peer_config) + cluster_key = peer_config.get_cluster_key + endpoint = peer_config.get_replication_endpoint_impl + + formatter.row(['Cluster Key', cluster_key]) unless cluster_key.nil? + formatter.row(['Replication Endpoint', endpoint]) unless endpoint.nil? + unless peer_config.get_configuration.nil? + peer_config.get_configuration.each do |config_entry| + formatter.row(config_entry) + end + end + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_rsgroup.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_rsgroup.rb new file mode 100644 index 0000000000..8f5b1c1f92 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_rsgroup.rb @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class GetRsgroup < Command + def help + <<-EOF +Get a RegionServer group's information. + +Example: + + hbase> get_rsgroup 'default' + +EOF + end + + def command(group_name) + group = rsgroup_admin.get_rsgroup(group_name) + + formatter.header(['SERVERS']) + group.getServers.each do |server| + formatter.row([server.toString]) + end + formatter.footer + + formatter.header(['TABLES']) + group.getTables.each do |table| + formatter.row([table.getNameAsString]) + end + formatter.footer + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_server_rsgroup.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_server_rsgroup.rb new file mode 100644 index 0000000000..042af2d19d --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_server_rsgroup.rb @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class GetServerRsgroup < Command + def help + <<-EOF +Get the group name the given RegionServer is a member of. + +Example: + + hbase> get_server_rsgroup 'server1:port1' + +EOF + end + + def command(server) + group_name = rsgroup_admin.get_rsgroup_of_server(server).getName + formatter.row([group_name]) + formatter.footer(1) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_slowlog_responses.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_slowlog_responses.rb new file mode 100644 index 0000000000..2dc108b1d6 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_slowlog_responses.rb @@ -0,0 +1,99 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Retrieve latest slowlog responses maintained in memory by RegionServers + +module Shell + module Commands + # Retrieve latest slowlog responses + class GetSlowlogResponses < Command + def help + <<-EOF +Retrieve latest SlowLog Responses maintained by each or specific RegionServers. +Specify '*' to include all RS otherwise array of server names for specific +RS. A server name is the host, port plus startcode of a RegionServer. +e.g.: host187.example.com,60020,1289493121758 (find servername in +master ui or when you do detailed status in shell) + +Provide optional filter parameters as Hash. +Default Limit of each server for providing no of slow log records is 10. User can specify +more limit by 'LIMIT' param in case more than 10 records should be retrieved. + +Examples: + + hbase> get_slowlog_responses '*' => get slowlog responses from all RS + hbase> get_slowlog_responses '*', {'LIMIT' => 50} => get slowlog responses from all RS + with 50 records limit (default limit: 10) + hbase> get_slowlog_responses ['SERVER_NAME1', 'SERVER_NAME2'] => get slowlog responses from SERVER_NAME1, + SERVER_NAME2 + hbase> get_slowlog_responses '*', {'REGION_NAME' => 'hbase:meta,,1'} + => get slowlog responses only related to meta + region + hbase> get_slowlog_responses '*', {'TABLE_NAME' => 't1'} => get slowlog responses only related to t1 table + hbase> get_slowlog_responses '*', {'CLIENT_IP' => '192.162.1.40:60225', 'LIMIT' => 100} + => get slowlog responses with given client + IP address and get 100 records limit + (default limit: 10) + hbase> get_slowlog_responses '*', {'REGION_NAME' => 'hbase:meta,,1', 'TABLE_NAME' => 't1'} + => get slowlog responses with given region name + or table name + hbase> get_slowlog_responses '*', {'USER' => 'user_name', 'CLIENT_IP' => '192.162.1.40:60225'} + => get slowlog responses that match either + provided client IP address or user name + +All of above queries with filters have default OR operation applied i.e. all +records with any of the provided filters applied will be returned. However, +we can also apply AND operator i.e. all records that match all (not any) of +the provided filters should be returned. + + hbase> get_slowlog_responses '*', {'REGION_NAME' => 'hbase:meta,,1', 'TABLE_NAME' => 't1', 'FILTER_BY_OP' => 'AND'} + => get slowlog responses with given region name + and table name, both should match + + hbase> get_slowlog_responses '*', {'REGION_NAME' => 'hbase:meta,,1', 'TABLE_NAME' => 't1', 'FILTER_BY_OP' => 'OR'} + => get slowlog responses with given region name + or table name, any one can match + + hbase> get_slowlog_responses '*', {'TABLE_NAME' => 't1', 'CLIENT_IP' => '192.163.41.53:52781', 'FILTER_BY_OP' => 'AND'} + => get slowlog responses with given region name + and client IP address, both should match + +Since OR is the default filter operator, without providing 'FILTER_BY_OP', query will have +same result as providing 'FILTER_BY_OP' => 'OR'. + +Sometimes output can be long pretty printed json for user to scroll in +a single screen and hence user might prefer +redirecting output of get_slowlog_responses to a file. + +Example: + +echo "get_slowlog_responses '*'" | hbase shell > xyz.out 2>&1 + + EOF + end + + def command(server_names, args = {}) + unless args.is_a? Hash + raise 'Filter parameters are not Hash' + end + slow_log_responses_arr = admin.get_slowlog_responses(server_names, args) + puts 'Retrieved SlowLog Responses from RegionServers' + puts slow_log_responses_arr + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_splits.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_splits.rb new file mode 100644 index 0000000000..49a7debaab --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_splits.rb @@ -0,0 +1,50 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class GetSplits < Command + def help + <<-EOF +Get the splits of the named table: + hbase> get_splits 't1' + hbase> get_splits 'ns1:t1' + +The same commands also can be run on a table reference. Suppose you had a reference +t to table 't1', the corresponding command would be: + + hbase> t.get_splits +EOF + end + + def command(table) + get_splits(table(table)) + end + + def get_splits(table) + splits = table._get_splits_internal + puts(format('Total number of splits = %d', + numsplits: (splits.size + 1))) + puts splits + splits + end + end + end +end + +::Hbase::Table.add_shell_command('get_splits') diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_table.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_table.rb new file mode 100644 index 0000000000..6ad4f33e3e --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_table.rb @@ -0,0 +1,45 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class GetTable < Command + def help + <<-EOF +Get the given table name and return it as an actual object to +be manipulated by the user. See table.help for more information +on how to use the table. +Eg. + + hbase> t1 = get_table 't1' + hbase> t1 = get_table 'ns1:t1' + +returns the table named 't1' as a table object. You can then do + + hbase> t1.help + +which will then print the help for that table. +EOF + end + + def command(table, *_args) + table(table) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_table_rsgroup.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_table_rsgroup.rb new file mode 100644 index 0000000000..dd48ca93c1 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/get_table_rsgroup.rb @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class GetTableRsgroup < Command + def help + <<-EOF +Get the RegionServer group name the given table is a member of. + +Example: + + hbase> get_table_rsgroup 'myTable' + +EOF + end + + def command(table) + group_name = + rsgroup_admin.get_rsgroup_of_table(table).getName + formatter.row([group_name]) + formatter.footer(1) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/grant.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/grant.rb new file mode 100644 index 0000000000..b5ee4ab863 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/grant.rb @@ -0,0 +1,119 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Grant < Command + def help + <<-EOF +Grant users specific rights. +Syntax: grant , [, [, [, ]]] +Syntax: grant , , <@namespace> + +permissions is either zero or more letters from the set "RWXCA". +READ('R'), WRITE('W'), EXEC('X'), CREATE('C'), ADMIN('A') + +Note: Groups and users are granted access in the same way, but groups are prefixed with an '@' + character. Tables and namespaces are specified the same way, but namespaces are + prefixed with an '@' character. + +For example: + + hbase> grant 'bobsmith', 'RWXCA' + hbase> grant '@admins', 'RWXCA' + hbase> grant 'bobsmith', 'RWXCA', '@ns1' + hbase> grant 'bobsmith', 'RW', 't1', 'f1', 'col1' + hbase> grant 'bobsmith', 'RW', 'ns1:t1', 'f1', 'col1' +EOF + end + + def command(*args) + # command form is ambiguous at first argument + table_name = user = args[0] + raise(ArgumentError, 'First argument should be a String') unless user.is_a?(String) + + if args[1].is_a?(String) + + # Original form of the command + # user in args[0] + # permissions in args[1] + # table_name in args[2] + # family in args[3] or nil + # qualifier in args[4] or nil + + permissions = args[1] + raise(ArgumentError, 'Permissions are not of String type') unless permissions.is_a?( + String + ) + table_name = family = qualifier = nil + table_name = args[2] # will be nil if unset + unless table_name.nil? + raise(ArgumentError, 'Table name is not of String type') unless table_name.is_a?( + String + ) + family = args[3] # will be nil if unset + unless family.nil? + raise(ArgumentError, 'Family is not of String type') unless family.is_a?(String) + qualifier = args[4] # will be nil if unset + unless qualifier.nil? + raise(ArgumentError, 'Qualifier is not of String type') unless qualifier.is_a?( + String + ) + end + end + end + @start_time = Time.now + security_admin.grant(user, permissions, table_name, family, qualifier) + + elsif args[1].is_a?(Hash) + + # New form of the command, a cell ACL update + # table_name in args[0], a string + # a Hash mapping users (or groups) to permisisons in args[1] + # a Hash argument suitable for passing to Table#_get_scanner in args[2] + # Useful for feature testing and debugging. + + permissions = args[1] + raise(ArgumentError, 'Permissions are not of Hash type') unless permissions.is_a?(Hash) + scan = args[2] + raise(ArgumentError, 'Scanner specification is not a Hash') unless scan.is_a?(Hash) + + t = table(table_name) + @start_time = Time.now + scanner = t._get_scanner(scan) + count = 0 + iter = scanner.iterator + while iter.hasNext + row = iter.next + row.listCells.each do |cell| + put = org.apache.hadoop.hbase.client.Put.new(row.getRow) + put.add(cell) + t.set_cell_permissions(put, permissions) + t.table.put(put) + end + count += 1 + end + formatter.footer(count) + + else + raise(ArgumentError, 'Second argument should be a String or Hash') + end + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/hbck_chore_run.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/hbck_chore_run.rb new file mode 100644 index 0000000000..47ff636c83 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/hbck_chore_run.rb @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class HbckChoreRun < Command + def help + <<-EOF +Request HBCK chore to run at master side. It will try to find the orphan +regions on RegionServer or FileSystem and find the inconsistent regions. +You can check the HBCK report at Master web UI. Returns true if HBCK chore +ran, or false if HBCK chore is already running. + + hbase> hbck_chore_run + +EOF + end + + def command + admin.hbck_chore_run + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/incr.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/incr.rb new file mode 100644 index 0000000000..f1c31210af --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/incr.rb @@ -0,0 +1,63 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Incr < Command + def help + <<-EOF +Increments a cell 'value' at specified table/row/column coordinates. +To increment a cell value in table 'ns1:t1' or 't1' at row 'r1' under column +'c1' by 1 (can be omitted) or 10 do: + + hbase> incr 'ns1:t1', 'r1', 'c1' + hbase> incr 't1', 'r1', 'c1' + hbase> incr 't1', 'r1', 'c1', 1 + hbase> incr 't1', 'r1', 'c1', 10 + hbase> incr 't1', 'r1', 'c1', 10, {ATTRIBUTES=>{'mykey'=>'myvalue'}} + hbase> incr 't1', 'r1', 'c1', {ATTRIBUTES=>{'mykey'=>'myvalue'}} + hbase> incr 't1', 'r1', 'c1', 10, {VISIBILITY=>'PRIVATE|SECRET'} + +The same commands also can be run on a table reference. Suppose you had a reference +t to table 't1', the corresponding command would be: + + hbase> t.incr 'r1', 'c1' + hbase> t.incr 'r1', 'c1', 1 + hbase> t.incr 'r1', 'c1', 10, {ATTRIBUTES=>{'mykey'=>'myvalue'}} + hbase> t.incr 'r1', 'c1', 10, {VISIBILITY=>'PRIVATE|SECRET'} +EOF + end + + def command(table, row, column, value = nil, args = {}) + incr(table(table), row, column, value, args) + end + + def incr(table, row, column, value = nil, args = {}) + if cnt = table._incr_internal(row, column, value, args) + puts "COUNTER VALUE = #{cnt}" + else + puts 'No counter found at specified coordinates' + end + end + end + end +end + +# add incr comamnd to Table +::Hbase::Table.add_shell_command('incr') diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/is_disabled.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/is_disabled.rb new file mode 100644 index 0000000000..477ba57a02 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/is_disabled.rb @@ -0,0 +1,38 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class IsDisabled < Command + def help + <<-EOF +Is named table disabled? For example: + hbase> is_disabled 't1' + hbase> is_disabled 'ns1:t1' +EOF + end + + def command(table) + disabled = !!admin.disabled?(table) + formatter.row([disabled.to_s]) + disabled + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/is_enabled.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/is_enabled.rb new file mode 100644 index 0000000000..fa122db374 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/is_enabled.rb @@ -0,0 +1,38 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class IsEnabled < Command + def help + <<-EOF +Is named table enabled? For example: + hbase> is_enabled 't1' + hbase> is_enabled 'ns1:t1' +EOF + end + + def command(table) + enabled = admin.enabled?(table) + formatter.row([enabled ? 'true' : 'false']) + enabled + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/is_in_maintenance_mode.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/is_in_maintenance_mode.rb new file mode 100644 index 0000000000..8ed244cebb --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/is_in_maintenance_mode.rb @@ -0,0 +1,38 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class IsInMaintenanceMode < Command + def help + <<-EOF +Is master in maintenance mode? For example: + + hbase> is_in_maintenance_mode +EOF + end + + def command + state = admin.in_maintenance_mode? + formatter.row([state.to_s]) + state + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list.rb new file mode 100644 index 0000000000..676b5bfd22 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list.rb @@ -0,0 +1,48 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class List < Command + def help + <<-EOF +List all user tables in hbase. Optional regular expression parameter could +be used to filter the output. Examples: + + hbase> list + hbase> list 'abc.*' + hbase> list 'ns:abc.*' + hbase> list 'ns:.*' +EOF + end + + def command(regex = '.*') + formatter.header(['TABLE']) + + list = admin.list(regex) + list.each do |table| + formatter.row([table]) + end + + formatter.footer(list.size) + list + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_deadservers.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_deadservers.rb new file mode 100644 index 0000000000..133dcfa187 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_deadservers.rb @@ -0,0 +1,44 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ListDeadservers < Command + def help + <<~EOF + List all dead region servers in hbase + Examples: + hbase> list_deadservers + EOF + end + + def command + now = Time.now + formatter.header(['SERVERNAME']) + + servers = admin.list_deadservers + servers.each do |server| + formatter.row([server.toString]) + end + + formatter.footer(servers.size) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_decommissioned_regionservers.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_decommissioned_regionservers.rb new file mode 100644 index 0000000000..5e03cfd952 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_decommissioned_regionservers.rb @@ -0,0 +1,42 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + # List decommissioned region servers + class ListDecommissionedRegionservers < Command + def help + <<~EOF + List region servers marked as decommissioned, which can not be assigned regions. +EOF + end + + def command + formatter.header(['DECOMMISSIONED REGION SERVERS']) + + list = admin.list_decommissioned_regionservers + list.each do |server_name| + formatter.row([server_name.getServerName]) + end + + formatter.footer(list.size) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_labels.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_labels.rb new file mode 100644 index 0000000000..8619642351 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_labels.rb @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ListLabels < Command + def help + <<-EOF +List the visibility labels defined in the system. +Optional regular expression parameter could be used to filter the labels being returned. +Syntax : list_labels + +For example: + + hbase> list_labels 'secret.*' + hbase> list_labels +EOF + end + + def command(regex = '.*') + list = visibility_labels_admin.list_labels(regex) + list.each do |label| + formatter.row([org.apache.hadoop.hbase.util.Bytes.toStringBinary(label.toByteArray)]) + end + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_liveservers.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_liveservers.rb new file mode 100644 index 0000000000..b57ef75c73 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_liveservers.rb @@ -0,0 +1,44 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + # List live region servers + class ListLiveservers < Command + def help + <<~EOF + List all live region servers in hbase + Examples: + hbase> list_liveservers + EOF + end + + def command + formatter.header(['SERVERNAME']) + + servers = admin.list_liveservers + servers.each do |server| + formatter.row([server.toString]) + end + + formatter.footer(servers.size) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_locks.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_locks.rb new file mode 100644 index 0000000000..1bebdd514e --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_locks.rb @@ -0,0 +1,63 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'json' + +module Shell + module Commands + class ListLocks < Command + def help + <<-EOF +List all locks in hbase. Examples: + + hbase> list_locks +EOF + end + + def command + list = JSON.parse(admin.list_locks) + + list.each do |lock| + formatter.output_strln("#{lock['resourceType']}(#{lock['resourceName']})") + + case lock['lockType'] + when 'EXCLUSIVE' then + formatter.output_strln("Lock type: #{lock['lockType']}, " \ + "procedure: #{lock['exclusiveLockOwnerProcedure']}") + when 'SHARED' then + formatter.output_strln("Lock type: #{lock['lockType']}, " \ + "count: #{lock['sharedLockCount']}") + end + + if lock['waitingProcedures'] + formatter.header(['Waiting procedures']) + + lock['waitingProcedures'].each do |waiting_procedure| + formatter.row([waiting_procedure]) + end + + formatter.footer(lock['waitingProcedures'].size) + end + + formatter.output_strln('') + end + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_namespace.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_namespace.rb new file mode 100644 index 0000000000..131337f482 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_namespace.rb @@ -0,0 +1,45 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ListNamespace < Command + def help + <<-EOF +List all namespaces in hbase. Optional regular expression parameter could +be used to filter the output. Examples: + + hbase> list_namespace + hbase> list_namespace 'abc.*' +EOF + end + + def command(regex = '.*') + formatter.header(['NAMESPACE']) + + list = admin.list_namespace(regex) + list.each do |table| + formatter.row([table]) + end + + formatter.footer(list.size) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_namespace_tables.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_namespace_tables.rb new file mode 100644 index 0000000000..ff08568ee4 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_namespace_tables.rb @@ -0,0 +1,45 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ListNamespaceTables < Command + def help + <<-EOF +List all tables that are members of the namespace. +Examples: + + hbase> list_namespace_tables 'ns1' +EOF + end + + def command(namespace) + formatter.header(['TABLE']) + + list = admin.list_namespace_tables(namespace) + list.each do |table| + formatter.row([table]) + end + + formatter.footer(list.size) + list + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_peer_configs.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_peer_configs.rb new file mode 100644 index 0000000000..85edc7e76d --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_peer_configs.rb @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ListPeerConfigs < Command + def help + <<~EOF + No-argument method that outputs the replication peer configuration for each peer defined on this cluster. + EOF + end + + def command + peer_configs = replication_admin.list_peer_configs + unless peer_configs.nil? + peer_configs.each do |peer_config_entry| + peer_id = peer_config_entry[0] + peer_config = peer_config_entry[1] + formatter.row(['PeerId', peer_id]) + GetPeerConfig.new(@shell).format_peer_config(peer_config) + formatter.row([' ']) + end + end + peer_configs + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_peers.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_peers.rb new file mode 100644 index 0000000000..40816a4891 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_peers.rb @@ -0,0 +1,67 @@ +# +# Copyright The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ListPeers < Command + def help + <<~EOF + List all replication peer clusters. + + If replicate_all flag is false, the namespaces and table-cfs in peer config + will be replicated to peer cluster. + + If replicate_all flag is true, all user tables will be replicate to peer + cluster, except that the namespaces and table-cfs in peer config. + + hbase> list_peers +EOF + end + + def command + peers = replication_admin.list_peers + + formatter.header(%w[PEER_ID CLUSTER_KEY ENDPOINT_CLASSNAME + STATE REPLICATE_ALL NAMESPACES TABLE_CFS BANDWIDTH + SERIAL]) + + peers.each do |peer| + id = peer.getPeerId + state = peer.isEnabled ? 'ENABLED' : 'DISABLED' + config = peer.getPeerConfig + if config.replicateAllUserTables + namespaces = replication_admin.show_peer_exclude_namespaces(config) + tableCFs = replication_admin.show_peer_exclude_tableCFs(config) + else + namespaces = replication_admin.show_peer_namespaces(config) + tableCFs = replication_admin.show_peer_tableCFs_by_config(config) + end + formatter.row([id, config.getClusterKey, + config.getReplicationEndpointImpl, state, + config.replicateAllUserTables, namespaces, tableCFs, + config.getBandwidth, config.isSerial]) + end + + formatter.footer + peers + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_procedures.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_procedures.rb new file mode 100644 index 0000000000..1d86b8dcdd --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_procedures.rb @@ -0,0 +1,47 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'json' + +module Shell + module Commands + class ListProcedures < Command + def help + <<-EOF +List all procedures in hbase. For example: + + hbase> list_procedures +EOF + end + + def command + formatter.header(%w[PID Name State Submitted Last_Update Parameters]) + list = JSON.parse(admin.list_procedures) + list.each do |proc| + submitted_time = Time.at(Integer(proc['submittedTime']) / 1000).to_s + last_update = Time.at(Integer(proc['lastUpdate']) / 1000).to_s + formatter.row([proc['procId'], proc['className'], proc['state'], + submitted_time, last_update, proc['stateMessage']]) + end + + formatter.footer(list.size) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_quota_snapshots.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_quota_snapshots.rb new file mode 100644 index 0000000000..08a96673b7 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_quota_snapshots.rb @@ -0,0 +1,87 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ListQuotaSnapshots < Command + def help + <<-EOF +Lists the current space quota snapshots with optional selection criteria. +Snapshots encapsulate relevant information to space quotas such as space +use, configured limits, and quota violation details. This command is +useful for understanding the current state of a cluster with space quotas. + +By default, this command will read all snapshots stored in the system from +the hbase:quota table. A table name or namespace can be provided to filter +the snapshots returned. RegionServers maintain a copy of snapshots, refreshing +at a regular interval; by providing a RegionServer option, snapshots will +be retreived from that RegionServer instead of the quota table. + +For example: + + hbase> list_quota_snapshots + hbase> list_quota_snapshots({TABLE => 'table1'}) + hbase> list_quota_snapshots({NAMESPACE => 'org1'}) + hbase> list_quota_snapshots({REGIONSERVER => 'server1.domain,16020,1483482894742'}) + hbase> list_quota_snapshots({NAMESPACE => 'org1', REGIONSERVER => 'server1.domain,16020,1483482894742'}) +EOF + end + + def command(args = {}) + # All arguments may be nil + desired_table = args[::HBaseConstants::TABLE] + desired_namespace = args[::HBaseConstants::NAMESPACE] + desired_regionserver = args[::HBaseConstants::REGIONSERVER] + formatter.header(%w[TABLE USAGE LIMIT IN_VIOLATION POLICY]) + count = 0 + quotas_admin.get_quota_snapshots(desired_regionserver).each do |table_name, snapshot| + # Skip this snapshot if it's for a table/namespace the user did not ask for + next unless accept? table_name, desired_table, desired_namespace + status = snapshot.getQuotaStatus + policy = get_policy(status) + formatter.row([table_name.to_s, snapshot.getUsage.to_s, snapshot.getLimit.to_s, + status.isInViolation.to_s, policy]) + count += 1 + end + formatter.footer(count) + end + + def get_policy(status) + # Unwrap the violation policy if it exists + if status.isInViolation + status.getPolicy.get.name + else + 'None' + end + end + + def accept?(table_name, desired_table = nil, desired_namespace = nil) + # Check the table name if given one + if desired_table && table_name.getQualifierAsString != desired_table + return false + end + # Check the namespace if given one + if desired_namespace && table_name.getNamespaceAsString != desired_namespace + return false + end + true + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_quota_table_sizes.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_quota_table_sizes.rb new file mode 100644 index 0000000000..ef7505e8c1 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_quota_table_sizes.rb @@ -0,0 +1,47 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ListQuotaTableSizes < Command + def help + <<-EOF +Lists the computed size of each table in the cluster as computed by +all RegionServers. This is the raw information that the Master uses to +make decisions about space quotas. Most times, using `list_quota_snapshots` +provides a higher-level of insight than this command. + +For example: + + hbase> list_quota_table_sizes +EOF + end + + def command(_args = {}) + formatter.header(%w[TABLE SIZE]) + count = 0 + quotas_admin.get_master_table_sizes.each do |tableName, size| + formatter.row([tableName.to_s, size.to_s]) + count += 1 + end + formatter.footer(count) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_quotas.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_quotas.rb new file mode 100644 index 0000000000..b20aae6762 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_quotas.rb @@ -0,0 +1,51 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ListQuotas < Command + def help + <<-EOF +List the quota settings added to the system. +You can filter the result based on USER, TABLE, or NAMESPACE. + +For example: + + hbase> list_quotas + hbase> list_quotas USER => 'bob.*' + hbase> list_quotas USER => 'bob.*', TABLE => 't1' + hbase> list_quotas USER => 'bob.*', NAMESPACE => 'ns.*' + hbase> list_quotas TABLE => 'myTable' + hbase> list_quotas NAMESPACE => 'ns.*' +EOF + end + + def command(args = {}) + formatter.header(%w[OWNER QUOTAS]) + + # actually do the scanning + count = quotas_admin.list_quotas(args) do |row, cells| + formatter.row([row, cells]) + end + + formatter.footer(count) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_regions.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_regions.rb new file mode 100644 index 0000000000..5027e27c0c --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_regions.rb @@ -0,0 +1,244 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ListRegions < Command + def help + <<~EOF + List all regions for a particular table as an array and also filter them by server name (optional) as prefix + and maximum locality (optional). By default, it will return all the regions for the table with any locality. + The command displays server name, region name, start key, end key, size of the region in MB, number of requests + and the locality. The information can be projected out via an array as third parameter. By default all these information + is displayed. Possible array values are SERVER_NAME, REGION_NAME, START_KEY, END_KEY, SIZE, REQ and LOCALITY. Values + are not case sensitive. If you don't want to filter by server name, pass an empty hash / string as shown below. + + Examples: + hbase> list_regions 'table_name' + hbase> list_regions 'table_name', 'server_name' + hbase> list_regions 'table_name', {SERVER_NAME => 'server_name', LOCALITY_THRESHOLD => 0.8} + hbase> list_regions 'table_name', {SERVER_NAME => 'server_name', LOCALITY_THRESHOLD => 0.8}, ['SERVER_NAME'] + hbase> list_regions 'table_name', {}, ['SERVER_NAME', 'start_key'] + hbase> list_regions 'table_name', '', ['SERVER_NAME', 'start_key'] + + EOF + end + + def command(table_name, options = nil, cols = nil) + if options.nil? + options = {} + elsif !options.is_a? Hash + # When options isn't a hash, assume it's the server name + # and create the hash internally + options = { ::HBaseConstants::SERVER_NAME => options } + end + + raise "Table #{table_name} must be enabled." unless admin.enabled?(table_name) + + size_hash = {} + if cols.nil? + size_hash = { 'SERVER_NAME' => 12, 'REGION_NAME' => 12, 'START_KEY' => 10, 'END_KEY' => 10, 'SIZE' => 5, 'REQ' => 5, 'LOCALITY' => 10 } + elsif cols.is_a?(Array) + cols.each do |col| + if col.casecmp('SERVER_NAME').zero? + size_hash.store('SERVER_NAME', 12) + elsif col.casecmp('REGION_NAME').zero? + size_hash.store('REGION_NAME', 12) + elsif col.casecmp('START_KEY').zero? + size_hash.store('START_KEY', 10) + elsif col.casecmp('END_KEY').zero? + size_hash.store('END_KEY', 10) + elsif col.casecmp('SIZE').zero? + size_hash.store('SIZE', 5) + elsif col.casecmp('REQ').zero? + size_hash.store('REQ', 5) + elsif col.casecmp('LOCALITY').zero? + size_hash.store('LOCALITY', 10) + else + raise "#{col} is not a valid column. Possible values are SERVER_NAME, REGION_NAME, START_KEY, END_KEY, SIZE, REQ, LOCALITY." + end + end + else + raise "#{cols} must be an array of strings. Possible values are SERVER_NAME, REGION_NAME, START_KEY, END_KEY, SIZE, REQ, LOCALITY." + end + + admin_instance = admin.instance_variable_get('@admin') + conn_instance = admin_instance.getConnection + cluster_status = org.apache.hadoop.hbase.ClusterStatus.new(admin_instance.getClusterMetrics) + hregion_locator_instance = conn_instance.getRegionLocator(TableName.valueOf(table_name)) + hregion_locator_list = hregion_locator_instance.getAllRegionLocations.to_a + results = [] + desired_server_name = options[::HBaseConstants::SERVER_NAME] + + begin + # Filter out region servers which we don't want, default to all RS + regions = get_regions_for_table_and_server(table_name, conn_instance, desired_server_name) + # A locality threshold of "1.0" would be all regions (cannot have greater than 1 locality) + # Regions which have a `dataLocality` less-than-or-equal to this value are accepted + locality_threshold = 1.0 + if options.key? ::HBaseConstants::LOCALITY_THRESHOLD + value = options[::HBaseConstants::LOCALITY_THRESHOLD] + # Value validation. Must be a Float, and must be between [0, 1.0] + unless value.is_a? Float + raise "#{::HBaseConstants::LOCALITY_THRESHOLD} must be a float value" + end + unless valid_locality_threshold? value + raise "#{::HBaseConstants::LOCALITY_THRESHOLD} must be between 0 and 1.0, inclusive" + end + + locality_threshold = value + end + + regions.each do |hregion| + hregion_info = hregion.getRegion + server_name = hregion.getServerName + server_load = cluster_status.getLoad(server_name) + if server_load.nil? + region_load_map = java.util.HashMap.new + else + region_load_map = server_load.getRegionsLoad + end + region_name = hregion_info.getRegionNameAsString + region_load = region_load_map.get(hregion_info.getRegionName) + + if region_load.nil? + puts "Can not find all details for region: " \ + "#{region_name.strip} ," \ + " it may be disabled or in transition\n" + else + # Ignore regions which exceed our locality threshold + next unless accept_region_for_locality? region_load.getDataLocality, + locality_threshold + end + result_hash = {} + + if size_hash.key?('SERVER_NAME') + result_hash.store('SERVER_NAME', server_name.toString.strip) + size_hash['SERVER_NAME'] = [size_hash['SERVER_NAME'], server_name.toString.strip.length].max + end + + if size_hash.key?('REGION_NAME') + result_hash.store('REGION_NAME', region_name.strip) + size_hash['REGION_NAME'] = [size_hash['REGION_NAME'], region_name.length].max + end + + if size_hash.key?('START_KEY') + start_key = Bytes.toStringBinary(hregion_info.getStartKey).strip + result_hash.store('START_KEY', start_key) + size_hash['START_KEY'] = [size_hash['START_KEY'], start_key.length].max + end + + if size_hash.key?('END_KEY') + end_key = Bytes.toStringBinary(hregion_info.getEndKey).strip + result_hash.store('END_KEY', end_key) + size_hash['END_KEY'] = [size_hash['END_KEY'], end_key.length].max + end + + if size_hash.key?('SIZE') + if region_load.nil? + region_store_file_size = '' + else + region_store_file_size = region_load.getStorefileSizeMB.to_s.strip + end + result_hash.store('SIZE', region_store_file_size) + size_hash['SIZE'] = [size_hash['SIZE'], region_store_file_size.length].max + end + + if size_hash.key?('REQ') + if region_load.nil? + region_requests = '' + else + region_requests = region_load.getRequestsCount.to_s.strip + end + result_hash.store('REQ', region_requests) + size_hash['REQ'] = [size_hash['REQ'], region_requests.length].max + end + + if size_hash.key?('LOCALITY') + if region_load.nil? + locality = '' + else + locality = region_load.getDataLocality.to_s.strip + end + result_hash.store('LOCALITY', locality) + size_hash['LOCALITY'] = [size_hash['LOCALITY'], locality.length].max + end + + results << result_hash + end + ensure + hregion_locator_instance.close + end + + @end_time = Time.now + + size_hash.each do |param, length| + printf(" %#{length}s |", param) + end + printf("\n") + + size_hash.each_value do |length| + str = '-' * length + printf(" %#{length}s |", str) + end + printf("\n") + + results.each do |result| + size_hash.each do |param, length| + printf(" %#{length}s |", result[param]) + end + printf("\n") + end + + printf(" %d rows\n", results.size) + end + + def valid_locality_threshold?(value) + value >= 0 && value <= 1.0 + end + + def get_regions_for_table_and_server(table_name, conn, server_name) + get_regions_for_server(get_regions_for_table(table_name, conn), server_name) + end + + def get_regions_for_server(regions_for_table, server_name) + regions_for_table.select do |hregion| + actual_server_name = hregion.getServerName + if actual_server_name == nil + raise "Some regions might be splitting or merging or transitioning due to other" \ + " reasons" + end + accept_server_name? server_name, actual_server_name.toString + end + end + + def get_regions_for_table(table_name, conn) + conn.getRegionLocator(TableName.valueOf(table_name)).getAllRegionLocations.to_a + end + + def accept_server_name?(desired_server_name, actual_server_name) + desired_server_name.nil? || actual_server_name.start_with?(desired_server_name) + end + + def accept_region_for_locality?(actual_locality, locality_threshold) + actual_locality <= locality_threshold + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_replicated_tables.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_replicated_tables.rb new file mode 100644 index 0000000000..1fc6e76517 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_replicated_tables.rb @@ -0,0 +1,55 @@ +# +# Copyright The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ListReplicatedTables < Command + def help + <<-EOF +List all the tables and column families replicated from this cluster + + hbase> list_replicated_tables + hbase> list_replicated_tables 'abc.*' +EOF + end + + def command(regex = '.*') + formatter.header(['TABLE:COLUMNFAMILY', 'ReplicationType'], [32]) + list = replication_admin.list_replicated_tables(regex) + list.each do |e| + map = e.getColumnFamilyMap + map.each do |cf| + if cf[1] == org.apache.hadoop.hbase.HConstants::REPLICATION_SCOPE_LOCAL + replicateType = 'LOCAL' + elsif cf[1] == org.apache.hadoop.hbase.HConstants::REPLICATION_SCOPE_GLOBAL + replicateType = 'GLOBAL' + elsif cf[1] == org.apache.hadoop.hbase.HConstants::REPLICATION_SCOPE_SERIAL + replicateType = 'SERIAL' + else + replicateType = 'UNKNOWN' + end + formatter.row([e.getTable.getNameAsString + ':' + cf[0], replicateType], true, [32]) + end + end + formatter.footer + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_rsgroups.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_rsgroups.rb new file mode 100644 index 0000000000..dc33b851dc --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_rsgroups.rb @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ListRsgroups < Command + def help + <<-EOF +List all RegionServer groups. Optional regular expression parameter can +be used to filter the output. + +Example: + + hbase> list_rsgroups + hbase> list_rsgroups 'abc.*' + +EOF + end + + def command(regex = '.*') + formatter.header(['NAME', 'SERVER / TABLE']) + + regex = /#{regex}/ unless regex.is_a?(Regexp) + list = rsgroup_admin.list_rs_groups + groups = 0 + + list.each do |group| + next unless group.getName.match(regex) + + groups += 1 + group_name_printed = false + + group.getServers.each do |server| + if group_name_printed + group_name = '' + else + group_name = group.getName + group_name_printed = true + end + + formatter.row([group_name, 'server ' + server.toString]) + end + + group.getTables.each do |table| + if group_name_printed + group_name = '' + else + group_name = group.getName + group_name_printed = true + end + + formatter.row([group_name, 'table ' + table.getNameAsString]) + end + + formatter.row([group.getName, '']) unless group_name_printed + end + + formatter.footer(groups) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_security_capabilities.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_security_capabilities.rb new file mode 100644 index 0000000000..873a1f5af9 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_security_capabilities.rb @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ListSecurityCapabilities < Command + def help + <<-EOF +List supported security capabilities + +Example: + hbase> list_security_capabilities +EOF + end + + def command + list = admin.get_security_capabilities + list.each do |s| + puts s.getName + end + return list.map(&:getName) + rescue Exception => e + if e.to_s.include? 'UnsupportedOperationException' + puts 'ERROR: Master does not support getSecurityCapabilities' + return [] + end + raise e + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_snapshot_sizes.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_snapshot_sizes.rb new file mode 100644 index 0000000000..72439fe66f --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_snapshot_sizes.rb @@ -0,0 +1,42 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ListSnapshotSizes < Command + def help + return <<-EOF +Lists the size of every HBase snapshot given the space quota size computation +algorithms. An HBase snapshot only "owns" the size of a file when the table +from which the snapshot was created no longer refers to that file. +EOF + end + + def command(args = {}) + formatter.header(["SNAPSHOT", "SIZE"]) + count = 0 + quotas_admin.list_snapshot_sizes.each do |snapshot,size| + formatter.row([snapshot.to_s, size.to_s]) + count += 1 + end + formatter.footer(count) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_snapshots.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_snapshots.rb new file mode 100644 index 0000000000..1b3b8f26c3 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_snapshots.rb @@ -0,0 +1,50 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'time' + +module Shell + module Commands + class ListSnapshots < Command + def help + <<-EOF +List all snapshots taken (by printing the names and relative information). +Optional regular expression parameter could be used to filter the output +by snapshot name. + +Examples: + hbase> list_snapshots + hbase> list_snapshots 'abc.*' +EOF + end + + def command(regex = '.*') + formatter.header(['SNAPSHOT', 'TABLE + CREATION TIME']) + + list = admin.list_snapshot(regex) + list.each do |snapshot| + creation_time = Time.at(snapshot.getCreationTime / 1000).to_s + formatter.row([snapshot.getName, snapshot.getTable + ' (' + creation_time + ')']) + end + + formatter.footer(list.size) + list.map(&:getName) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_table_snapshots.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_table_snapshots.rb new file mode 100644 index 0000000000..558fda8ded --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_table_snapshots.rb @@ -0,0 +1,55 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'time' + +module Shell + module Commands + class ListTableSnapshots < Command + def help + <<-EOF +List all completed snapshots matching the table name regular expression and the +snapshot name regular expression (by printing the names and relative information). +Optional snapshot name regular expression parameter could be used to filter the output +by snapshot name. + +Examples: + hbase> list_table_snapshots 'tableName' + hbase> list_table_snapshots 'tableName.*' + hbase> list_table_snapshots 'tableName', 'snapshotName' + hbase> list_table_snapshots 'tableName', 'snapshotName.*' + hbase> list_table_snapshots 'tableName.*', 'snapshotName.*' + hbase> list_table_snapshots 'ns:tableName.*', 'snapshotName.*' +EOF + end + + def command(tableNameRegex, snapshotNameRegex = '.*') + formatter.header(['SNAPSHOT', 'TABLE + CREATION TIME']) + + list = admin.list_table_snapshots(tableNameRegex, snapshotNameRegex) + list.each do |snapshot| + creation_time = Time.at(snapshot.getCreationTime / 1000).to_s + formatter.row([snapshot.getName, snapshot.getTable + ' (' + creation_time + ')']) + end + + formatter.footer(list.size) + list.map(&:getName) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_unknownservers.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_unknownservers.rb new file mode 100644 index 0000000000..29f5e54acf --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/list_unknownservers.rb @@ -0,0 +1,44 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ListUnknownservers < Command + def help + <<~EOF + List all unknown region servers + Examples: + hbase> list_unknownservers + EOF + end + + def command + now = Time.now + formatter.header(['SERVERNAME']) + + servers = admin.list_unknownservers + servers.each do |server| + formatter.row([server.toString]) + end + + formatter.footer(servers.size) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/locate_region.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/locate_region.rb new file mode 100644 index 0000000000..9afbbec4cd --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/locate_region.rb @@ -0,0 +1,43 @@ +# +# Copyright The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class LocateRegion < Command + def help + <<-EOF +Locate the region given a table name and a row-key + + hbase> locate_region 'tableName', 'key0' +EOF + end + + def command(table, row_key) + region_location = admin.locate_region(table, row_key) + hri = region_location.getRegionInfo + + formatter.header(%w[HOST REGION]) + formatter.row([region_location.getHostnamePort, hri.toString]) + formatter.footer(1) + region_location + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/major_compact.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/major_compact.rb new file mode 100644 index 0000000000..75d2838d56 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/major_compact.rb @@ -0,0 +1,51 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class MajorCompact < Command + def help + <<~EOF + Run major compaction on passed table or pass a region row + to major compact an individual region. To compact a single + column family within a region specify the region name + followed by the column family name. + Examples: + Compact all regions in a table: + hbase> major_compact 't1' + hbase> major_compact 'ns1:t1' + Compact an entire region: + hbase> major_compact 'r1' + Compact a single column family within a region: + hbase> major_compact 'r1', 'c1' + Compact a single column family within a table: + hbase> major_compact 't1', 'c1' + Compact table with type "MOB" + hbase> major_compact 't1', nil, 'MOB' + Compact a column family using "MOB" type within a table + hbase> major_compact 't1', 'c1', 'MOB' + EOF + end + + def command(table_or_region_name, family = nil, type = 'NORMAL') + admin.major_compact(table_or_region_name, family, type) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/merge_region.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/merge_region.rb new file mode 100644 index 0000000000..ed17236277 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/merge_region.rb @@ -0,0 +1,73 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class MergeRegion < Command + def help + <<-EOF +Merge multiple (2 or more) regions. Passing 'true' as the optional third parameter will force +a merge ('force' merges regardless else merge will fail unless passed +adjacent regions. 'force' is for expert use only). + +You can pass the encoded region name or the full region name. The encoded +region name is the hash suffix on region names: e.g. if the region name were +TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396. then +the encoded region name portion is 527db22f95c8a9e0116f0cc13c680396 + +You can either pass the list of regions as comma separated values or as an +array of regions as shown: + +Examples: + + hbase> merge_region 'FULL_REGIONNAME', 'FULL_REGIONNAME' + hbase> merge_region 'FULL_REGIONNAME', 'FULL_REGIONNAME', 'FULL_REGIONNAME', ... + hbase> merge_region 'FULL_REGIONNAME', 'FULL_REGIONNAME', 'FULL_REGIONNAME', ..., true + + hbase> merge_region ['FULL_REGIONNAME', 'FULL_REGIONNAME'] + hbase> merge_region ['FULL_REGIONNAME', 'FULL_REGIONNAME', 'FULL_REGIONNAME', ...] + hbase> merge_region ['FULL_REGIONNAME', 'FULL_REGIONNAME', 'FULL_REGIONNAME', ...], true + + hbase> merge_region 'ENCODED_REGIONNAME', 'ENCODED_REGIONNAME' + hbase> merge_region 'ENCODED_REGIONNAME', 'ENCODED_REGIONNAME', 'ENCODED_REGIONNAME', ... + hbase> merge_region 'ENCODED_REGIONNAME', 'ENCODED_REGIONNAME', 'ENCODED_REGIONNAME', ..., true + + hbase> merge_region ['ENCODED_REGIONNAME', 'ENCODED_REGIONNAME'] + hbase> merge_region ['ENCODED_REGIONNAME', 'ENCODED_REGIONNAME', 'ENCODED_REGIONNAME', ...] + hbase> merge_region ['ENCODED_REGIONNAME', 'ENCODED_REGIONNAME', 'ENCODED_REGIONNAME', ...], true +EOF + end + + def command(*args) + args = args.flatten.compact + args_len = args.length + raise(ArgumentError, 'Must pass at least 2 regions to merge') unless args_len > 1 + force = false + if(args_len > 2) + last = args[args_len-1] + if [true, false].include? last + force = last + args = args[0...-1] + end + end + admin.merge_region(args, force) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/move.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/move.rb new file mode 100644 index 0000000000..3bb2ebc160 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/move.rb @@ -0,0 +1,45 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Move < Command + def help + <<-EOF +Move a region. Optionally specify target regionserver else we choose one +at random. NOTE: You pass the encoded region name, not the region name so +this command is a little different to the others. The encoded region name +is the hash suffix on region names: e.g. if the region name were +TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396. then +the encoded region name portion is 527db22f95c8a9e0116f0cc13c680396 +A server name is its host, port plus startcode. For example: +host187.example.com,60020,1289493121758 +Examples: + + hbase> move 'ENCODED_REGIONNAME' + hbase> move 'ENCODED_REGIONNAME', 'SERVER_NAME' +EOF + end + + def command(encoded_region_name, server_name = nil) + admin.move(encoded_region_name, server_name) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/move_namespaces_rsgroup.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/move_namespaces_rsgroup.rb new file mode 100644 index 0000000000..60b0bff62c --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/move_namespaces_rsgroup.rb @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + # Reassign tables of specified namespaces + # from one RegionServer group to another. + class MoveNamespacesRsgroup < Command + def help + <<~CMD + + Example: + hbase> move_namespaces_rsgroup 'dest',['ns1','ns2'] + +CMD + end + + def command(dest, namespaces) + rsgroup_admin.move_namespaces(dest, namespaces) + namespaces.each do |ns| + arg = {'METHOD' => 'set', 'hbase.rsgroup.name' => dest} + admin.alter_namespace(ns, arg) + end + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/move_servers_namespaces_rsgroup.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/move_servers_namespaces_rsgroup.rb new file mode 100644 index 0000000000..93db81a546 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/move_servers_namespaces_rsgroup.rb @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + # Reassign RegionServers and Tables of + # specified namespaces from one group to another. + class MoveServersNamespacesRsgroup < Command + def help + <<~CMD + + Example: + hbase> move_servers_namespaces_rsgroup 'dest',['server1:port','server2:port'],['ns1','ns2'] + +CMD + end + + def command(dest, servers, namespaces) + rsgroup_admin.move_servers_namespaces(dest, servers, namespaces) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/move_servers_rsgroup.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/move_servers_rsgroup.rb new file mode 100644 index 0000000000..5370f483ff --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/move_servers_rsgroup.rb @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class MoveServersRsgroup < Command + def help + <<-EOF +Reassign RegionServers from one group to another. Every region of the +RegionServer will be moved to another RegionServer. + +Example: + + hbase> move_servers_rsgroup 'dest',['server1:port','server2:port'] + +EOF + end + + def command(dest, servers) + rsgroup_admin.move_servers(dest, servers) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/move_servers_tables_rsgroup.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/move_servers_tables_rsgroup.rb new file mode 100644 index 0000000000..1789a3579e --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/move_servers_tables_rsgroup.rb @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class MoveServersTablesRsgroup < Command + def help + <<-EOF +Reassign RegionServers and Tables from one group to another. + +Example: + + hbase> move_servers_tables_rsgroup 'dest',['server1:port','server2:port'],['table1','table2'] + +EOF + end + + def command(dest, servers, tables) + rsgroup_admin.move_servers_tables(dest, servers, tables) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/move_tables_rsgroup.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/move_tables_rsgroup.rb new file mode 100644 index 0000000000..ff644d1cb4 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/move_tables_rsgroup.rb @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class MoveTablesRsgroup < Command + def help + <<-EOF +Reassign tables from one RegionServer group to another. + +Example: + + hbase> move_tables_rsgroup 'dest',['table1','table2'] + +EOF + end + + def command(dest, tables) + rsgroup_admin.move_tables(dest, tables) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/normalize.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/normalize.rb new file mode 100644 index 0000000000..70e524ae11 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/normalize.rb @@ -0,0 +1,48 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Normalize < Command + def help + <<-EOF +Trigger the region normalizer. Without arguments, invokes the normalizer without a table filter. +Any arguments are used to limit table selection. Returns true if the normalize request was +submitted successfully, false otherwise. Note that this command has no effect if region normalizer +is disabled (make sure it's turned on using 'normalizer_switch' command). + +Examples: + + hbase> normalize + hbase> normalize TABLE_NAME => 'my_table' + hbase> normalize TABLE_NAMES => ['foo', 'bar', 'baz'] + hbase> normalize REGEX => 'my_.*' + hbase> normalize NAMESPACE => 'ns1' + hbase> normalize NAMESPACE => 'ns', REGEX => '*._BIG_.*' +EOF + end + + def command(*args) + did_normalize_run = !!admin.normalize(*args) + formatter.row([did_normalize_run.to_s]) + did_normalize_run + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/normalizer_enabled.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/normalizer_enabled.rb new file mode 100644 index 0000000000..0bd2a79df2 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/normalizer_enabled.rb @@ -0,0 +1,39 @@ +#!/usr/bin/env hbase-jruby +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Prints current region normalizer status + +module Shell + module Commands + class NormalizerEnabled < Command + def help + <<-EOF +Query the state of region normalizer. +Examples: + + hbase> normalizer_enabled +EOF + end + + def command + current_state = admin.normalizer_enabled? + formatter.row([current_state.to_s]) + current_state + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/normalizer_switch.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/normalizer_switch.rb new file mode 100644 index 0000000000..36e730a9b0 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/normalizer_switch.rb @@ -0,0 +1,41 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class NormalizerSwitch < Command + def help + <<-EOF +Enable/Disable region normalizer. Returns previous normalizer state. +When normalizer is enabled, it handles all tables with 'NORMALIZATION_ENABLED' => true. +Examples: + + hbase> normalizer_switch true + hbase> normalizer_switch false +EOF + end + + def command(enableDisable) + previous_state = !!admin.normalizer_switch(enableDisable) + formatter.row([previous_state.to_s]) + previous_state + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/processlist.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/processlist.rb new file mode 100644 index 0000000000..4b61886d66 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/processlist.rb @@ -0,0 +1,66 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Processlist < Command + def help + <<-EOF +Show regionserver task list. + + hbase> processlist + hbase> processlist 'all' + # list non-RPC Tasks, such as compact, flush etc + hbase> processlist 'general' + # list RPC Handler Tasks + hbase> processlist 'handler' + # list RPC Handler Tasks which state is RUNNING + hbase> processlist 'rpc' + # list RPC Handler Tasks which state is RUNNING and from client + hbase> processlist 'operation' + hbase> processlist 'all','host187.example.com' + hbase> processlist 'all','host187.example.com,16020' + hbase> processlist 'all','host187.example.com,16020,1289493121758' + +EOF + end + + def command(*args) + if %w[all general handler rpc operation].include? args[0] + # if the first argument is a valid filter specifier, use it as such + filter = args[0] + hosts = args[1, args.length] + else + # otherwise, treat all arguments as host addresses by default + filter = 'general' + hosts = args + end + + hosts = admin.getServerNames(hosts, true) + + if hosts.nil? + puts 'No regionservers available.' + else + taskmonitor.tasks(filter, hosts) + end + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/put.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/put.rb new file mode 100644 index 0000000000..118902a400 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/put.rb @@ -0,0 +1,56 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Put < Command + def help + <<-EOF +Put a cell 'value' at specified table/row/column and optionally +timestamp coordinates. To put a cell value into table 'ns1:t1' or 't1' +at row 'r1' under column 'c1' marked with the time 'ts1', do: + + hbase> put 'ns1:t1', 'r1', 'c1', 'value' + hbase> put 't1', 'r1', 'c1', 'value' + hbase> put 't1', 'r1', 'c1', 'value', ts1 + hbase> put 't1', 'r1', 'c1', 'value', {ATTRIBUTES=>{'mykey'=>'myvalue'}} + hbase> put 't1', 'r1', 'c1', 'value', ts1, {ATTRIBUTES=>{'mykey'=>'myvalue'}} + hbase> put 't1', 'r1', 'c1', 'value', ts1, {VISIBILITY=>'PRIVATE|SECRET'} + +The same commands also can be run on a table reference. Suppose you had a reference +t to table 't1', the corresponding command would be: + + hbase> t.put 'r1', 'c1', 'value', ts1, {ATTRIBUTES=>{'mykey'=>'myvalue'}} +EOF + end + + def command(table, row, column, value, timestamp = nil, args = {}) + put table(table), row, column, value, timestamp, args + end + + def put(table, row, column, value, timestamp = nil, args = {}) + @start_time = Time.now + table._put_internal(row, column, value, timestamp, args) + end + end + end +end + +# Add the method table.put that calls Put.put +::Hbase::Table.add_shell_command('put') diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/recommission_regionserver.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/recommission_regionserver.rb new file mode 100644 index 0000000000..b112a8d498 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/recommission_regionserver.rb @@ -0,0 +1,44 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + # Recommission a region server, optionally load a list of passed regions + class RecommissionRegionserver < Command + def help + <<~EOF + Remove decommission marker from a region server to allow regions assignments. + + Optionally, load regions onto the server by passing a list of encoded region names. + NOTE: Region loading is asynchronous. + + Examples: + hbase> recommission_regionserver 'server' + hbase> recommission_regionserver 'server,port' + hbase> recommission_regionserver 'server,port,starttime' + hbase> recommission_regionserver 'server,port,starttime', ['encoded_region_name1', 'encoded_region_name1'] +EOF + end + + def command(server_name, encoded_region_names = []) + admin.recommission_regionserver(server_name, encoded_region_names) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/regioninfo.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/regioninfo.rb new file mode 100644 index 0000000000..28a13636b3 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/regioninfo.rb @@ -0,0 +1,43 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Regioninfo < Command + def help + <<-EOF +Return RegionInfo. Takes Region name or an encoded Region name +(Of use when all you have is an encoded Region name). + +Examples: +Below we pass first encoded region name and then full region name. + + hbase(main):002:0> regioninfo '1588230740' + {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} + hbase(main):002:0> regioninfo 'hbase:meta,,1' + {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} + +EOF + end + + def command(regionname) + admin.get_region_info(regionname) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/remove_peer.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/remove_peer.rb new file mode 100644 index 0000000000..26da40887a --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/remove_peer.rb @@ -0,0 +1,37 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class RemovePeer < Command + def help + <<-EOF +Stops the specified replication stream and deletes all the meta +information kept about it. Examples: + + hbase> remove_peer '1' +EOF + end + + def command(id) + replication_admin.remove_peer(id) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/remove_peer_exclude_namespaces.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/remove_peer_exclude_namespaces.rb new file mode 100644 index 0000000000..0298ce9b6d --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/remove_peer_exclude_namespaces.rb @@ -0,0 +1,45 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class RemovePeerExcludeNamespaces < Command + def help + <<-EOF +Remove the namespaces which not replicated for the specified peer. + +Note: + 1. The replicate_all flag need to be true when remove exclude namespaces. + 2. Remove a exclude namespace in the peer config means that all tables in this + namespace will be replicated to the peer cluster. + +Examples: + + # remove ns1 from the not replicable namespaces for peer '2'. + hbase> remove_peer_exclude_namespaces '2', ["ns1", "ns2"] + + EOF + end + + def command(id, namespaces) + replication_admin.remove_peer_exclude_namespaces(id, namespaces) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/remove_peer_exclude_tableCFs.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/remove_peer_exclude_tableCFs.rb new file mode 100644 index 0000000000..4f452072b3 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/remove_peer_exclude_tableCFs.rb @@ -0,0 +1,42 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class RemovePeerExcludeTableCFs < Command + def help + <<-EOF +Remove table-cfs config from the specified peer' exclude table-cfs to make them replicable +Examples: + + # remove tables / table-cfs from peer' exclude table-cfs + hbase> remove_peer_exclude_tableCFs '2', { "table1" => [], "ns2:table2" => ["cfA", "cfB"]} + EOF + end + + def command(id, table_cfs) + replication_admin.remove_peer_exclude_tableCFs(id, table_cfs) + end + + def command_name + 'remove_peer_exclude_tableCFs' + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/remove_peer_namespaces.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/remove_peer_namespaces.rb new file mode 100644 index 0000000000..fc771313ba --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/remove_peer_namespaces.rb @@ -0,0 +1,41 @@ +# +# Copyright The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class RemovePeerNamespaces < Command + def help + <<~EOF + Remove some namespaces from the namespaces config for the specified peer. + + Examples: + + # remove ns1 from the replicable namespaces for peer '2'. + hbase> remove_peer_namespaces '2', ["ns1"] + + EOF + end + + def command(id, namespaces) + replication_admin.remove_peer_namespaces(id, namespaces) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/remove_peer_tableCFs.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/remove_peer_tableCFs.rb new file mode 100644 index 0000000000..595cf32851 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/remove_peer_tableCFs.rb @@ -0,0 +1,44 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class RemovePeerTableCFs < Command + def help + <<-EOF +Remove a table / table-cf from the table-cfs config for the specified peer +Examples: + + # Remove a table / table-cf from the replicable table-cfs for a peer + hbase> remove_peer_tableCFs '2', { "ns1:table1" => []} + hbase> remove_peer_tableCFs '2', { "ns1:table1" => ["cf1"]} + +EOF + end + + def command(id, table_cfs) + replication_admin.remove_peer_tableCFs(id, table_cfs) + end + + def command_name + 'remove_peer_tableCFs' + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/remove_rsgroup.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/remove_rsgroup.rb new file mode 100644 index 0000000000..9d39aac9f5 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/remove_rsgroup.rb @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class RemoveRsgroup < Command + def help + <<-EOF +Remove a RegionServer group. + + hbase> remove_rsgroup 'my_group' + +EOF + end + + def command(group_name) + rsgroup_admin.remove_rs_group(group_name) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/remove_servers_rsgroup.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/remove_servers_rsgroup.rb new file mode 100644 index 0000000000..ba8e60c8d2 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/remove_servers_rsgroup.rb @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class RemoveServersRsgroup < Command + def help + <<-EOF +Remove decommissioned servers from rsgroup. +Dead/recovering/live servers will be disallowed. +Example: + hbase> remove_servers_rsgroup ['server1:port','server2:port'] +EOF + end + + def command(servers) + rsgroup_admin.remove_servers(servers) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/rename_rsgroup.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/rename_rsgroup.rb new file mode 100644 index 0000000000..5403c32ec2 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/rename_rsgroup.rb @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class RenameRsgroup < Command + def help + <<-EOF +Rename a RegionServer group. + + hbase> rename_rsgroup 'old_rsgroup_name', 'new_rsgroup_name' + +EOF + end + + def command(oldname, newname) + rsgroup_admin.rename_rsgroup(oldname, newname) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/restore_snapshot.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/restore_snapshot.rb new file mode 100644 index 0000000000..56168fa4a6 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/restore_snapshot.rb @@ -0,0 +1,45 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class RestoreSnapshot < Command + def help + <<-EOF +Restore a specified snapshot. +The restore will replace the content of the original table, +bringing back the content to the snapshot state. +The table must be disabled. + +Examples: + hbase> restore_snapshot 'snapshotName' + +Following command will restore all acl from snapshot table into the table. + + hbase> restore_snapshot 'snapshotName', {RESTORE_ACL=>true} +EOF + end + + def command(snapshot_name, args = {}) + raise(ArgumentError, 'Arguments should be a Hash') unless args.is_a?(Hash) + restore_acl = args.delete(::HBaseConstants::RESTORE_ACL) || false + admin.restore_snapshot(snapshot_name, restore_acl) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/revoke.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/revoke.rb new file mode 100644 index 0000000000..4742bd79f6 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/revoke.rb @@ -0,0 +1,47 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Revoke < Command + def help + <<-EOF +Revoke a user's access rights. +Syntax: revoke [,
[, [, ]]] +Syntax: revoke , <@namespace> + +Note: Groups and users access are revoked in the same way, but groups are prefixed with an '@' + character. Tables and namespaces are specified the same way, but namespaces are + prefixed with an '@' character. + +For example: + + hbase> revoke 'bobsmith' + hbase> revoke '@admins' + hbase> revoke 'bobsmith', '@ns1' + hbase> revoke 'bobsmith', 't1', 'f1', 'col1' + hbase> revoke 'bobsmith', 'ns1:t1', 'f1', 'col1' +EOF + end + + def command(user, table_name = nil, family = nil, qualifier = nil) + security_admin.revoke(user, table_name, family, qualifier) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/rit.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/rit.rb new file mode 100644 index 0000000000..cdb49cf1f3 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/rit.rb @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + # List all regions in transition + class Rit < Command + def help + <<-EOF +List all regions in transition. +Examples: + hbase> rit + EOF + end + + def command + rit = admin.getClusterStatus.getRegionStatesInTransition + rit.each do |v| + formatter.row([v.toDescriptiveString]) + end + formatter.footer(rit.size) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/scan.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/scan.rb new file mode 100644 index 0000000000..2dc377e402 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/scan.rb @@ -0,0 +1,142 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Scan < Command + def help + <<-EOF +Scan a table; pass table name and optionally a dictionary of scanner +specifications. Scanner specifications may include one or more of: +TIMERANGE, FILTER, LIMIT, STARTROW, STOPROW, ROWPREFIXFILTER, TIMESTAMP, +MAXLENGTH, COLUMNS, CACHE, RAW, VERSIONS, ALL_METRICS, METRICS, +REGION_REPLICA_ID, ISOLATION_LEVEL, READ_TYPE, ALLOW_PARTIAL_RESULTS, +BATCH or MAX_RESULT_SIZE + +If no columns are specified, all columns will be scanned. +To scan all members of a column family, leave the qualifier empty as in +'col_family'. + +The filter can be specified in two ways: +1. Using a filterString - more information on this is available in the +Filter Language document attached to the HBASE-4176 JIRA +2. Using the entire package name of the filter. + +If you wish to see metrics regarding the execution of the scan, the +ALL_METRICS boolean should be set to true. Alternatively, if you would +prefer to see only a subset of the metrics, the METRICS array can be +defined to include the names of only the metrics you care about. + +Some examples: + + hbase> scan 'hbase:meta' + hbase> scan 'hbase:meta', {COLUMNS => 'info:regioninfo'} + hbase> scan 'ns1:t1', {COLUMNS => ['c1', 'c2'], LIMIT => 10, STARTROW => 'xyz'} + hbase> scan 't1', {COLUMNS => ['c1', 'c2'], LIMIT => 10, STARTROW => 'xyz'} + hbase> scan 't1', {COLUMNS => 'c1', TIMERANGE => [1303668804000, 1303668904000]} + hbase> scan 't1', {REVERSED => true} + hbase> scan 't1', {ALL_METRICS => true} + hbase> scan 't1', {METRICS => ['RPC_RETRIES', 'ROWS_FILTERED']} + hbase> scan 't1', {ROWPREFIXFILTER => 'row2', FILTER => " + (QualifierFilter (>=, 'binary:xyz')) AND (TimestampsFilter ( 123, 456))"} + hbase> scan 't1', {FILTER => + org.apache.hadoop.hbase.filter.ColumnPaginationFilter.new(1, 0)} + hbase> scan 't1', {CONSISTENCY => 'TIMELINE'} + hbase> scan 't1', {ISOLATION_LEVEL => 'READ_UNCOMMITTED'} + hbase> scan 't1', {MAX_RESULT_SIZE => 123456} +For setting the Operation Attributes + hbase> scan 't1', { COLUMNS => ['c1', 'c2'], ATTRIBUTES => {'mykey' => 'myvalue'}} + hbase> scan 't1', { COLUMNS => ['c1', 'c2'], AUTHORIZATIONS => ['PRIVATE','SECRET']} +For experts, there is an additional option -- CACHE_BLOCKS -- which +switches block caching for the scanner on (true) or off (false). By +default it is enabled. Examples: + + hbase> scan 't1', {COLUMNS => ['c1', 'c2'], CACHE_BLOCKS => false} + +Also for experts, there is an advanced option -- RAW -- which instructs the +scanner to return all cells (including delete markers and uncollected deleted +cells). This option cannot be combined with requesting specific COLUMNS. +Disabled by default. Example: + + hbase> scan 't1', {RAW => true, VERSIONS => 10} + +There is yet another option -- READ_TYPE -- which instructs the scanner to +use a specific read type. Example: + + hbase> scan 't1', {READ_TYPE => 'PREAD'} + +Besides the default 'toStringBinary' format, 'scan' supports custom formatting +by column. A user can define a FORMATTER by adding it to the column name in +the scan specification. The FORMATTER can be stipulated: + + 1. either as a org.apache.hadoop.hbase.util.Bytes method name (e.g, toInt, toString) + 2. or as a custom class followed by method name: e.g. 'c(MyFormatterClass).format'. + +Example formatting cf:qualifier1 and cf:qualifier2 both as Integers: + hbase> scan 't1', {COLUMNS => ['cf:qualifier1:toInt', + 'cf:qualifier2:c(org.apache.hadoop.hbase.util.Bytes).toInt'] } + +Note that you can specify a FORMATTER by column only (cf:qualifier). You can set a +formatter for all columns (including, all key parts) using the "FORMATTER" +and "FORMATTER_CLASS" options. The default "FORMATTER_CLASS" is +"org.apache.hadoop.hbase.util.Bytes". + + hbase> scan 't1', {FORMATTER => 'toString'} + hbase> scan 't1', {FORMATTER_CLASS => 'org.apache.hadoop.hbase.util.Bytes', FORMATTER => 'toString'} + +Scan can also be used directly from a table, by first getting a reference to a +table, like such: + + hbase> t = get_table 't' + hbase> t.scan + +Note in the above situation, you can still provide all the filtering, columns, +options, etc as described above. + +EOF + end + + def command(table, args = {}) + scan(table(table), args) + end + + # internal command that actually does the scanning + def scan(table, args = {}) + formatter.header(['ROW', 'COLUMN+CELL']) + + scan = table._hash_to_scan(args) + # actually do the scanning + @start_time = Time.now + count, is_stale = table._scan_internal(args, scan) do |row, cells| + formatter.row([row, cells]) + end + @end_time = Time.now + + formatter.footer(count, is_stale) + # if scan metrics were enabled, print them after the results + if !scan.nil? && scan.isScanMetricsEnabled + formatter.scan_metrics(scan.getScanMetrics, args['METRICS']) + end + end + end + end +end + +# Add the method table.scan that calls Scan.scan +::Hbase::Table.add_shell_command('scan') diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/set_auths.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/set_auths.rb new file mode 100644 index 0000000000..ddd04aa6df --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/set_auths.rb @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class SetAuths < Command + def help + <<-EOF +Add a set of visibility labels for a user or group +Syntax : set_auths 'user',[label1, label2] + +For example: + + hbase> set_auths 'user1', ['SECRET','PRIVATE'] + hbase> set_auths '@group1', ['SECRET','PRIVATE'] +EOF + end + + def command(user, *args) + visibility_labels_admin.set_auths(user, args) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/set_peer_bandwidth.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/set_peer_bandwidth.rb new file mode 100644 index 0000000000..e932b9367d --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/set_peer_bandwidth.rb @@ -0,0 +1,42 @@ +# +# Copyright The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class SetPeerBandwidth < Command + def help + <<-EOF +Set the replication source per node bandwidth for the specified peer. +Examples: + + # set bandwidth=2MB per regionserver for a peer + hbase> set_peer_bandwidth '1', 2097152 + # unset bandwidth for a peer to use the default bandwidth configured in server-side + hbase> set_peer_bandwidth '1' + +EOF + end + + def command(id, bandwidth = 0) + replication_admin.set_peer_bandwidth(id, bandwidth) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/set_peer_exclude_namespaces.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/set_peer_exclude_namespaces.rb new file mode 100644 index 0000000000..bc8d867483 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/set_peer_exclude_namespaces.rb @@ -0,0 +1,52 @@ +# +# Copyright The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class SetPeerExcludeNamespaces < Command + def help + <<~EOF + Set the namespaces which not replicated for the specified peer. + + Note: + 1. The replicate_all flag need to be true when set exclude namespaces. + 2. Set a exclude namespace in the peer config means that all tables in this + namespace will not be replicated to the peer cluster. If peer config + already has a exclude table, then not allow set this table's namespace + as a exclude namespace. + + Examples: + + # set exclude namespaces config to null + hbase> set_peer_exclude_namespaces '1', [] + # set namespaces which not replicated for a peer. + # set a exclude namespace in the peer config means that all tables in this + # namespace will not be replicated. + hbase> set_peer_exclude_namespaces '2', ["ns1", "ns2"] + + EOF + end + + def command(id, exclude_namespaces) + replication_admin.set_peer_exclude_namespaces(id, exclude_namespaces) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/set_peer_exclude_tableCFs.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/set_peer_exclude_tableCFs.rb new file mode 100644 index 0000000000..4727790fb4 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/set_peer_exclude_tableCFs.rb @@ -0,0 +1,55 @@ +# +# Copyright The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class SetPeerExcludeTableCFs < Command + def help + <<~EOF + Set the table-cfs which not replicated for the specified peer. + + Note: + 1. The replicate_all flag need to be true when set exclude table-cfs. + 2. If peer config already has a exclude namespace, then not allow set any + exclude table of this namespace to the peer config. + + Examples: + + # set exclude table-cfs to null + hbase> set_peer_exclude_tableCFs '1' + # set table / table-cf which not replicated for a peer, for a table without + # an explicit column-family list, all column-families will not be replicated + hbase> set_peer_exclude_tableCFs '2', { "ns1:table1" => [], + "ns2:table2" => ["cf1", "cf2"], + "ns3:table3" => ["cfA", "cfB"]} + + EOF + end + + def command(id, exclude_peer_table_cfs = nil) + replication_admin.set_peer_exclude_tableCFs(id, exclude_peer_table_cfs) + end + + def command_name + 'set_peer_exclude_tableCFs' + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/set_peer_namespaces.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/set_peer_namespaces.rb new file mode 100644 index 0000000000..ee14a9ac8d --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/set_peer_namespaces.rb @@ -0,0 +1,51 @@ +# +# Copyright The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class SetPeerNamespaces < Command + def help + <<~EOF + Set the replicable namespaces config for the specified peer. + + 1. The replicate_all flag need to be false when set the replicable namespaces. + 2. Set a namespace in the peer config means that all tables in this namespace + will be replicated to the peer cluster. If peer config already has a table, + then not allow set this table's namespace to the peer config. + + Examples: + + # set namespaces config is null, then the table-cfs config decide + # which table to be replicated. + hbase> set_peer_namespaces '1', [] + # set namespaces to be replicable for a peer. + # set a namespace in the peer config means that all tables in this + # namespace (with replication_scope != 0 ) will be replicated. + hbase> set_peer_namespaces '2', ["ns1", "ns2"] + + EOF + end + + def command(id, namespaces) + replication_admin.set_peer_namespaces(id, namespaces) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/set_peer_replicate_all.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/set_peer_replicate_all.rb new file mode 100644 index 0000000000..87ffa02fcd --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/set_peer_replicate_all.rb @@ -0,0 +1,60 @@ +# +# Copyright The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class SetPeerReplicateAll < Command + def help + <<~EOF + Set the replicate_all flag to true or false for the specified peer. + + If replicate_all flag is true, then all user tables (REPLICATION_SCOPE != 0) + will be replicate to peer cluster. But you can use 'set_peer_exclude_namespaces' + to set which namespaces can't be replicated to peer cluster. And you can use + 'set_peer_exclude_tableCFs' to set which tables can't be replicated to peer + cluster. + + If replicate_all flag is false, then all user tables cannot be replicate to + peer cluster. Then you can use 'set_peer_namespaces' or 'append_peer_namespaces' + to set which namespaces will be replicated to peer cluster. And you can use + 'set_peer_tableCFs' or 'append_peer_tableCFs' to set which tables will be + replicated to peer cluster. + + Notice: When you want to change a peer's replicate_all flag from false to true, + you need clean the peer's NAMESPACES and TABLECFS config firstly. + When you want to change a peer's replicate_all flag from true to false, + you need clean the peer's EXCLUDE_NAMESPACES and EXCLUDE_TABLECFS + config firstly. + + Examples: + + # set replicate_all flag to true + hbase> set_peer_replicate_all '1', true + # set replicate_all flag to false + hbase> set_peer_replicate_all '1', false +EOF + end + + def command(id, replicate_all) + replication_admin.set_peer_replicate_all(id, replicate_all) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/set_peer_serial.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/set_peer_serial.rb new file mode 100644 index 0000000000..cb18182dd3 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/set_peer_serial.rb @@ -0,0 +1,49 @@ +# +# Copyright The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class SetPeerSerial < Command + def help + <<~EOF + Set the serial flag to true or false for the specified peer. + + If serial flag is true, then all logs of user tables (REPLICATION_SCOPE != 0) will be + replicated to peer cluster serially, which means that each segment of log for replicated + table will be pushed to peer cluster in order of their log sequence id. + + If serial flag is false, then the source cluster won't ensure that the logs of replicated + table will be pushed to peer cluster serially. + + Examples: + + # set serial flag to true + hbase> set_peer_serial '1', true + # set serial flag to false + hbase> set_peer_serial '1', false + EOF + end + + def command(id, serial) + replication_admin.set_peer_serial(id, serial) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/set_peer_tableCFs.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/set_peer_tableCFs.rb new file mode 100644 index 0000000000..73d57edc34 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/set_peer_tableCFs.rb @@ -0,0 +1,54 @@ +# +# Copyright The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class SetPeerTableCFs < Command + def help + <<~EOF + Set the replicable table-cf config for the specified peer. + + Note: + 1. The replicate_all flag need to be false when set the replicable table-cfs. + 2. Can't set a table to table-cfs config if it's namespace already was in + namespaces config of this peer. + + Examples: + + # set table-cfs config is null, then the namespaces config decide which + # table to be replicated. + hbase> set_peer_tableCFs '1' + # set table / table-cf to be replicable for a peer, for a table without + # an explicit column-family list, all replicable column-families (with + # replication_scope == 1) will be replicated + hbase> set_peer_tableCFs '2', + { "ns1:table1" => [], + "ns2:table2" => ["cf1", "cf2"], + "ns3:table3" => ["cfA", "cfB"]} + + EOF + end + + def command(id, peer_table_cfs = nil) + replication_admin.set_peer_tableCFs(id, peer_table_cfs) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/set_quota.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/set_quota.rb new file mode 100644 index 0000000000..5a5669a9cc --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/set_quota.rb @@ -0,0 +1,162 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class SetQuota < Command + def help + <<-EOF +Set a quota for a user, table, namespace or region server. +Syntax : set_quota TYPE => , + +TYPE => THROTTLE +1. User can set throttle quota for user, namespace, table, region server, user over namespace, + user over table by USER, NAMESPACE, TABLE, REGIONSERVER keys. + Note: Setting specified region server quota isn't supported currently and using 'all' to + represent all region servers. +2. User can set throttle quota type either on read, write or on both the requests together( + read+write, default throttle type) by THROTTLE_TYPE => READ, WRITE, REQUEST. +3. The request limit can be expressed using the form 100req/sec, 100req/min; or can be expressed + using the form 100k/sec, 100M/min with (B, K, M, G, T, P) as valid size unit; or can be expressed + using the form 100CU/sec as capacity unit by LIMIT key. + The valid time units are (sec, min, hour, day). +4. User can set throttle scope to be either MACHINE(default throttle scope) or CLUSTER by + SCOPE => MACHINE, CLUSTER. MACHINE scope quota means the throttle limit is used by single + region server, CLUSTER scope quota means the throttle limit is shared by all region servers. + Region server throttle quota must be MACHINE scope. + Note: because currently use [ClusterLimit / RsNum] to divide cluster limit to machine limit, + so it's better to do not use cluster scope quota when you use rs group feature. + +For example: + + hbase> set_quota TYPE => THROTTLE, USER => 'u1', LIMIT => '10req/sec' + hbase> set_quota TYPE => THROTTLE, THROTTLE_TYPE => READ, USER => 'u1', LIMIT => '10req/sec' + Unthrottle number of requests: + hbase> set_quota TYPE => THROTTLE, THROTTLE_TYPE => REQUEST_NUMBER, USER => 'u1', LIMIT => 'NONE' + Unthrottle number of read requests: + hbase> set_quota TYPE => THROTTLE, THROTTLE_TYPE => READ_NUMBER, USER => 'u1', LIMIT => NONE + Unthrottle number of write requests: + hbase> set_quota TYPE => THROTTLE, THROTTLE_TYPE => WRITE_NUMBER, USER => 'u1', LIMIT => NONE + + hbase> set_quota TYPE => THROTTLE, USER => 'u1', LIMIT => '10M/sec' + hbase> set_quota TYPE => THROTTLE, THROTTLE_TYPE => WRITE, USER => 'u1', LIMIT => '10M/sec' + Unthrottle data size: + hbase> set_quota TYPE => THROTTLE, THROTTLE_TYPE => REQUEST_SIZE, USER => 'u1', LIMIT => 'NONE' + Unthrottle read data size: + hbase> set_quota TYPE => THROTTLE, USER => 'u1', THROTTLE_TYPE => READ_SIZE, LIMIT => 'NONE' + Unthrottle write data size: + hbase> set_quota TYPE => THROTTLE, USER => 'u1', THROTTLE_TYPE => WRITE_SIZE, LIMIT => 'NONE' + + hbase> set_quota TYPE => THROTTLE, USER => 'u1', LIMIT => '10CU/sec' + hbase> set_quota TYPE => THROTTLE, THROTTLE_TYPE => WRITE, USER => 'u1', LIMIT => '10CU/sec' + Unthrottle capacity unit: + hbase> set_quota TYPE => THROTTLE, THROTTLE_TYPE => REQUEST_CAPACITY_UNIT, USER => 'u1', LIMIT => 'NONE' + Unthrottle read capacity unit: + hbase> set_quota TYPE => THROTTLE, THROTTLE_TYPE => READ_CAPACITY_UNIT, USER => 'u1', LIMIT => 'NONE' + Unthrottle write capacity unit: + hbase> set_quota TYPE => THROTTLE, THROTTLE_TYPE => WRITE_CAPACITY_UNIT, USER => 'u1', LIMIT => 'NONE' + + hbase> set_quota TYPE => THROTTLE, USER => 'u1', TABLE => 't2', LIMIT => '5K/min' + hbase> set_quota TYPE => THROTTLE, USER => 'u1', NAMESPACE => 'ns2', LIMIT => NONE + + hbase> set_quota TYPE => THROTTLE, NAMESPACE => 'ns1', LIMIT => '10req/sec' + hbase> set_quota TYPE => THROTTLE, TABLE => 't1', LIMIT => '10M/sec' + hbase> set_quota TYPE => THROTTLE, THROTTLE_TYPE => WRITE, TABLE => 't1', LIMIT => '10M/sec' + hbase> set_quota TYPE => THROTTLE, USER => 'u1', LIMIT => NONE + + hbase> set_quota TYPE => THROTTLE, REGIONSERVER => 'all', LIMIT => '30000req/sec' + hbase> set_quota TYPE => THROTTLE, REGIONSERVER => 'all', THROTTLE_TYPE => WRITE, LIMIT => '20000req/sec' + hbase> set_quota TYPE => THROTTLE, REGIONSERVER => 'all', LIMIT => NONE + + hbase> set_quota TYPE => THROTTLE, NAMESPACE => 'ns1', LIMIT => '10req/sec', SCOPE => CLUSTER + hbase> set_quota TYPE => THROTTLE, NAMESPACE => 'ns1', LIMIT => '10req/sec', SCOPE => MACHINE + + hbase> set_quota USER => 'u1', GLOBAL_BYPASS => true + +TYPE => SPACE +Users can either set a quota on a table or a namespace. The quota is a limit on the target's +size on the FileSystem and some action to take when the target exceeds that limit. The limit +is in bytes and can expressed using standard metric suffixes (B, K, M, G, T, P), defaulting +to bytes if not provided. Different quotas can be applied to one table at the table and namespace +level; table-level quotas take priority over namespace-level quotas. + +There are a limited number of policies to take when a quota is violation, listed in order of +least strict to most strict. + + NO_INSERTS - No new data is allowed to be ingested (e.g. Put, Increment, Append). + NO_WRITES - Same as NO_INSERTS but Deletes are also disallowed. + NO_WRITES_COMPACTIONS - Same as NO_WRITES but compactions are also disallowed. + DISABLE - The table(s) are disabled. + +For example: + + hbase> set_quota TYPE => SPACE, TABLE => 't1', LIMIT => '1G', POLICY => NO_INSERTS + hbase> set_quota TYPE => SPACE, TABLE => 't2', LIMIT => '50G', POLICY => DISABLE + hbase> set_quota TYPE => SPACE, TABLE => 't3', LIMIT => '2T', POLICY => NO_WRITES_COMPACTIONS + hbase> set_quota TYPE => SPACE, NAMESPACE => 'ns1', LIMIT => '50T', POLICY => NO_WRITES + +Space quotas can also be removed via this command. To remove a space quota, provide NONE +for the limit. + +For example: + + hbase> set_quota TYPE => SPACE, TABLE => 't1', LIMIT => NONE + hbase> set_quota TYPE => SPACE, NAMESPACE => 'ns1', LIMIT => NONE + +EOF + end + + def command(args = {}) + if args.key?(::HBaseConstants::TYPE) + qtype = args.delete(::HBaseConstants::TYPE) + case qtype + when ::HBaseQuotasConstants::THROTTLE + if args[::HBaseConstants::LIMIT].eql? ::HBaseConstants::NONE + args.delete(::HBaseConstants::LIMIT) + quotas_admin.unthrottle(args) + else + quotas_admin.throttle(args) + end + when ::HBaseQuotasConstants::SPACE + if args[::HBaseConstants::LIMIT].eql? ::HBaseConstants::NONE + args.delete(::HBaseConstants::LIMIT) + # Table/Namespace argument is verified in remove_space_limit + quotas_admin.remove_space_limit(args) + else + unless args.key?(::HBaseConstants::LIMIT) + raise(ArgumentError, 'Expected a LIMIT to be provided') + end + unless args.key?(::HBaseConstants::POLICY) + raise(ArgumentError, 'Expected a POLICY to be provided') + end + + quotas_admin.limit_space(args) + end + else + raise 'Invalid TYPE argument. got ' + qtype + end + elsif args.key?(::HBaseQuotasConstants::GLOBAL_BYPASS) + quotas_admin.set_global_bypass(args.delete(::HBaseQuotasConstants::GLOBAL_BYPASS), args) + else + raise 'Expected TYPE argument' + end + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/set_visibility.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/set_visibility.rb new file mode 100644 index 0000000000..5c57d9c528 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/set_visibility.rb @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class SetVisibility < Command + def help + <<-EOF +Set the visibility expression on one or more existing cells. + +Pass table name, visibility expression, and a dictionary containing +scanner specifications. Scanner specifications may include one or more +of: TIMERANGE, FILTER, STARTROW, STOPROW, ROWPREFIXFILTER, TIMESTAMP, or COLUMNS + +If no columns are specified, all columns will be included. +To include all members of a column family, leave the qualifier empty as in +'col_family:'. + +The filter can be specified in two ways: +1. Using a filterString - more information on this is available in the +Filter Language document attached to the HBASE-4176 JIRA +2. Using the entire package name of the filter. + +Examples: + + hbase> set_visibility 't1', 'A|B', {COLUMNS => ['c1', 'c2']} + hbase> set_visibility 't1', '(A&B)|C', {COLUMNS => 'c1', + TIMERANGE => [1303668804000, 1303668904000]} + hbase> set_visibility 't1', 'A&B&C', {ROWPREFIXFILTER => 'row2', + FILTER => "(QualifierFilter (>=, 'binary:xyz')) AND + (TimestampsFilter ( 123, 456))"} + +This command will only affect existing cells and is expected to be mainly +useful for feature testing and functional verification. +EOF + end + + def command(table, visibility, scan) + t = table(table) + @start_time = Time.now + scanner = t._get_scanner(scan) + count = 0 + iter = scanner.iterator + while iter.hasNext + row = iter.next + row.listCells.each do |cell| + put = org.apache.hadoop.hbase.client.Put.new(row.getRow) + put.add(cell) + t.set_cell_visibility(put, visibility) + t.table.put(put) + end + count += 1 + end + formatter.footer(count) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/show_filters.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/show_filters.rb new file mode 100644 index 0000000000..0017ed0e26 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/show_filters.rb @@ -0,0 +1,48 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +java_import org.apache.hadoop.hbase.filter.ParseFilter + +module Shell + module Commands + class ShowFilters < Command + def help + <<-EOF +Show all the filters in hbase. Example: + hbase> show_filters + + ColumnPrefixFilter + TimestampsFilter + PageFilter + ..... + KeyOnlyFilter +EOF + end + + def command + parseFilter = ParseFilter.new + supportedFilters = parseFilter.getSupportedFilters + + supportedFilters.each do |filter| + formatter.row([filter]) + end + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/show_peer_tableCFs.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/show_peer_tableCFs.rb new file mode 100644 index 0000000000..48197a7ff1 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/show_peer_tableCFs.rb @@ -0,0 +1,43 @@ +# +# Copyright The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ShowPeerTableCFs < Command + def help + <<~EOF + Show replicable table-cf config for the specified peer. + + hbase> show_peer_tableCFs '2' + EOF + end + + def command(id) + peer_table_cfs = replication_admin.show_peer_tableCFs(id) + puts peer_table_cfs + peer_table_cfs + end + + def command_name + 'show_peer_tableCFs' + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/show_rsgroup_config.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/show_rsgroup_config.rb new file mode 100644 index 0000000000..54344a1707 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/show_rsgroup_config.rb @@ -0,0 +1,41 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ShowRsgroupConfig < Command + def help + <<-EOF +Show the configuration of a special RSGroup. +Example: + hbase> show_rsgroup_config 'group' +EOF + end + + def command(group) + formatter.header(%w['KEY' 'VALUE']) + config = rsgroup_admin.get_rsgroup(group).getConfiguration + config.each { |key, val| + formatter.row([key, val]) + } + formatter.footer(config.size) + end + end + end +end \ No newline at end of file diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/snapshot.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/snapshot.rb new file mode 100644 index 0000000000..9984494708 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/snapshot.rb @@ -0,0 +1,36 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Snapshot < Command + def help + <<-EOF +Take a snapshot of specified table. Examples: + + hbase> snapshot 'sourceTable', 'snapshotName' + hbase> snapshot 'namespace:sourceTable', 'snapshotName', {SKIP_FLUSH => true, MAX_FILESIZE => 21474836480} +EOF + end + + def command(table, snapshot_name, *args) + admin.snapshot(table, snapshot_name, *args) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/snapshot_cleanup_enabled.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/snapshot_cleanup_enabled.rb new file mode 100644 index 0000000000..15122f5c7d --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/snapshot_cleanup_enabled.rb @@ -0,0 +1,39 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Prints if snapshot auto cleanup based on TTL is enabled + +module Shell + module Commands + class SnapshotCleanupEnabled < Command + def help + <<-EOF +Query the snapshot auto-cleanup state. +Examples: + + hbase> snapshot_cleanup_enabled + EOF + end + + def command + state = admin.snapshot_cleanup_enabled? + formatter.row([state.to_s]) + state + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/snapshot_cleanup_switch.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/snapshot_cleanup_switch.rb new file mode 100644 index 0000000000..1500af01e0 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/snapshot_cleanup_switch.rb @@ -0,0 +1,43 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Switch snapshot auto-cleanup based on TTL expiration + +module Shell + module Commands + class SnapshotCleanupSwitch < Command + def help + <<-EOF +Enable/Disable snapshot auto-cleanup based on snapshot TTL. +Returns previous snapshot auto-cleanup switch state. +Examples: + + hbase> snapshot_cleanup_switch true + hbase> snapshot_cleanup_switch false + EOF + end + + def command(enable_disable) + prev_state = !!admin.snapshot_cleanup_switch(enable_disable) + formatter.row(["Previous snapshot cleanup state : #{prev_state}"]) + prev_state + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/split.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/split.rb new file mode 100644 index 0000000000..ede69873ae --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/split.rb @@ -0,0 +1,42 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Split < Command + def help + <<-EOF +Split entire table or pass a region to split individual region. With the +second parameter, you can specify an explicit split key for the region. +Examples: + split 'TABLENAME' + split 'REGIONNAME' + split 'ENCODED_REGIONNAME' + split 'TABLENAME', 'splitKey' + split 'REGIONNAME', 'splitKey' + split 'ENCODED_REGIONNAME', 'splitKey' +EOF + end + + def command(table_or_region_name, split_point = nil) + admin.split(table_or_region_name, split_point) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/splitormerge_enabled.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/splitormerge_enabled.rb new file mode 100644 index 0000000000..be5d3db0f3 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/splitormerge_enabled.rb @@ -0,0 +1,39 @@ +#!/usr/bin/env hbase-jruby +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Prints the current split or merge status +module Shell + module Commands + # Command for check split or merge switch status + class SplitormergeEnabled < Command + def help + <<-EOF +Query the switch's state. You can set switch type, 'SPLIT' or 'MERGE' +Examples: + + hbase> splitormerge_enabled 'SPLIT' + EOF + end + + def command(switch_type) + current_state = !!admin.splitormerge_enabled(switch_type) + formatter.row([current_state.to_s]) + current_state + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/splitormerge_switch.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/splitormerge_switch.rb new file mode 100644 index 0000000000..22888f2987 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/splitormerge_switch.rb @@ -0,0 +1,43 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + # Command for set switch for split and merge + class SplitormergeSwitch < Command + def help + <<-EOF +Enable/Disable one switch. You can set switch type 'SPLIT' or 'MERGE'. Returns previous split state. +Examples: + + hbase> splitormerge_switch 'SPLIT', true + hbase> splitormerge_switch 'SPLIT', false + EOF + end + + def command(switch_type, enabled) + previous_state = !!admin.splitormerge_switch(switch_type, enabled) + formatter.row( + [previous_state.to_s] + ) + previous_state + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/status.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/status.rb new file mode 100644 index 0000000000..eac49a9056 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/status.rb @@ -0,0 +1,44 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Status < Command + def help + <<-EOF +Show cluster status. Can be 'summary', 'simple', 'detailed', 'tasks', or 'replication'. The +default is 'summary'. Examples: + + hbase> status + hbase> status 'summary' + hbase> status 'simple' + hbase> status 'detailed' + hbase> status 'tasks' + hbase> status 'replication' + hbase> status 'replication', 'source' + hbase> status 'replication', 'sink' +EOF + end + + def command(format = 'summary', type = 'both') + admin.status(format, type) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/stop_master.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/stop_master.rb new file mode 100644 index 0000000000..45a2b1121a --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/stop_master.rb @@ -0,0 +1,38 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + # Stop active Master + class StopMaster < Command + def help + <<-EOF +Stop active Master. For experts only. +Examples: + + hbase> stop_master +EOF + end + + def command + admin.stop_master + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/stop_regionserver.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/stop_regionserver.rb new file mode 100644 index 0000000000..127088d436 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/stop_regionserver.rb @@ -0,0 +1,40 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + # Stop a RegionServer + class StopRegionserver < Command + def help + <<-EOF +Stop a RegionServer. For experts only. +Consider using graceful_stop.sh script instead! + +Examples: + + hbase> stop_regionserver 'hostname:port' +EOF + end + + def command(hostport) + admin.stop_regionserver(hostport) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/table_help.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/table_help.rb new file mode 100644 index 0000000000..0ffed753ab --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/table_help.rb @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class TableHelp < Command + def help + Hbase::Table.help + end + + # just print the help + def command + # call the shell to get the nice formatting there + @shell.help_command 'table_help' + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/trace.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/trace.rb new file mode 100644 index 0000000000..90c11ae5e8 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/trace.rb @@ -0,0 +1,76 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Trace < Command + @@tracer = org.apache.hadoop.hbase.trace.TraceUtil.getGlobalTracer() + @@tracespan = nil + @@tracescope = nil + + def help + <<-EOF +Start or Stop tracing using OpenTelemetry. +Always returns true if tracing is running, otherwise false. +If the first argument is 'start', new span is started. +If the first argument is 'stop', current running span is stopped. +('stop' returns false on success.) +If the first argument is 'status', just returns if or not tracing is running. +On 'start'-ing, you can optionally pass the name of span as the second argument. +The default name of span is 'HBaseShell'. +Repeating 'start' does not start nested span. + +Examples: + + hbase> trace 'start' + hbase> trace 'status' + hbase> trace 'stop' + + hbase> trace 'start', 'MySpanName' + hbase> trace 'stop' + +EOF + end + + def command(startstop = 'status', spanname = 'HBaseShell') + trace(startstop, spanname) + end + + def trace(startstop, spanname) + if startstop == 'start' + unless tracing? + @@tracespan = @@tracer.spanBuilder(spanname).startSpan() + @@tracescope = @@tracespan.makeCurrent() + end + elsif startstop == 'stop' + if tracing? + @@tracescope.close() + @@tracespan.end() + @@tracescope = nil + end + end + tracing? + end + + def tracing? + @@tracescope != nil + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/truncate.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/truncate.rb new file mode 100644 index 0000000000..0a061fbf78 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/truncate.rb @@ -0,0 +1,37 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Truncate < Command + def help + <<~EOF + Disables, drops and recreates the specified table. + + Example: + hbase> truncate 'my_table' +EOF + end + + def command(table) + admin.truncate(table) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/truncate_preserve.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/truncate_preserve.rb new file mode 100644 index 0000000000..2195951223 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/truncate_preserve.rb @@ -0,0 +1,37 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class TruncatePreserve < Command + def help + <<~EOF + Disables, drops and recreates the specified table while still maintaining the previous region boundaries. + + Example: + hbase> truncate_preserve 'my_table' +EOF + end + + def command(table) + admin.truncate_preserve(table) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/unassign.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/unassign.rb new file mode 100644 index 0000000000..8bd74e8274 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/unassign.rb @@ -0,0 +1,42 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Unassign < Command + def help + <<-EOF +Unassign a region. It could be executed only when region in expected state(OPEN). +In addition, you can use "unassigns" command available on HBCK2 tool to skip the state check. +(For more info on HBCK2: https://github.com/apache/hbase-operator-tools/blob/master/hbase-hbck2/README.md) +Use with caution. For experts only. +Examples: + + hbase> unassign 'REGIONNAME' + hbase> unassign 'ENCODED_REGIONNAME' +EOF + end + + # the force parameter is deprecated, if it is specified, will be ignored. + def command(region_name, force = nil) + admin.unassign(region_name) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/update_all_config.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/update_all_config.rb new file mode 100644 index 0000000000..a3907a17f0 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/update_all_config.rb @@ -0,0 +1,37 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class UpdateAllConfig < Command + def help + <<-EOF +Reload a subset of configuration on all servers in the cluster. See +http://hbase.apache.org/book.html#dyn_config for more details. Here is how +you would run the command in the hbase shell: + hbase> update_all_config +EOF + end + + def command + admin.update_all_config + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/update_config.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/update_config.rb new file mode 100644 index 0000000000..e492db56f7 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/update_config.rb @@ -0,0 +1,38 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class UpdateConfig < Command + def help + <<-EOF +Reload a subset of configuration on server 'servername' where servername is +host, port plus startcode. For example: host187.example.com,60020,1289493121758 +See http://hbase.apache.org/book.html#dyn_config for more details. Here is how +you would run the command in the hbase shell: + hbase> update_config 'servername' +EOF + end + + def command(serverName) + admin.update_config(serverName) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/update_peer_config.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/update_peer_config.rb new file mode 100644 index 0000000000..cdded82b0c --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/update_peer_config.rb @@ -0,0 +1,47 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class UpdatePeerConfig < Command + def help + <<-EOF +A peer can either be another HBase cluster or a custom replication endpoint. In either case an id +must be specified to identify the peer. This command does not interrupt processing on an enabled replication peer. + +Two optional arguments are DATA and CONFIG which can be specified to set different values for either +the peer_data or configuration for a custom replication endpoint. Any existing values not updated by this command +are left unchanged. + +CLUSTER_KEY, REPLICATION_ENDPOINT, and TABLE_CFs cannot be updated with this command. +To update TABLE_CFs, see the append_peer_tableCFs and remove_peer_tableCFs commands. + + hbase> update_peer_config '1', DATA => { "key1" => 1 } + hbase> update_peer_config '2', CONFIG => { "config1" => "value1", "config2" => "value2" } + hbase> update_peer_config '3', DATA => { "key1" => 1 }, CONFIG => { "config1" => "value1", "config2" => "value2" }, + + EOF + end + + def command(id, args = {}) + replication_admin.update_peer_config(id, args) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/update_rsgroup_config.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/update_rsgroup_config.rb new file mode 100644 index 0000000000..4587a71ae8 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/update_rsgroup_config.rb @@ -0,0 +1,37 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class UpdateRsgroupConfig < Command + def help + <<-EOF +Reload a subset of configuration on all servers in the rsgroup. See +http://hbase.apache.org/book.html#dyn_config for more details. Here is how +you would run the command in the hbase shell: + hbase> update_rsgroup_config 'groupName' +EOF + end + + def command(groupName) + rsgroup_admin.update_rsgroup_config(groupName) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/user_permission.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/user_permission.rb new file mode 100644 index 0000000000..ba40c6c23d --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/user_permission.rb @@ -0,0 +1,54 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class UserPermission < Command + def help + <<-EOF +Show all permissions for the particular user. +Syntax : user_permission
+ +Note: A namespace must always precede with '@' character. + +For example: + + hbase> user_permission + hbase> user_permission '@ns1' + hbase> user_permission '@.*' + hbase> user_permission '@^[a-c].*' + hbase> user_permission 'table1' + hbase> user_permission 'namespace1:table1' + hbase> user_permission '.*' + hbase> user_permission '^[A-C].*' +EOF + end + + def command(table_regex = nil) + # admin.user_permission(table_regex) + formatter.header(['User', 'Namespace,Table,Family,Qualifier:Permission']) + + count = security_admin.user_permission(table_regex) do |user, permission| + formatter.row([user, permission]) + end + + formatter.footer(count) + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/version.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/version.rb new file mode 100644 index 0000000000..788501a55a --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/version.rb @@ -0,0 +1,37 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Version < Command + def help + <<-EOF +Output this HBase version +EOF + end + + def command + # Output version. + puts "#{org.apache.hadoop.hbase.util.VersionInfo.getVersion}, " \ + "r#{org.apache.hadoop.hbase.util.VersionInfo.getRevision}, " \ + "#{org.apache.hadoop.hbase.util.VersionInfo.getDate}" + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/wal_roll.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/wal_roll.rb new file mode 100644 index 0000000000..8ee1abc436 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/wal_roll.rb @@ -0,0 +1,41 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +module Shell + module Commands + class WalRoll < Command + def help + <<-EOF +Roll the log writer. That is, start writing log messages to a new file. +The name of the regionserver should be given as the parameter. A +'server_name' is the host, port plus startcode of a regionserver. For +example: host187.example.com,60020,1289493121758 (find servername in +master ui or when you do detailed status in shell) +EOF + end + + def command(server_name) + admin.wal_roll(server_name) + end + end + + # TODO: remove old HLog version + class HlogRoll < WalRoll + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/whoami.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/whoami.rb new file mode 100644 index 0000000000..aac4e3bae9 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/whoami.rb @@ -0,0 +1,42 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Whoami < Command + def help + <<-EOF +Show the current hbase user. +Syntax : whoami +For example: + + hbase> whoami +EOF + end + + def command + user = org.apache.hadoop.hbase.security.User.getCurrent + puts user.toString.to_s + groups = user.getGroupNames.to_a + if !groups.nil? && !groups.empty? + puts " groups: #{groups.join(', ')}" + end + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/zk_dump.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/zk_dump.rb new file mode 100644 index 0000000000..fe89014a97 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/commands/zk_dump.rb @@ -0,0 +1,34 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ZkDump < Command + def help + <<-EOF +Dump status of HBase cluster as seen by ZooKeeper. +EOF + end + + def command + puts admin.zk_dump + end + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/formatter.rb b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/formatter.rb new file mode 100644 index 0000000000..30402f3178 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/resources/hbase-ruby/shell/formatter.rb @@ -0,0 +1,206 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'stringio' + +# Results formatter +module Shell + module Formatter + # Base abstract class for results formatting. + class Base + attr_reader :row_count + + def is_valid_io?(obj) + obj.instance_of?(IO) || obj.instance_of?(StringIO) || obj == Kernel + end + + def refresh_width + @max_width = 0 + if $stdout.tty? + begin + @max_width = Java.jline.TerminalFactory.get.getWidth + rescue NameError => e + # nocommit debug log and ignore + end + end + end + + # Takes an output stream and a print width. + def initialize(opts = {}) + options = { + output_stream: Kernel + }.merge(opts) + + @out = options[:output_stream] + refresh_width + @row_count = 0 + + # raise an error if the stream is not valid + raise(TypeError, "Type #{@out.class} of parameter #{@out} is not IO") unless is_valid_io?(@out) + end + + def header(args = [], widths = []) + refresh_width + row(args, false, widths) unless args.empty? + @row_count = 0 + end + + # Output a row. + # Inset is whether or not to offset row by a space. + def row(args = [], inset = true, widths = []) + # Print out nothing + return if !args || args.empty? + + # Print a string + if args.is_a?(String) + output_str(args) + @out.puts + return + end + + # TODO: Look at the type. Is it RowResult? + if args.length == 1 + splits = split(@max_width, dump(args[0])) + for l in splits + output(@max_width, l) + @out.puts + end + elsif args.length == 2 + if @max_width == 0 + col1width = col2width = 0 + else + col1width = !widths || widths.empty? ? @max_width / 4 : @max_width * widths[0] / 100 + col2width = !widths || widths.length < 2 ? @max_width - col1width - 2 : @max_width * widths[1] / 100 - 2 + end + splits1 = split(col1width, dump(args[0])) + splits2 = split(col2width, dump(args[1])) + biggest = splits2.length > splits1.length ? splits2.length : splits1.length + index = 0 + while index < biggest + # Inset by one space if inset is set. + @out.print(' ') if inset + output(col1width, splits1[index]) + # Add extra space so second column lines up w/ second column output + @out.print(' ') unless inset + @out.print(' ') + output(col2width, splits2[index]) + index += 1 + @out.puts + end + else + # Print a space to set off multi-column rows + print ' ' + first = true + for e in args + @out.print ' ' unless first + first = false + @out.print e + end + puts + end + @row_count += 1 + end + + # Output the scan metrics. Can be filtered to output only those metrics whose keys exists + # in the metric_filter + def scan_metrics(scan_metrics = nil, metric_filter = []) + return if scan_metrics.nil? + unless scan_metrics.is_a?(org.apache.hadoop.hbase.client.metrics.ScanMetrics) + raise(ArgumentError, \ + 'Argument should be org.apache.hadoop.hbase.client.metrics.ScanMetrics') + end + # prefix output with empty line + @out.puts + # save row count to restore after printing metrics + # (metrics should not count towards row count) + saved_row_count = @row_count + iter = scan_metrics.getMetricsMap.entrySet.iterator + metric_hash = {} + # put keys in hash so they can be sorted easily + while iter.hasNext + metric = iter.next + metric_hash[metric.getKey.to_s] = metric.getValue.to_s + end + # print in alphabetical order + row(%w[METRIC VALUE], false) + metric_hash.sort.map do |key, value| + if !metric_filter || metric_filter.empty? || metric_filter.include?(key) + row([key, value]) + end + end + + @row_count = saved_row_count + nil + end + + def split(width, str) + return [str] if width == 0 + result = [] + index = 0 + while index < str.length + result << str.slice(index, width) + index += width + end + result + end + + def dump(str) + return if str.instance_of?(Integer) + # Remove double-quotes added by 'dump'. + str + end + + def output_str(str) + output(@max_width, str) + end + + def output_strln(str) + output_str(str) + @out.puts + end + + def output(width, str) + str = '' if str.nil? + if !width || width == str.length + @out.print(str) + else + @out.printf('%-*s', width, str) + end + end + + def footer(row_count = nil, is_stale = false) + row_count ||= @row_count + # Only output elapsed time and row count if startTime passed + @out.puts(format('%d row(s)', row_count)) + @out.puts(' (possible stale results) ') if is_stale == true + end + end + + class Console < Base + end + + class XHTMLFormatter < Base + # http://www.germane-software.com/software/rexml/doc/classes/REXML/Document.html + # http://www.crummy.com/writing/RubyCookbook/test_results/75942.html + end + + class JSON < Base + end + end +end diff --git a/linkis-engineconn-plugins/hbase/src/main/scala/org/apache/linkis/manager/engineplugin/hbase/exception/ExecutorInitException.java b/linkis-engineconn-plugins/hbase/src/main/scala/org/apache/linkis/manager/engineplugin/hbase/exception/ExecutorInitException.java new file mode 100644 index 0000000000..d0e739d9c4 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/scala/org/apache/linkis/manager/engineplugin/hbase/exception/ExecutorInitException.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.engineplugin.hbase.exception; + +import org.apache.linkis.common.exception.ErrorException; + +public class ExecutorInitException extends ErrorException { + public ExecutorInitException(int errCode, String desc) { + super(errCode, desc); + } +} diff --git a/linkis-engineconn-plugins/hbase/src/main/scala/org/apache/linkis/manager/engineplugin/hbase/exception/HBaseParamsIllegalException.scala b/linkis-engineconn-plugins/hbase/src/main/scala/org/apache/linkis/manager/engineplugin/hbase/exception/HBaseParamsIllegalException.scala new file mode 100644 index 0000000000..63f7e14ecc --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/scala/org/apache/linkis/manager/engineplugin/hbase/exception/HBaseParamsIllegalException.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.engineplugin.hbase.exception + +import org.apache.linkis.common.exception.ErrorException + +case class HBaseParamsIllegalException(errorCode: Int, errorMsg: String) + extends ErrorException(errorCode, errorMsg) diff --git a/linkis-engineconn-plugins/hbase/src/main/scala/org/apache/linkis/manager/engineplugin/hbase/exception/JobExecutorException.scala b/linkis-engineconn-plugins/hbase/src/main/scala/org/apache/linkis/manager/engineplugin/hbase/exception/JobExecutorException.scala new file mode 100644 index 0000000000..e2f129f6d6 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/main/scala/org/apache/linkis/manager/engineplugin/hbase/exception/JobExecutorException.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.engineplugin.hbase.exception + +import org.apache.linkis.common.exception.ErrorException + +case class JobExecutorException(errorCode: Int, errorMsg: String) + extends ErrorException(errorCode, errorMsg) diff --git a/linkis-engineconn-plugins/hbase/src/test/java/org/apache/linkis/manager/engineplugin/hbase/HBaseBaseTest.java b/linkis-engineconn-plugins/hbase/src/test/java/org/apache/linkis/manager/engineplugin/hbase/HBaseBaseTest.java new file mode 100644 index 0000000000..d684011646 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/test/java/org/apache/linkis/manager/engineplugin/hbase/HBaseBaseTest.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.engineplugin.hbase; + +import java.io.IOException; + +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.TableName; + +public abstract class HBaseBaseTest { + private static final HBaseTestingUtility TEST_UTILITY = new HBaseTestingUtility(); + + protected void startMiniCluster() throws Exception { + TEST_UTILITY.startMiniCluster(1); + } + + protected void shutdownMiniCluster() throws Exception { + TEST_UTILITY.shutdownMiniCluster(); + } + + protected void createTestTable() throws IOException { + TEST_UTILITY.createTable(TableName.valueOf("test_table"), new String[]{"f1", "f2"}); + } +} diff --git a/linkis-engineconn-plugins/hbase/src/test/java/org/apache/linkis/manager/engineplugin/hbase/HBaseConnectionManagerTest.java b/linkis-engineconn-plugins/hbase/src/test/java/org/apache/linkis/manager/engineplugin/hbase/HBaseConnectionManagerTest.java new file mode 100644 index 0000000000..0583e86168 --- /dev/null +++ b/linkis-engineconn-plugins/hbase/src/test/java/org/apache/linkis/manager/engineplugin/hbase/HBaseConnectionManagerTest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.engineplugin.hbase; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.linkis.manager.engineplugin.hbase.constant.HBaseEngineConnConstant; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +public class HBaseConnectionManagerTest { + @Disabled + @Test + @DisplayName("testGetConnection") + public void testGetConnection() throws IOException { + Map properties = new HashMap<>(8); + properties.put(HBaseEngineConnConstant.ZOOKEEPER_QUORUM, "localhost"); + properties.put(HBaseEngineConnConstant.ZOOKEEPER_CLIENT_PORT, "2181"); + properties.put(HBaseEngineConnConstant.HBASE_SECURITY_AUTH, "simple"); + // properties.put(HBaseEngineConnConstant.HBASE_SECURITY_AUTH, "kerberos"); + properties.put(HBaseEngineConnConstant.KERBEROS_PRINCIPAL, "hdfs@HADOOP.COM"); + properties.put(HBaseEngineConnConstant.KERBEROS_KEYTAB_FILE, "/etc/hbase/conf/hdfs.keytab"); + // 设置kerberos代理用户 + properties.put(HBaseEngineConnConstant.KERBEROS_PROXY_USER, "leojie"); + properties.put(HBaseEngineConnConstant.HBASE_REGION_SERVER_KERBEROS_PRINCIPAL, "hbase/_HOST@HADOOP.COM"); + properties.put(HBaseEngineConnConstant.HBASE_MASTER_KERBEROS_PRINCIPAL, "hbase/_HOST@HADOOP.COM"); + Connection connection = HBaseConnectionManager.getInstance().getConnection(properties); + try (Admin admin = connection.getAdmin()) { + TableName[] tableNames = admin.listTableNames(); + System.out.println(tableNames); + } + } + +}