Skip to content

Commit

Permalink
!8 搜索词提示数据导入功能
Browse files Browse the repository at this point in the history
Merge pull request !8 from 陈阳/master
  • Loading branch information
JoannaNil authored and gitee-org committed Mar 14, 2024
2 parents b95a310 + 7effefd commit a4318e8
Show file tree
Hide file tree
Showing 12 changed files with 842 additions and 17 deletions.
2 changes: 1 addition & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ RUN cd /EaseSearch-Import/import-task/target/classes \
&& chmod +x initDoc.sh \
&& ./initDoc.sh

RUN cp -r jdk-17.0.7 jre
RUN cp -r /jdk-17.0.7 /jre


FROM openeuler/openeuler:23.03
Expand Down
7 changes: 4 additions & 3 deletions es-client/src/main/java/EsClient.java
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,13 @@

public class EsClient {

private static final Logger logger = LogManager.getLogger(EsClientCer.class);
private static final Logger logger = LogManager.getLogger(EsClient.class);

/**
* 创建客户端的类,定义create函数用于创建客户端。
*/
public static void create(List<String> host, int port, String protocol, int connectTimeout, int connectionRequestTimeout, int socketTimeout, String username, String password) throws IOException {
public static RestHighLevelClient create(List<String> host, int port, String protocol, int connectTimeout, int connectionRequestTimeout, int socketTimeout, String username, String password) throws IOException {
// return new RestHighLevelClient(RestClient.builder(new HttpHost("192.168.1.203", 9200, "http")));
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
SSLContext sc = null;
Expand All @@ -59,7 +60,7 @@ public static void create(List<String> host, int port, String protocol, int conn
ClusterHealthResponse response = client.cluster().health(request, RequestOptions.DEFAULT);
logger.info("es rest client health response {} ", response);

PublicClient.restHighLevelClient = client;
return client;
}


Expand Down
4 changes: 2 additions & 2 deletions es-client/src/main/java/EsClientCer.java
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ public class EsClientCer {

private static final Logger logger = LogManager.getLogger(EsClientCer.class);

public static void create(List<String> host, int port, String protocol, int connectTimeout, int connectionRequestTimeout, int socketTimeout, String username, String password, String cerFilePath,
public static RestHighLevelClient create(List<String> host, int port, String protocol, int connectTimeout, int connectionRequestTimeout, int socketTimeout, String username, String password, String cerFilePath,
String cerPassword) throws IOException {

final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
Expand Down Expand Up @@ -66,7 +66,7 @@ public static void create(List<String> host, int port, String protocol, int conn
ClusterHealthResponse response = client.cluster().health(request, RequestOptions.DEFAULT);
logger.info("es rest client health response {} ", response);

PublicClient.restHighLevelClient = client;
return client;
}


Expand Down
25 changes: 15 additions & 10 deletions es-client/src/main/java/PublicClient.java
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,10 @@
import org.yaml.snakeyaml.Yaml;
import org.yaml.snakeyaml.constructor.Constructor;

import java.io.*;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.Set;
Expand All @@ -43,8 +46,12 @@ public static void CreateClientFormConfig(String configPath) throws Exception {
}
}

CreateClientFormConfigByConfig(yamlConfig);
}

public static void CreateClientFormConfigByConfig(YamlConfig yamlConfig) throws Exception {
if (yamlConfig.isUseCer()) {
EsClientCer.create(
restHighLevelClient = EsClientCer.create(
yamlConfig.getHost(),
yamlConfig.getPort(),
yamlConfig.getProtocol(),
Expand All @@ -57,7 +64,7 @@ public static void CreateClientFormConfig(String configPath) throws Exception {
yamlConfig.getCerPassword()
);
} else {
EsClient.create(
restHighLevelClient = EsClient.create(
yamlConfig.getHost(),
yamlConfig.getPort(),
yamlConfig.getProtocol(),
Expand All @@ -71,9 +78,6 @@ public static void CreateClientFormConfig(String configPath) throws Exception {
}





public static void makeIndex(String index, String mappingPath) throws IOException {
GetIndexRequest request = new GetIndexRequest(index);
request.local(false);
Expand All @@ -85,10 +89,12 @@ public static void makeIndex(String index, String mappingPath) throws IOExceptio
}

CreateIndexRequest request1 = new CreateIndexRequest(index);
File mappingJson = FileUtils.getFile(mappingPath);
String mapping = FileUtils.readFileToString(mappingJson, StandardCharsets.UTF_8);
if (!mappingPath.isBlank()) {
File mappingJson = FileUtils.getFile(mappingPath);
String mapping = FileUtils.readFileToString(mappingJson, StandardCharsets.UTF_8);
request1.mapping(mapping, XContentType.JSON);
}

request1.mapping(mapping, XContentType.JSON);
request1.setTimeout(TimeValue.timeValueMillis(1));

restHighLevelClient.indices().create(request1, RequestOptions.DEFAULT);
Expand All @@ -101,7 +107,6 @@ public static void insert(Map<String, Object> data, String index) throws Excepti

public static void deleteExpired(Set<String> idSet, String index) {
try {
long st = System.currentTimeMillis();
int scrollSize = 500;//一次读取的doc数量
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.matchAllQuery());//读取全量数据
Expand Down
2 changes: 1 addition & 1 deletion opengauss/src/main/resources/initDoc.sh
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ for r in $(git branch -r --list "origin/*"); do
# shellcheck disable=SC1020
# shellcheck disable=SC1009
# shellcheck disable=SC2053
if [[ "website" != $b ]] && [[ "HEAD" != $b ]] && [[ "->" != $b ]] && [[ "reconstruct-frozen" != $b ]] && [[ "master-bak" != $b ]] && [[ "website-v2" != $b ]] && [[ "z11" != $b ]]; then
if [[ "website" != $b ]] && [[ "HEAD" != $b ]] && [[ "->" != $b ]] && [[ "reconstruct-frozen" != $b ]] && [[ "master-bak" != $b ]] && [[ "website-v2" != $b ]] && [[ "z11" != $b ]] && [[ "revert-merge-6084-master" != $b ]]; then
git checkout $r
mkdir -p ${TARGET}/zh/docs/$b/docs
mkdir -p ${TARGET}/en/docs/$b/docs
Expand Down
50 changes: 50 additions & 0 deletions search-help/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
FROM gplane/pnpm as Builder
ENV LANG="C.UTF-8"

ARG COMMUNITY=openeuler

WORKDIR /

RUN apt update \
&& wget https://download.oracle.com/java/17/archive/jdk-17.0.7_linux-x64_bin.tar.gz \
&& tar -zxvf jdk-17.0.7_linux-x64_bin.tar.gz \
&& wget https://repo.huaweicloud.com/apache/maven/maven-3/3.8.1/binaries/apache-maven-3.8.1-bin.tar.gz \
&& tar -zxvf apache-maven-3.8.1-bin.tar.gz \
&& npm i pnpm -g

ENV JAVA_HOME=/jdk-17.0.7
ENV PATH=${JAVA_HOME}/bin:$PATH

ENV MAVEN_HOME=/apache-maven-3.8.1
ENV PATH=${MAVEN_HOME}/bin:$PATH

COPY . /EaseSearch-Import/search-help

RUN cd /EaseSearch-Import/search-help \
&& mvn clean install package -Dmaven.test.skip

RUN cp -r /jdk-17.0.7 /jre

FROM openeuler/openeuler:23.03
ENV LANG="C.UTF-8"

RUN yum update -y \
&& yum install -y shadow

RUN groupadd -g 1001 easysearch \
&& useradd -u 1001 -g easysearch -s /bin/bash -m easysearch

ENV WORKSPACE=/home/easysearch
WORKDIR ${WORKSPACE}

COPY --chown=easysearch --from=Builder /EaseSearch-Import/search-help/target ${WORKSPACE}/target
COPY --chown=easysearch --from=Builder /jre ${WORKSPACE}/jre

ENV JAVA_HOME=${WORKSPACE}/jre
ENV PATH=${JAVA_HOME}/bin:$PATH

USER easysearch

CMD java -jar ${WORKSPACE}/target/search-help.jar


127 changes: 127 additions & 0 deletions search-help/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<groupId>com.search</groupId>
<artifactId>search-help</artifactId>
<version>1.0-SNAPSHOT</version>

<properties>
<maven.compiler.source>17</maven.compiler.source>
<maven.compiler.target>17</maven.compiler.target>
</properties>

<dependencies>
<dependency>
<groupId>org.apache.pdfbox</groupId>
<artifactId>pdfbox</artifactId>
<version>2.0.19</version>
</dependency>

<dependency>
<groupId>com.itextpdf</groupId>
<artifactId>itextpdf</artifactId>
<version>5.5.13.2</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.7</version>
</dependency>

<dependency>
<groupId>org.commonmark</groupId>
<artifactId>commonmark</artifactId>
<version>0.21.0</version>
</dependency>

<dependency>
<groupId>org.commonmark</groupId>
<artifactId>commonmark-ext-gfm-tables</artifactId>
<version>0.21.0</version>
</dependency>

<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.15.3</version>
</dependency>

<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>7.6.2</version>
</dependency>

<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>7.6.2</version>
</dependency>

<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>2.0.7</version>
</dependency>

<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
<version>1.32</version>
</dependency>

<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.21.1</version>
</dependency>

<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.21.1</version>
</dependency>

<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.21</version>
</dependency>

</dependencies>

<build>
<finalName>search-help</finalName>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.5.0</version>
<configuration>
<appendAssemblyId>false</appendAssemblyId>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<archive>
<manifest>
<mainClass>App</mainClass>
</manifest>
</archive>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>

</project>
Loading

0 comments on commit a4318e8

Please sign in to comment.