diff --git a/openeuler/pom.xml b/openeuler/pom.xml index a4a9d86..dd1293c 100644 --- a/openeuler/pom.xml +++ b/openeuler/pom.xml @@ -94,6 +94,16 @@ es-client 1.0-SNAPSHOT + + com.fasterxml.jackson.core + jackson-databind + 2.17.3 + + + com.googlecode.json-simple + json-simple + 1.1.1 + diff --git a/openeuler/src/main/java/App.java b/openeuler/src/main/java/App.java index 29d39a6..ac5c175 100644 --- a/openeuler/src/main/java/App.java +++ b/openeuler/src/main/java/App.java @@ -1,20 +1,32 @@ +import java.io.File; +import java.util.*; +import java.util.stream.Collectors; + import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.util.*; -import java.util.stream.Collectors; +import etherpad.EPLiteClient; public class App { private static final String TARGET = System.getenv("TARGET"); + private static final String TARGET_SIG = System.getenv("TARGET") + "/sig"; + private static final String APPLICATION_PATH = System.getenv("APPLICATION_PATH"); private static final String MAPPING_PATH = System.getenv("MAPPING_PATH"); + private static final String SIG_PATH = System.getenv("SIG_PATH"); + + private static final String ETHERPAD_PATH = System.getenv("ETHERPAD_PATH"); + private static final String INDEX_PREFIX = "openeuler_articles"; + private static final String ETHERPAD_URL = System.getenv("ETHERPAD_URL"); + + private static final String ETHERPAD_KEY = System.getenv("ETHERPAD_KEY"); + private static final Logger logger = LoggerFactory.getLogger(App.class); public static void main(String[] args) { @@ -22,6 +34,8 @@ public static void main(String[] args) { PublicClient.CreateClientFormConfig(APPLICATION_PATH); PublicClient.makeIndex(INDEX_PREFIX + "_zh", MAPPING_PATH); PublicClient.makeIndex(INDEX_PREFIX + "_en", MAPPING_PATH); + sigData(); + etherpadData(); fileDate(); } catch (Exception e) { logger.error(e.getMessage()); @@ -29,7 +43,66 @@ public static void main(String[] args) { } logger.info("import end"); - System.exit(0); + } + + public static void sigData() throws Exception { + File indexFile = new File(TARGET_SIG); + if (!indexFile.exists()) { + logger.info("%s folder does not exist%n", indexFile.getPath()); + return; + } + + logger.info("begin to update sig data"); + Collection listFiles = new ArrayList<>(); + File[] subdirectories = indexFile.listFiles((dir, name) -> new File(dir, name).isDirectory()); + if (subdirectories != null) { + for (File subdir : subdirectories) { + Collection filesInSubdirectory = FileUtils.listFiles(subdir, new String[]{"yaml"}, false); + listFiles.addAll(filesInSubdirectory); + } + } else { + logger.info("sig data is null"); + return; + } + + for (File paresFile : listFiles) { + try { + // sig information has two language + Map escape = Parse.parseSigYaml(paresFile, "zh", SIG_PATH); + if (null != escape) { + PublicClient.insert(escape, INDEX_PREFIX + "_" + escape.get("lang")); + } else { + logger.info("parse null : " + paresFile.getPath()); + } + } catch (Exception e) { + logger.error(paresFile.getPath()); + logger.error("sig data imported error {}", e.getMessage()); + } + } + logger.info("sig data imported end"); + } + + public static void etherpadData(){ + EPLiteClient client = new EPLiteClient(ETHERPAD_URL, ETHERPAD_KEY); + + Map result = client.listAllPads(); + List padIds = (List) result.get("padIDs"); + + logger.info("begin to update etherpad data"); + for(Object padId : padIds) { + Map resMap = client.getText((String) padId); + if(resMap.containsKey("text")) { + try { + Map escape = Parse.parseEtherPad(resMap.get("text"), padId.toString(), ETHERPAD_PATH); + if (null != escape) { + PublicClient.insert(escape, INDEX_PREFIX + "_" + escape.get("lang")); + } + } catch (Exception e) { + logger.error("etherpad data imported error {}", e.getMessage()); + } + } + } + logger.info("etherpad data imported end"); } public static void fileDate() throws Exception { diff --git a/openeuler/src/main/java/Parse.java b/openeuler/src/main/java/Parse.java index 3b5ece1..182959e 100644 --- a/openeuler/src/main/java/Parse.java +++ b/openeuler/src/main/java/Parse.java @@ -1,10 +1,21 @@ +import java.io.*; +import java.net.*; +import java.nio.charset.StandardCharsets; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; +import java.security.SecureRandom; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + import com.alibaba.fastjson2.JSON; import com.alibaba.fastjson2.JSONArray; import com.alibaba.fastjson2.JSONObject; import org.apache.commons.io.FileUtils; import org.apache.pdfbox.pdmodel.PDDocument; -import org.apache.pdfbox.rendering.ImageType; -import org.apache.pdfbox.rendering.PDFRenderer; import org.apache.pdfbox.text.PDFTextStripper; import org.commonmark.node.Node; import org.commonmark.parser.Parser; @@ -17,18 +28,6 @@ import org.slf4j.LoggerFactory; import org.yaml.snakeyaml.Yaml; -import javax.imageio.ImageIO; -import java.awt.image.BufferedImage; -import java.io.*; -import java.net.*; -import java.nio.charset.StandardCharsets; -import java.text.SimpleDateFormat; -import java.util.*; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - public class Parse { public static final String BASEPATH = System.getenv("TARGET") + "/"; public static final String BLOG = "blog"; @@ -679,8 +678,8 @@ private static String ReadInput(InputStream is) throws IOException { return sbf.toString(); } - public static Map getRandomIpHeader() { - Random random = new Random(System.currentTimeMillis()); + public static Map getRandomIpHeader() throws NoSuchAlgorithmException, NoSuchProviderException { + SecureRandom random = SecureRandom.getInstance("SHA1PRNG", "SUN"); String ip = (random.nextInt(255) + 1) + "." + (random.nextInt(255) + 1) + "." + (random.nextInt(255) + 1) + "." + (random.nextInt(255) + 1); HashMap header = new HashMap<>(); @@ -690,4 +689,61 @@ public static Map getRandomIpHeader() { header.put("REMOTE_ADDR", ip); return header; } + + public static Map parseSigYaml(File paresFile, String lang, String sigPath) throws Exception{ + Yaml yaml = new Yaml(); + Map resMap = new HashMap<>(); + try (InputStream inputStream = new FileInputStream(paresFile)) { + Map dataMap = yaml.load(inputStream); + resMap.put("title", dataMap.get("name")); + resMap.put("lang", lang); + resMap.put("type", "sig"); + String path = sigPath + lang + "/sig/" + dataMap.get("name"); + resMap.put("path", path); + String textContent = "maintainers: "; + if (dataMap.containsKey("maintainers") && dataMap.get("maintainers") instanceof List) { + List maintainersList = (List) dataMap.get("maintainers"); + for (Object maintainerObj : maintainersList) { + if (maintainerObj instanceof Map) { + Map maintainerMap = (Map) maintainerObj; + textContent += maintainerMap.getOrDefault("name","") + ","; + textContent += maintainerMap.getOrDefault("gitee_id","") + ";"; + } + } + } + textContent += "\n" + "committers: "; + if (dataMap.containsKey("repositories") && dataMap.get("repositories") instanceof List) { + List reposList = (List) dataMap.get("repositories"); + for (Object repoObj : reposList) { + if (repoObj instanceof Map) { + Map repoMap = (Map) repoObj; + if(repoMap.containsKey("committers") && repoMap.get("committers") instanceof List){ + List committersList = (List) repoMap.get("committers"); + for (Object committerObj : committersList) { + if (committerObj instanceof Map) { + Map committerMap = (Map) committerObj; + textContent += committerMap.getOrDefault("name","") + ","; + textContent += committerMap.getOrDefault("gitee_id","") + ";"; + } + } + } + } + } + } + resMap.put("textContent", textContent); + } catch (IOException e) { + logger.error("sig yaml parse error: {}", e.getMessage()); + } + return resMap; + } + + public static Map parseEtherPad(Object text, String padId, String etherpadPath) { + Map resMap = new HashMap<>(); + resMap.put("textContent", text.toString()); + resMap.put("title", padId); + resMap.put("path", "p/" + etherpadPath + padId); + resMap.put("lang", "zh"); + resMap.put("type", "pad"); + return resMap; + } } diff --git a/openeuler/src/main/java/etherpad/EPLiteClient.java b/openeuler/src/main/java/etherpad/EPLiteClient.java new file mode 100644 index 0000000..ec29a8a --- /dev/null +++ b/openeuler/src/main/java/etherpad/EPLiteClient.java @@ -0,0 +1,25 @@ +package etherpad; + +import java.util.HashMap; +import java.util.Map; + +public class EPLiteClient { + private static final String DEFAULT_API_VERSION = "1.2.13"; + private static final String DEFAULT_ENCODING = "UTF-8"; + private final EPLiteConnection connection; + + public EPLiteClient(String url, String apiKey) { + this.connection = new EPLiteConnection(url, apiKey, DEFAULT_API_VERSION, DEFAULT_ENCODING); + } + + public Map listAllPads() { + return this.connection.get("listAllPads"); + } + + public Map getText(String padId) { + Map args = new HashMap<>(); + args.put("padID", padId); + return this.connection.get("getText", args); + } + +} diff --git a/openeuler/src/main/java/etherpad/EPLiteConnection.java b/openeuler/src/main/java/etherpad/EPLiteConnection.java new file mode 100644 index 0000000..7c4bffd --- /dev/null +++ b/openeuler/src/main/java/etherpad/EPLiteConnection.java @@ -0,0 +1,191 @@ +package etherpad; + +import java.io.UnsupportedEncodingException; +import java.util.Map; +import java.util.HashMap; +import java.util.Iterator; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.net.URLEncoder; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSession; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; + +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class EPLiteConnection { + + private static final Logger logger = LoggerFactory.getLogger(EPLiteConnection.class); + public static final int CODE_OK = 0; + public static final int CODE_INVALID_PARAMETERS = 1; + public static final int CODE_INTERNAL_ERROR = 2; + public static final int CODE_INVALID_METHOD = 3; + public static final int CODE_INVALID_API_KEY = 4; + + public final URI uri; + + public final String apiKey; + + public final String apiVersion; + + public final String encoding; + + public EPLiteConnection(String url, String apiKey, String apiVersion, String encoding) { + if (url.endsWith("/")) { + url = url.substring(0, url.length()-1); + } + this.uri = URI.create(url); + this.apiKey = apiKey; + this.apiVersion = apiVersion; + this.encoding = encoding; + } + + public Map get(String apiMethod) { + Map response = (Map) this.getObject(apiMethod, new HashMap<>()); + return (response != null) ? response : new HashMap(); + } + + public Map get(String apiMethod, Map apiArgs) { + Map response = (Map) this.getObject(apiMethod, apiArgs); + return (response != null) ? response : new HashMap(); + } + + public Object getObject(String apiMethod, Map apiArgs) { + String path = this.apiPath(apiMethod); + String query = this.queryString(apiArgs, true); + URL url = apiUrl(path, query); + Request request = new GETRequest(url); + return this.call(request); + } + + protected String apiPath(String apiMethod) { + return this.uri.getPath() + "/api/" + this.apiVersion + "/" + apiMethod; + } + + protected String queryString(Map apiArgs, boolean urlEncode) { + StringBuilder strArgs = new StringBuilder(); + apiArgs.put("apikey", this.apiKey); + Iterator i = apiArgs.entrySet().iterator(); + while (i.hasNext()) { + Map.Entry entry = (Map.Entry)i.next(); + Object key = entry.getKey(); + Object value = entry.getValue(); + if (urlEncode) { + try { + if (key instanceof String) { + URLEncoder.encode((String) key, this.encoding); + } + if (value instanceof String) { + value = URLEncoder.encode((String) value, this.encoding); + } + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(String.format( + "Unable to URLEncode using encoding '%s'", this.encoding), e); + } + } + strArgs.append(key).append("=").append(value); + if (i.hasNext()) { + strArgs.append("&"); + } + } + return strArgs.toString(); + } + + protected URL apiUrl(String path, String query) { + try { + return new URL(new URI(this.uri.getScheme(), null, this.uri.getHost(), + this.uri.getPort(), path, query, null).toString()); + } catch (MalformedURLException | URISyntaxException e) { + throw new RuntimeException("Error in the URL to the Etherpad Lite instance (" + e.getClass() + "): " + e.getMessage()); + } + } + + private Object call(Request request) { + trustServerAndCertificate(); + + try { + String response = request.send(); + return this.handleResponse(response); + } + catch (RuntimeException e) { + logger.error(e.getMessage()); + } + catch (Exception e) { + logger.error("Unable to connect to Etherpad Lite instance (" + e.getClass() + "): " + e.getMessage()); + } + return null; + } + + protected Object handleResponse(String jsonString) { + try { + JSONParser parser = new JSONParser(); + Map response = (Map) parser.parse(jsonString); + if (null != response.get("code")) { + int code = ((Long) response.get("code")).intValue(); + switch ( code ) { + case CODE_OK: + return response.get("data"); + case CODE_INVALID_PARAMETERS: + break; + case CODE_INTERNAL_ERROR: + break; + case CODE_INVALID_METHOD: + break; + case CODE_INVALID_API_KEY: + throw new RuntimeException((String)response.get("message") + jsonString); + default: + throw new RuntimeException("An unknown error has occurred while handling the response: " + jsonString); + } + } else { + throw new RuntimeException("An unexpected response from the server: " + jsonString); + } + } catch (ParseException e) { + logger.error("Unable to parse JSON response + jsonString + {}", e.getMessage()); + } + return null; + } + + private void trustServerAndCertificate() { + TrustManager[] trustAllCerts = new TrustManager[] { new X509TrustManager() { + public java.security.cert.X509Certificate[] getAcceptedIssuers() { + return null; + } + + public void checkClientTrusted( + java.security.cert.X509Certificate[] certs, String authType) { + } + + public void checkServerTrusted( + java.security.cert.X509Certificate[] certs, String authType) { + } + } + }; + + try { + SSLContext sc = SSLContext.getInstance("SSL"); + sc.init(null, trustAllCerts, new java.security.SecureRandom()); + HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); + } catch (NoSuchAlgorithmException | KeyManagementException e) { + logger.error("Unable to create SSL context {}", e); + } + + HostnameVerifier hv = new HostnameVerifier() { + public boolean verify(String hostname, SSLSession session) { + return true; + } + }; + HttpsURLConnection.setDefaultHostnameVerifier(hv); + } + +} diff --git a/openeuler/src/main/java/etherpad/GETRequest.java b/openeuler/src/main/java/etherpad/GETRequest.java new file mode 100644 index 0000000..d4f9174 --- /dev/null +++ b/openeuler/src/main/java/etherpad/GETRequest.java @@ -0,0 +1,24 @@ +package etherpad; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.net.URL; + +public class GETRequest implements Request { + private final URL url; + + public GETRequest(URL url) { + this.url = url; + } + + public String send() throws Exception { + BufferedReader in = new BufferedReader(new InputStreamReader(url.openStream(), "UTF-8")); + StringBuilder response = new StringBuilder(); + String buffer; + while ((buffer = in.readLine()) != null) { + response.append(buffer); + } + in.close(); + return response.toString(); + } +} diff --git a/openeuler/src/main/java/etherpad/Request.java b/openeuler/src/main/java/etherpad/Request.java new file mode 100644 index 0000000..2725764 --- /dev/null +++ b/openeuler/src/main/java/etherpad/Request.java @@ -0,0 +1,5 @@ +package etherpad; + +public interface Request { + String send() throws Exception; +} diff --git a/openeuler/src/main/resources/initDoc.sh b/openeuler/src/main/resources/initDoc.sh index 410bb31..304e4f6 100644 --- a/openeuler/src/main/resources/initDoc.sh +++ b/openeuler/src/main/resources/initDoc.sh @@ -9,6 +9,7 @@ mkdir -p ${SOURCE} cd ${SOURCE} git clone https://gitee.com/openeuler/openEuler-portal.git git clone https://gitee.com/openeuler/docs.git +git clone https://gitee.com/openeuler/community.git # shellcheck disable=SC2164 cd openEuler-portal pnpm install @@ -17,6 +18,12 @@ pnpm build mkdir -p ${TARGET}/zh/ mkdir -p ${TARGET}/en/ mkdir -p ${TARGET}/ru/ +mkdir -p ${TARGET}/sig/ + +if [ ! -d ${SOURCE}/community ]; then + rm -rf ${TARGET} + exit +fi if [ ! -d ${SOURCE}/openEuler-portal ]; then rm -rf ${TARGET} @@ -82,6 +89,9 @@ do cp -r ${SOURCE}/docs/docs/en/docs/* ${TARGET}/en/docs/$b/docs/ done +cd ${SOURCE}/community +cp -r ${SOURCE}/community/sig/* ${TARGET}/sig/ +