openeuler新增sig、etherpad数据

openeuler/pom.xml

@@ -94,6 +94,16 @@
             <artifactId>es-client</artifactId>
             <version>1.0-SNAPSHOT</version>
         </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <version>2.17.3</version>
+        </dependency>
+        <dependency>
+            <groupId>com.googlecode.json-simple</groupId>
+            <artifactId>json-simple</artifactId>
+            <version>1.1.1</version>
+        </dependency>
 
     </dependencies>
 

openeuler/src/main/java/App.java

@@ -2,13 +2,17 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import etherpad.EPLiteClient;
+
 import java.io.File;
 import java.util.*;
 import java.util.stream.Collectors;
 
 public class App {
     private static final String TARGET = System.getenv("TARGET");
 
+    private static final String TARGET_SIG = System.getenv("TARGET") + "/sig";
+
     private static final String APPLICATION_PATH = System.getenv("APPLICATION_PATH");
 
     private static final String MAPPING_PATH = System.getenv("MAPPING_PATH");
@@ -17,11 +21,15 @@ public class App {
 
     private static final Logger logger = LoggerFactory.getLogger(App.class);
 
+    private static final ImportConfig importConfig = new ImportConfig();
+
     public static void main(String[] args) {
         try {
             PublicClient.CreateClientFormConfig(APPLICATION_PATH);
             PublicClient.makeIndex(INDEX_PREFIX + "_zh", MAPPING_PATH);
             PublicClient.makeIndex(INDEX_PREFIX + "_en", MAPPING_PATH);
+            sigData();
+            etherpadData();
             fileDate();
         } catch (Exception e) {
             logger.error(e.getMessage());
@@ -32,6 +40,65 @@ public static void main(String[] args) {
         System.exit(0);
     }
 
+    public static void sigData() throws Exception {
+        File indexFile = new File(TARGET_SIG);
+        if (!indexFile.exists()) {
+            logger.info("%s folder does not exist%n", indexFile.getPath());
+            return;
+        }
+
+        logger.info("begin to update sig data");
+        Collection<File> listFiles = new ArrayList<>();
+        File[] subdirectories = indexFile.listFiles((dir, name) -> new File(dir, name).isDirectory());
+        if (subdirectories != null) {
+            for (File subdir : subdirectories) {
+                Collection<File> filesInSubdirectory = FileUtils.listFiles(subdir, new String[]{"yaml"}, false);
+                listFiles.addAll(filesInSubdirectory);
+            }
+        }
+
+        for (File paresFile : listFiles) {
+                try {
+                    // sig information has two language
+                    Map<String, Object> escape = Parse.parseSigYaml(paresFile, "zh");
+                    if (null != escape) {
+                        PublicClient.insert(escape, INDEX_PREFIX + "_" + escape.get("lang"));
+                    } else {
+                        logger.info("parse null : " + paresFile.getPath());
+                    }
+                } catch (Exception e) {
+                    logger.error(paresFile.getPath());
+                    logger.error(e.getMessage());
+                }
+            }
+        logger.info("sig data imported successfully");
+    }
+
+    public static void etherpadData(){
+        String url = importConfig.getUrl();
+        String apiKey = importConfig.getApiKey();
+        EPLiteClient client = new EPLiteClient(url, apiKey);
+
+        Map result = client.listAllPads();
+        List padIds = (List) result.get("padIDs");
+
+        logger.info("begin to update etherpad data");
+        for(Object padId : padIds) {
+            Map<String, Object> resMap = client.getText((String) padId);
+            if(resMap.containsKey("text")) {
+                try {
+                    Map<String, Object> escape = Parse.parseEtherPad(resMap.get("text"), padId.toString());
+                    if (null != escape) {
+                        PublicClient.insert(escape, INDEX_PREFIX + "_" + escape.get("lang"));
+                    }
+                } catch (Exception e) {
+                    logger.error(e.getMessage());
+                }
+            }
+        }
+        logger.info("etherpad data imported successfully");
+    }
+
     public static void fileDate() throws Exception {
         File indexFile = new File(TARGET);
         if (!indexFile.exists()) {

openeuler/src/main/java/ImportConfig.java

@@ -0,0 +1,30 @@
+import java.io.InputStream;
+import java.util.Properties;
+
+public class ImportConfig {
+    private String url;
+    private String apiKey;
+
+    public ImportConfig() {
+        try (InputStream input = getClass().getClassLoader().getResourceAsStream("application.properties")) {
+            if (input == null) {
+                System.out.println("unable to find related config");
+                return;
+            }
+            Properties prop = new Properties();
+            prop.load(input);
+            this.url = prop.getProperty("url");
+            this.apiKey = prop.getProperty("apiKey");
+        } catch (Exception ex) {
+            ex.printStackTrace();
+        }
+    }
+
+    public String getUrl() {
+        return url;
+    }
+
+    public String getApiKey() {
+        return apiKey;
+    }
+}

openeuler/src/main/java/Parse.java

@@ -3,8 +3,6 @@
 import com.alibaba.fastjson2.JSONObject;
 import org.apache.commons.io.FileUtils;
 import org.apache.pdfbox.pdmodel.PDDocument;
-import org.apache.pdfbox.rendering.ImageType;
-import org.apache.pdfbox.rendering.PDFRenderer;
 import org.apache.pdfbox.text.PDFTextStripper;
 import org.commonmark.node.Node;
 import org.commonmark.parser.Parser;
@@ -17,11 +15,13 @@
 import org.slf4j.LoggerFactory;
 import org.yaml.snakeyaml.Yaml;
 
-import javax.imageio.ImageIO;
-import java.awt.image.BufferedImage;
+
 import java.io.*;
 import java.net.*;
 import java.nio.charset.StandardCharsets;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.SecureRandom;
 import java.text.SimpleDateFormat;
 import java.util.*;
 import java.util.concurrent.CountDownLatch;
@@ -679,8 +679,8 @@ private static String ReadInput(InputStream is) throws IOException {
         return sbf.toString();
     }
 
-    public static Map<String, String> getRandomIpHeader() {
-        Random random = new Random(System.currentTimeMillis());
+    public static Map<String, String> getRandomIpHeader() throws NoSuchAlgorithmException, NoSuchProviderException {
+        SecureRandom random = SecureRandom.getInstance("SHA1PRNG", "SUN");
         String ip = (random.nextInt(255) + 1) + "." + (random.nextInt(255) + 1) + "." + (random.nextInt(255) + 1) + "."
                 + (random.nextInt(255) + 1);
         HashMap<String, String> header = new HashMap<>();
@@ -690,4 +690,61 @@ public static Map<String, String> getRandomIpHeader() {
         header.put("REMOTE_ADDR", ip);
         return header;
     }
+
+    public static Map<String, Object> parseSigYaml(File paresFile, String lang) throws Exception{
+            Yaml yaml = new Yaml();
+            Map<String, Object> resMap = new HashMap<>();
+            try (InputStream inputStream = new FileInputStream(paresFile)) {
+                Map<String, Object> dataMap = yaml.load(inputStream);
+                resMap.put("title", dataMap.get("name"));
+                resMap.put("lang", lang);
+                resMap.put("type", "sig");
+                String path = "https://www.openeuler.org/" + lang + "/sig/" + dataMap.get("name");
+                resMap.put("path", path);
+                String textContent = "maintainers: ";
+                if (dataMap.containsKey("maintainers") && dataMap.get("maintainers") instanceof List) {
+                    List<?> maintainersList = (List<?>) dataMap.get("maintainers");
+                    for (Object maintainerObj : maintainersList) {
+                        if (maintainerObj instanceof Map) {
+                            Map<String, Object> maintainerMap = (Map<String, Object>) maintainerObj;
+                            textContent += maintainerMap.get("name") + ","; 
+                            textContent += maintainerMap.get("gitee_id") + ";";
+                        }
+                    }
+                }
+                textContent += "\n" + "committers: ";
+                if (dataMap.containsKey("repositories") && dataMap.get("repositories") instanceof List) {
+                    List<?> reposList = (List<?>) dataMap.get("repositories");
+                    for (Object repoObj : reposList) {
+                        if (repoObj instanceof Map) {
+                            Map<String, Object> repoMap = (Map<String, Object>) repoObj;
+                            if(repoMap.containsKey("committers") && repoMap.get("committers") instanceof List){
+                                List<?> committersList = (List<?>) repoMap.get("committers");
+                                for (Object committerObj : committersList) {
+                                    if (committerObj instanceof Map) {
+                                        Map<String, Object> committerMap = (Map<String, Object>) committerObj;
+                                        textContent += committerMap.get("name") + ","; 
+                                        textContent += committerMap.get("gitee_id") + ";";
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+                resMap.put("textContent", textContent);
+            } catch (IOException e) {
+                e.printStackTrace();  
+            }
+            return resMap;
+    }
+
+    public static Map<String, Object> parseEtherPad(Object text, String padId) {
+        Map<String, Object> resMap = new HashMap<>();
+        resMap.put("textContent", text.toString());
+        resMap.put("title", padId);
+        resMap.put("path", "https://etherpad.openeuler.org/p/" + padId);
+        resMap.put("lang", "zh");
+        resMap.put("type", "pad");
+        return resMap;
+    }
 }

openeuler/src/main/java/etherpad/EPLiteClient.java

@@ -0,0 +1,25 @@
+package etherpad;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class EPLiteClient {
+    private static final String DEFAULT_API_VERSION = "1.2.13";
+    private static final String DEFAULT_ENCODING = "UTF-8";
+    private final EPLiteConnection connection;
+
+    public EPLiteClient(String url, String apiKey) {
+        this.connection = new EPLiteConnection(url, apiKey, DEFAULT_API_VERSION, DEFAULT_ENCODING);
+    }
+
+    public Map listAllPads() {
+        return this.connection.get("listAllPads");
+    }
+
+    public Map getText(String padId) {
+        Map<String,Object> args = new HashMap<>();
+        args.put("padID", padId);
+        return this.connection.get("getText", args);
+    }
+
+}