diff --git a/openeuler/pom.xml b/openeuler/pom.xml index a4a9d86..dd1293c 100644 --- a/openeuler/pom.xml +++ b/openeuler/pom.xml @@ -94,6 +94,16 @@ es-client 1.0-SNAPSHOT + + com.fasterxml.jackson.core + jackson-databind + 2.17.3 + + + com.googlecode.json-simple + json-simple + 1.1.1 + diff --git a/openeuler/src/main/java/App.java b/openeuler/src/main/java/App.java index 29d39a6..83b35bb 100644 --- a/openeuler/src/main/java/App.java +++ b/openeuler/src/main/java/App.java @@ -2,6 +2,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import etherpad.EPLiteClient; + import java.io.File; import java.util.*; import java.util.stream.Collectors; @@ -9,6 +11,8 @@ public class App { private static final String TARGET = System.getenv("TARGET"); + private static final String TARGET_SIG = System.getenv("TARGET") + "/sig"; + private static final String APPLICATION_PATH = System.getenv("APPLICATION_PATH"); private static final String MAPPING_PATH = System.getenv("MAPPING_PATH"); @@ -17,11 +21,15 @@ public class App { private static final Logger logger = LoggerFactory.getLogger(App.class); + private static final ImportConfig importConfig = new ImportConfig(); + public static void main(String[] args) { try { PublicClient.CreateClientFormConfig(APPLICATION_PATH); PublicClient.makeIndex(INDEX_PREFIX + "_zh", MAPPING_PATH); PublicClient.makeIndex(INDEX_PREFIX + "_en", MAPPING_PATH); + sigData(); + etherpadData(); fileDate(); } catch (Exception e) { logger.error(e.getMessage()); @@ -32,6 +40,65 @@ public static void main(String[] args) { System.exit(0); } + public static void sigData() throws Exception { + File indexFile = new File(TARGET_SIG); + if (!indexFile.exists()) { + logger.info("%s folder does not exist%n", indexFile.getPath()); + return; + } + + logger.info("begin to update sig data"); + Collection listFiles = new ArrayList<>(); + File[] subdirectories = indexFile.listFiles((dir, name) -> new File(dir, name).isDirectory()); + if (subdirectories != null) { + for (File subdir : subdirectories) { + Collection filesInSubdirectory = FileUtils.listFiles(subdir, new String[]{"yaml"}, false); + listFiles.addAll(filesInSubdirectory); + } + } + + for (File paresFile : listFiles) { + try { + // sig information has two language + Map escape = Parse.parseSigYaml(paresFile, "zh"); + if (null != escape) { + PublicClient.insert(escape, INDEX_PREFIX + "_" + escape.get("lang")); + } else { + logger.info("parse null : " + paresFile.getPath()); + } + } catch (Exception e) { + logger.error(paresFile.getPath()); + logger.error(e.getMessage()); + } + } + logger.info("sig data imported successfully"); + } + + public static void etherpadData(){ + String url = importConfig.getUrl(); + String apiKey = importConfig.getApiKey(); + EPLiteClient client = new EPLiteClient(url, apiKey); + + Map result = client.listAllPads(); + List padIds = (List) result.get("padIDs"); + + logger.info("begin to update etherpad data"); + for(Object padId : padIds) { + Map resMap = client.getText((String) padId); + if(resMap.containsKey("text")) { + try { + Map escape = Parse.parseEtherPad(resMap.get("text"), padId.toString()); + if (null != escape) { + PublicClient.insert(escape, INDEX_PREFIX + "_" + escape.get("lang")); + } + } catch (Exception e) { + logger.error(e.getMessage()); + } + } + } + logger.info("etherpad data imported successfully"); + } + public static void fileDate() throws Exception { File indexFile = new File(TARGET); if (!indexFile.exists()) { diff --git a/openeuler/src/main/java/ImportConfig.java b/openeuler/src/main/java/ImportConfig.java new file mode 100644 index 0000000..80e3a8f --- /dev/null +++ b/openeuler/src/main/java/ImportConfig.java @@ -0,0 +1,30 @@ +import java.io.InputStream; +import java.util.Properties; + +public class ImportConfig { + private String url; + private String apiKey; + + public ImportConfig() { + try (InputStream input = getClass().getClassLoader().getResourceAsStream("application.properties")) { + if (input == null) { + System.out.println("unable to find related config"); + return; + } + Properties prop = new Properties(); + prop.load(input); + this.url = prop.getProperty("url"); + this.apiKey = prop.getProperty("apiKey"); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + public String getUrl() { + return url; + } + + public String getApiKey() { + return apiKey; + } +} diff --git a/openeuler/src/main/java/Parse.java b/openeuler/src/main/java/Parse.java index 3b5ece1..919127d 100644 --- a/openeuler/src/main/java/Parse.java +++ b/openeuler/src/main/java/Parse.java @@ -3,8 +3,6 @@ import com.alibaba.fastjson2.JSONObject; import org.apache.commons.io.FileUtils; import org.apache.pdfbox.pdmodel.PDDocument; -import org.apache.pdfbox.rendering.ImageType; -import org.apache.pdfbox.rendering.PDFRenderer; import org.apache.pdfbox.text.PDFTextStripper; import org.commonmark.node.Node; import org.commonmark.parser.Parser; @@ -17,11 +15,13 @@ import org.slf4j.LoggerFactory; import org.yaml.snakeyaml.Yaml; -import javax.imageio.ImageIO; -import java.awt.image.BufferedImage; + import java.io.*; import java.net.*; import java.nio.charset.StandardCharsets; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; +import java.security.SecureRandom; import java.text.SimpleDateFormat; import java.util.*; import java.util.concurrent.CountDownLatch; @@ -679,8 +679,8 @@ private static String ReadInput(InputStream is) throws IOException { return sbf.toString(); } - public static Map getRandomIpHeader() { - Random random = new Random(System.currentTimeMillis()); + public static Map getRandomIpHeader() throws NoSuchAlgorithmException, NoSuchProviderException { + SecureRandom random = SecureRandom.getInstance("SHA1PRNG", "SUN"); String ip = (random.nextInt(255) + 1) + "." + (random.nextInt(255) + 1) + "." + (random.nextInt(255) + 1) + "." + (random.nextInt(255) + 1); HashMap header = new HashMap<>(); @@ -690,4 +690,61 @@ public static Map getRandomIpHeader() { header.put("REMOTE_ADDR", ip); return header; } + + public static Map parseSigYaml(File paresFile, String lang) throws Exception{ + Yaml yaml = new Yaml(); + Map resMap = new HashMap<>(); + try (InputStream inputStream = new FileInputStream(paresFile)) { + Map dataMap = yaml.load(inputStream); + resMap.put("title", dataMap.get("name")); + resMap.put("lang", lang); + resMap.put("type", "sig"); + String path = "https://www.openeuler.org/" + lang + "/sig/" + dataMap.get("name"); + resMap.put("path", path); + String textContent = "maintainers: "; + if (dataMap.containsKey("maintainers") && dataMap.get("maintainers") instanceof List) { + List maintainersList = (List) dataMap.get("maintainers"); + for (Object maintainerObj : maintainersList) { + if (maintainerObj instanceof Map) { + Map maintainerMap = (Map) maintainerObj; + textContent += maintainerMap.get("name") + ","; + textContent += maintainerMap.get("gitee_id") + ";"; + } + } + } + textContent += "\n" + "committers: "; + if (dataMap.containsKey("repositories") && dataMap.get("repositories") instanceof List) { + List reposList = (List) dataMap.get("repositories"); + for (Object repoObj : reposList) { + if (repoObj instanceof Map) { + Map repoMap = (Map) repoObj; + if(repoMap.containsKey("committers") && repoMap.get("committers") instanceof List){ + List committersList = (List) repoMap.get("committers"); + for (Object committerObj : committersList) { + if (committerObj instanceof Map) { + Map committerMap = (Map) committerObj; + textContent += committerMap.get("name") + ","; + textContent += committerMap.get("gitee_id") + ";"; + } + } + } + } + } + } + resMap.put("textContent", textContent); + } catch (IOException e) { + e.printStackTrace(); + } + return resMap; + } + + public static Map parseEtherPad(Object text, String padId) { + Map resMap = new HashMap<>(); + resMap.put("textContent", text.toString()); + resMap.put("title", padId); + resMap.put("path", "https://etherpad.openeuler.org/p/" + padId); + resMap.put("lang", "zh"); + resMap.put("type", "pad"); + return resMap; + } } diff --git a/openeuler/src/main/java/etherpad/EPLiteClient.java b/openeuler/src/main/java/etherpad/EPLiteClient.java new file mode 100644 index 0000000..ec29a8a --- /dev/null +++ b/openeuler/src/main/java/etherpad/EPLiteClient.java @@ -0,0 +1,25 @@ +package etherpad; + +import java.util.HashMap; +import java.util.Map; + +public class EPLiteClient { + private static final String DEFAULT_API_VERSION = "1.2.13"; + private static final String DEFAULT_ENCODING = "UTF-8"; + private final EPLiteConnection connection; + + public EPLiteClient(String url, String apiKey) { + this.connection = new EPLiteConnection(url, apiKey, DEFAULT_API_VERSION, DEFAULT_ENCODING); + } + + public Map listAllPads() { + return this.connection.get("listAllPads"); + } + + public Map getText(String padId) { + Map args = new HashMap<>(); + args.put("padID", padId); + return this.connection.get("getText", args); + } + +} diff --git a/openeuler/src/main/java/etherpad/EPLiteConnection.java b/openeuler/src/main/java/etherpad/EPLiteConnection.java new file mode 100644 index 0000000..685897a --- /dev/null +++ b/openeuler/src/main/java/etherpad/EPLiteConnection.java @@ -0,0 +1,188 @@ +package etherpad; + +import java.io.UnsupportedEncodingException; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.net.URLEncoder; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.util.Map; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSession; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; + +import java.util.HashMap; +import java.util.Iterator; + +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; + +public class EPLiteConnection { + + public static final int CODE_OK = 0; + public static final int CODE_INVALID_PARAMETERS = 1; + public static final int CODE_INTERNAL_ERROR = 2; + public static final int CODE_INVALID_METHOD = 3; + public static final int CODE_INVALID_API_KEY = 4; + + public final URI uri; + + public final String apiKey; + + public final String apiVersion; + + public final String encoding; + + public EPLiteConnection(String url, String apiKey, String apiVersion, String encoding) { + if (url.endsWith("/")) { + url = url.substring(0, url.length()-1); + } + this.uri = URI.create(url); + this.apiKey = apiKey; + this.apiVersion = apiVersion; + this.encoding = encoding; + } + + public Map get(String apiMethod) { + Map response = (Map) this.getObject(apiMethod, new HashMap<>()); + return (response != null) ? response : new HashMap(); + } + + public Map get(String apiMethod, Map apiArgs) { + Map response = (Map) this.getObject(apiMethod, apiArgs); + return (response != null) ? response : new HashMap(); + } + + public Object getObject(String apiMethod, Map apiArgs) { + String path = this.apiPath(apiMethod); + String query = this.queryString(apiArgs, true); + URL url = apiUrl(path, query); + Request request = new GETRequest(url); + return this.call(request); + } + + protected String apiPath(String apiMethod) { + return this.uri.getPath() + "/api/" + this.apiVersion + "/" + apiMethod; + } + + protected String queryString(Map apiArgs, boolean urlEncode) { + StringBuilder strArgs = new StringBuilder(); + apiArgs.put("apikey", this.apiKey); + Iterator i = apiArgs.entrySet().iterator(); + while (i.hasNext()) { + Map.Entry entry = (Map.Entry)i.next(); + Object key = entry.getKey(); + Object value = entry.getValue(); + if (urlEncode) { + try { + if (key instanceof String) { + URLEncoder.encode((String) key, this.encoding); + } + if (value instanceof String) { + value = URLEncoder.encode((String) value, this.encoding); + } + } catch (UnsupportedEncodingException e) { + throw new EPLiteException(String.format( + "Unable to URLEncode using encoding '%s'", this.encoding), e); + } + } + strArgs.append(key).append("=").append(value); + if (i.hasNext()) { + strArgs.append("&"); + } + } + return strArgs.toString(); + } + + protected URL apiUrl(String path, String query) { + try { + return new URL(new URI(this.uri.getScheme(), null, this.uri.getHost(), this.uri.getPort(), path, query, null).toString()); + } catch (MalformedURLException | URISyntaxException e) { + throw new EPLiteException("Error in the URL to the Etherpad Lite instance (" + e.getClass() + "): " + e.getMessage()); + } + } + + private Object call(Request request) { + trustServerAndCertificate(); + + try { + String response = request.send(); + return this.handleResponse(response); + } + catch (EPLiteException e) { + throw e; + } + catch (Exception e) { + throw new EPLiteException("Unable to connect to Etherpad Lite instance (" + e.getClass() + "): " + e.getMessage()); + } + } + + protected Object handleResponse(String jsonString) { + try { + JSONParser parser = new JSONParser(); + Map response = (Map) parser.parse(jsonString); + if (response.get("code") != null) { + int code = ((Long) response.get("code")).intValue(); + switch ( code ) { + case 0: + return response.get("data"); + case 1: + break; + case 2: + break; + case 3: + break; + case 4: + throw new EPLiteException((String)response.get("message") + jsonString); + default: + throw new EPLiteException("An unknown error has occurred while handling the response: " + jsonString); + } + } else { + throw new EPLiteException("An unexpected response from the server: " + jsonString); + } + } catch (ParseException e) { + throw new EPLiteException("Unable to parse JSON response (" + jsonString + ")", e); + } + return null; + } + + private void trustServerAndCertificate() { + TrustManager[] trustAllCerts = new TrustManager[] { new X509TrustManager() { + public java.security.cert.X509Certificate[] getAcceptedIssuers() { + return null; + } + + public void checkClientTrusted( + java.security.cert.X509Certificate[] certs, String authType) { + } + + public void checkServerTrusted( + java.security.cert.X509Certificate[] certs, String authType) { + } + } + }; + + try { + SSLContext sc = SSLContext.getInstance("SSL"); + sc.init(null, trustAllCerts, new java.security.SecureRandom()); + HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); + } catch (NoSuchAlgorithmException | KeyManagementException e) { + throw new EPLiteException("Unable to create SSL context", e); + } + + HostnameVerifier hv = new HostnameVerifier() { + //@Override + public boolean verify(String hostname, SSLSession session) { + return true; + } + }; + HttpsURLConnection.setDefaultHostnameVerifier(hv); + } + +} diff --git a/openeuler/src/main/java/etherpad/EPLiteException.java b/openeuler/src/main/java/etherpad/EPLiteException.java new file mode 100644 index 0000000..89781a1 --- /dev/null +++ b/openeuler/src/main/java/etherpad/EPLiteException.java @@ -0,0 +1,11 @@ +package etherpad; + +public class EPLiteException extends RuntimeException { + public EPLiteException(String message) { + super(message); + } + + public EPLiteException(String message, Throwable cause) { + super(message, cause); + } +} \ No newline at end of file diff --git a/openeuler/src/main/java/etherpad/GETRequest.java b/openeuler/src/main/java/etherpad/GETRequest.java new file mode 100644 index 0000000..c0f9e66 --- /dev/null +++ b/openeuler/src/main/java/etherpad/GETRequest.java @@ -0,0 +1,23 @@ +package etherpad; + +import java.net.URL; +import java.io.BufferedReader; +import java.io.InputStreamReader; +public class GETRequest implements Request { + private final URL url; + + public GETRequest(URL url) { + this.url = url; + } + + public String send() throws Exception { + BufferedReader in = new BufferedReader(new InputStreamReader(url.openStream(), "UTF-8")); + StringBuilder response = new StringBuilder(); + String buffer; + while ((buffer = in.readLine()) != null) { + response.append(buffer); + } + in.close(); + return response.toString(); + } +} diff --git a/openeuler/src/main/java/etherpad/Request.java b/openeuler/src/main/java/etherpad/Request.java new file mode 100644 index 0000000..2725764 --- /dev/null +++ b/openeuler/src/main/java/etherpad/Request.java @@ -0,0 +1,5 @@ +package etherpad; + +public interface Request { + String send() throws Exception; +} diff --git a/openeuler/src/main/resources/initDoc.sh b/openeuler/src/main/resources/initDoc.sh index 410bb31..1900602 100644 --- a/openeuler/src/main/resources/initDoc.sh +++ b/openeuler/src/main/resources/initDoc.sh @@ -9,6 +9,7 @@ mkdir -p ${SOURCE} cd ${SOURCE} git clone https://gitee.com/openeuler/openEuler-portal.git git clone https://gitee.com/openeuler/docs.git +git clone https://gitee.com/openeuler/community.git # shellcheck disable=SC2164 cd openEuler-portal pnpm install @@ -17,6 +18,7 @@ pnpm build mkdir -p ${TARGET}/zh/ mkdir -p ${TARGET}/en/ mkdir -p ${TARGET}/ru/ +mkdir -p ${TARGET}/sig/ if [ ! -d ${SOURCE}/openEuler-portal ]; then rm -rf ${TARGET} @@ -82,6 +84,9 @@ do cp -r ${SOURCE}/docs/docs/en/docs/* ${TARGET}/en/docs/$b/docs/ done +cd ${SOURCE}/community +cp -r ${SOURCE}/community/sig/* ${TARGET}/sig/ +