add oscs crawler

Cargo.toml

@@ -28,3 +28,7 @@ scraper = {version = "0.19", features = ["atomic"]}
 
 tracing-subscriber = { version = "0.3.18", features = ["env-filter", "json"] }
 tracing = "0.1.40"
+
+chrono ={version = "0.4", features = ["serde"]}
+
+async-trait = "0.1"

src/grab/avd.rs

@@ -37,7 +37,7 @@ impl AVDCrawler {
         }
     }
 
-    pub async fn get_update(&self, page_limit: i32) -> Result<Vec<VulnInfo>> {
+    async fn _get_update(&self, page_limit: i32) -> Result<Vec<VulnInfo>> {
         let mut page_count = self.get_page_count().await?;
         if page_count > page_limit {
             page_count = page_limit;

src/grab/mod.rs

@@ -1,4 +1,5 @@
 pub mod avd;
+pub mod oscs;
 
 pub use avd::AVDCrawler;
 
@@ -16,7 +17,7 @@ pub struct VulnInfo {
     pub tags: Vec<String>,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq)]
 pub enum Severity {
     Low,
     Medium,

src/grab/oscs.rs

@@ -0,0 +1,265 @@
+use chrono::{DateTime, FixedOffset};
+use serde::{Deserialize, Serialize};
+use tracing::error;
+
+use crate::{
+    grab::{Severity, VulnInfo},
+    help::Help,
+    utils::timestamp_to_date,
+    Error, Result,
+};
+
+const OSCS_PAGE_SIZE: i32 = 10;
+
+pub struct OscCrawler {
+    pub name: String,
+    pub display_name: String,
+    pub link: String,
+    pub help: Help,
+}
+
+impl Default for OscCrawler {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl OscCrawler {
+    pub fn new() -> Self {
+        let help = Help::new();
+        OscCrawler {
+            name: "oscs".to_string(),
+            display_name: "OSCS开源安全情报预警".to_string(),
+            link: "https://www.oscs1024.com/cm".to_string(),
+            help,
+        }
+    }
+
+    async fn _get_update(&self, page_limit: i32) -> Result<Vec<VulnInfo>> {
+        let mut page_count = self.get_page_count().await?;
+        if page_count > page_limit {
+            page_count = page_limit;
+        }
+        let mut res = Vec::new();
+        if let Some(i) = (1..=page_count).next() {
+            let data = self.parse_page(i).await?;
+            res.extend(data)
+        }
+        println!("{:?}", res);
+        Ok(res)
+    }
+
+    pub async fn get_page_count(&self) -> Result<i32> {
+        let list_url = "https://www.oscs1024.com/oscs/v1/intelligence/list";
+        let params = serde_json::json!({
+            "page": 1,
+            "per_page": 10,
+        });
+        let oscs_list_resp: OscsListResp =
+            self.help.post_json(list_url, &params).await?.json().await?;
+        let total = oscs_list_resp.data.total;
+        if total <= 0 {
+            return Err(Error::Message("oscs get total error".to_owned()));
+        }
+        let page_count = total / OSCS_PAGE_SIZE;
+        if page_count == 0 {
+            return Ok(1);
+        }
+        if total % page_count != 0 {
+            return Ok(page_count + 1);
+        }
+        Ok(page_count)
+    }
+
+    pub async fn parse_page(&self, page: i32) -> Result<Vec<VulnInfo>> {
+        let list_url = "https://www.oscs1024.com/oscs/v1/intelligence/list";
+        let params = serde_json::json!({
+            "page": page,
+            "per_page": OSCS_PAGE_SIZE,
+        });
+        let oscs_list_resp: OscsListResp =
+            self.help.post_json(list_url, &params).await?.json().await?;
+        let mut res = Vec::with_capacity(oscs_list_resp.data.data.len());
+        for item in oscs_list_resp.data.data {
+            let mut tags = Vec::new();
+            if item.is_push == 1 {
+                tags.push("发布预警".to_string());
+            }
+            let event_type = match item.intelligence_type {
+                1 => "公开漏洞",
+                2 => "墨菲安全独家",
+                3 => "投毒情报",
+                _ => "公开漏洞",
+            };
+            tags.push(event_type.to_string());
+
+            let vuln_info = self.parse_detail(&item.mps).await;
+            match vuln_info {
+                Ok(mut vuln) => {
+                    vuln.tags = tags;
+                    res.push(vuln)
+                }
+                Err(e) => {
+                    error!("oscs parse {} detail error: {}", &item.mps, e.to_string());
+                    continue;
+                }
+            }
+        }
+
+        Ok(res)
+    }
+
+    pub async fn parse_detail(&self, mps: &str) -> Result<VulnInfo> {
+        let detail_url = "https://www.oscs1024.com/oscs/v1/vdb/info";
+        let params = serde_json::json!({
+            "vuln_no": mps,
+        });
+        let detail: OscsDetailResp = self
+            .help
+            .post_json(detail_url, &params)
+            .await?
+            .json()
+            .await?;
+        if detail.code != 200 || !detail.success || detail.data.is_empty() {
+            return Err(Error::Message(format!("oscs get: {} detail error", mps)));
+        };
+        let data = detail.data[0].clone();
+        let severity = match data.level.as_str() {
+            "Critical" => Severity::Critical,
+            "High" => Severity::High,
+            "Medium" => Severity::Medium,
+            "Low" => Severity::Low,
+            _ => Severity::Low,
+        };
+        let disclosure = timestamp_to_date(data.publish_time)?;
+        let references = data
+            .references
+            .iter()
+            .map(|r| r.url.clone())
+            .collect::<Vec<_>>();
+
+        let solutions = self.get_solutions(data.soulution_data);
+
+        let data = VulnInfo {
+            unique_key: data.vuln_no,
+            title: data.vuln_title,
+            description: data.description,
+            severity,
+            cve: data.cve_id,
+            disclosure,
+            references,
+            solutions,
+            from: self.link.clone(),
+            tags: vec![],
+        };
+        Ok(data)
+    }
+
+    pub fn get_solutions(&self, solutions: Vec<String>) -> String {
+        solutions
+            .iter()
+            .enumerate()
+            .map(|(index, item)| format!("{}.{}.\n", index + 1, item))
+            .collect::<Vec<_>>()
+            .join("")
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct OscsListResp {
+    pub success: bool,
+    pub code: i32,
+    pub time: i32,
+    pub info: String,
+    pub data: OscsData,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct OscsData {
+    pub total: i32,
+    pub data: Vec<OscsItem>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct OscsItem {
+    pub id: String,
+    pub title: String,
+    pub url: String,
+    pub mps: String,
+    pub public_time: DateTime<FixedOffset>,
+    pub intelligence_type: i32,
+    pub is_push: i8,
+    pub is_poc: i8,
+    pub is_exp: i8,
+    pub level: String,
+    pub created_at: DateTime<FixedOffset>,
+    pub updated_at: DateTime<FixedOffset>,
+    pub is_subscribe: i8,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct OscsDetailResp {
+    pub data: Vec<OscsDetail>,
+    pub success: bool,
+    pub code: i32,
+    pub time: i32,
+    pub info: String,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct OscsDetail {
+    pub attack_vector: String,
+    pub cvss_vector: String,
+    pub exp: bool,
+    pub exploit_requirement_cost: String,
+    pub exploitability: String,
+    pub scope_influence: String,
+    pub source: String,
+    pub vuln_type: String,
+    pub cvss_score: f64,
+    pub cve_id: String,
+    pub vuln_cve_id: String,
+    pub cnvd_id: String,
+    pub is_origin: bool,
+    pub languages: Vec<String>,
+    pub description: String,
+    pub effect: Vec<Effect>,
+    pub influence: i32,
+    pub level: String,
+    pub patch: String,
+    pub poc: bool,
+    pub publish_time: i64,
+    pub references: Vec<References>,
+    pub suggest_level: String,
+    pub vuln_suggest: String,
+    pub title: String,
+    pub troubleshooting: Vec<String>,
+    pub vuln_title: String,
+    pub vuln_code_usage: Vec<String>,
+    pub vuln_no: String,
+    pub soulution_data: Vec<String>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Effect {
+    pub affected_all_version: bool,
+    pub affected_version: String,
+    pub effect_id: i32,
+    pub java_qn_list: Vec<String>,
+    pub min_fixed_version: String,
+    pub name: String,
+    pub solutions: Vec<Solutions>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Solutions {
+    pub compatibility: i32,
+    pub description: String,
+    pub r#type: String,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct References {
+    pub name: String,
+    pub url: String,
+}

src/help.rs

@@ -30,4 +30,12 @@ impl Help {
         let content = self.http_client.get(url).send().await?.text().await?;
         Ok(content)
     }
+
+    pub async fn post_json<Body>(&self, url: &str, body: &Body) -> Result<reqwest::Response>
+    where
+        Body: serde::Serialize,
+    {
+        let content = self.http_client.post(url).json(body).send().await?;
+        Ok(content)
+    }
 }

src/lib.rs

@@ -6,6 +6,7 @@ pub mod grab;
 pub mod help;
 pub mod logger;
 pub mod tera;
+pub mod utils;
 
 use std::{sync::Arc, time::Duration};
 
@@ -16,8 +17,6 @@ pub use error::Error;
 use tokio::time;
 use tokio_cron_scheduler::{Job, JobScheduler};
 
-use crate::grab::AVDCrawler;
-
 pub type Result<T, E = Error> = std::result::Result<T, E>;
 
 #[derive(Clone)]
@@ -55,7 +54,11 @@ async fn my_task(app_context: Arc<AppContext>) -> Result<()> {
     tracing::info!("{:?}", app_context.config);
 
     tracing::info!("Executing task...");
-    let avd = AVDCrawler::new();
-    avd.get_update(1).await?;
+    // let avd = AVDCrawler::new();
+    // avd.get_update(1).await?;
+    // let oscs = OscCrawler::new();
+    // oscs.get_update(1).await?;
+    // oscs.get_page_count().await?;
+    // oscs.get_detail_links("MPS-1nsb-30rg").await?;
     Ok(())
 }

src/utils.rs

@@ -0,0 +1,10 @@
+use crate::{Error, Result};
+use chrono::DateTime;
+
+pub fn timestamp_to_date(timestamp: i64) -> Result<String> {
+    let dt = DateTime::from_timestamp(timestamp, 0);
+    if let Some(dt) = dt {
+        return Ok(dt.format("%Y-%m-%d").to_string());
+    }
+    Err(Error::Message("convert timestamp to date error".to_owned()))
+}