From f7e50860d24afa7cae4817494d65f20c35158112 Mon Sep 17 00:00:00 2001 From: Johannes Helmold Date: Mon, 23 Sep 2024 11:08:55 +0200 Subject: [PATCH 01/26] Add: Added the new CVE scan method with the new CPE matching. --- src/manage.c | 398 ++++++++++++++++++++++++++++++++------- src/manage.h | 47 ++++- src/manage_pg.c | 13 +- src/manage_sql.c | 206 +++++++++++++++++++- src/manage_sql_secinfo.c | 105 ++++++++--- 5 files changed, 667 insertions(+), 102 deletions(-) diff --git a/src/manage.c b/src/manage.c index c924ca36e..97688b391 100644 --- a/src/manage.c +++ b/src/manage.c @@ -60,6 +60,7 @@ #include "manage_sql_nvts.h" #include "manage_sql_tickets.h" #include "manage_sql_tls_certificates.h" +#include "sql.h" #include "utils.h" #include @@ -86,9 +87,11 @@ #include #include #include +#include #include #include #include +#include #include #undef G_LOG_DOMAIN @@ -3106,6 +3109,138 @@ set_scanner_connection_retry (int new_retry) /* CVE tasks. */ +static int +check_version (const gchar *target, const gchar *start_incl, const gchar *start_excl, const gchar *end_incl, const gchar *end_excl) +{ + int result; + + if (start_incl != NULL) + { + result = cmp_versions (start_incl, target); + if (result == -5) + return -1; + if (result > 0) + { + return 0; + } + } + if (start_excl != NULL) + { + result = cmp_versions (start_excl, target); + if (result == -5) + return -1; + if (result >= 0) + { + return 0; + } + } + + if (end_incl != NULL) + { + result = cmp_versions (end_incl, target); + if (result == -5) + return -1; + if (result < 0) + { + return 0; + } + } + + if (end_excl != NULL) + { + result = cmp_versions (end_excl, target); + if (result == -5) + return -1; + if (result <= 0) + { + return 0; + } + } + + return (1); +} + +static void +check_cpe_match_rule (long long int node, gboolean *match, gboolean *vulnerable, report_host_t report_host, const char *host_cpe) +{ + iterator_t cpe_match_node_childs; + gchar *operator; + operator = sql_string ("SELECT operator FROM scap.cpe_match_nodes WHERE id = %llu", node); + init_cpe_match_node_childs_iterator (&cpe_match_node_childs, node); + while (next (&cpe_match_node_childs)) + { + long long int child_node; + child_node = cpe_match_node_childs_iterator_id (&cpe_match_node_childs); + check_cpe_match_rule (child_node, match, vulnerable, report_host, host_cpe); + if (strcmp (operator, "AND") == 0 && !(*match)) + return; + if (strcmp (operator, "OR") == 0 && (*match) && (*vulnerable)) + return; + } + iterator_t cpe_match_ranges; + init_cpe_match_range_iterator (&cpe_match_ranges, node); + while (next (&cpe_match_ranges)) + { + iterator_t cpe_host_details_products; + gchar *range_fs_cpe; + gchar *range_uri_product; + gchar *vsi, *vse, *vei, *vee; + range_fs_cpe = vsi = vse = vei = vee = NULL; + range_fs_cpe = g_strdup (cpe_match_range_iterator_cpe (&cpe_match_ranges)); + vsi = (gchar*) cpe_match_range_iterator_version_start_incl(&cpe_match_ranges); + if (vsi != NULL) + vsi = g_strdup (cpe_match_range_iterator_version_start_incl(&cpe_match_ranges)); + vse = g_strdup (cpe_match_range_iterator_version_start_excl(&cpe_match_ranges)); + vei = g_strdup (cpe_match_range_iterator_version_end_incl(&cpe_match_ranges)); + vee = g_strdup (cpe_match_range_iterator_version_end_excl(&cpe_match_ranges)); + range_uri_product = fs_cpe_to_uri_product (range_fs_cpe); + init_host_details_cpe_product_iterator (&cpe_host_details_products, range_uri_product, report_host); + while (next (&cpe_host_details_products)) + { + cpe_struct_t source, target; + const char *host_details_cpe; + gboolean matches; + host_details_cpe = host_details_cpe_product_iterator_value(&cpe_host_details_products); + cpe_struct_init (&source); + cpe_struct_init (&target); + fs_cpe_to_cpe_struct (range_fs_cpe, &source); + uri_cpe_to_cpe_struct (host_details_cpe, &target); + matches = cpe_struct_match (source, target); + if (matches) + { + int result; + result = check_version (target.version, vsi, vse, vei, vee); + if (result == 1) + *match = TRUE; + } + cpe_struct_free (&source); + cpe_struct_free (&target); + } + if (*match && cpe_match_range_iterator_vulnerable(&cpe_match_ranges) == 1) + { + cpe_struct_t source, target; + cpe_struct_init (&source); + cpe_struct_init (&target); + fs_cpe_to_cpe_struct (range_fs_cpe, &source); + uri_cpe_to_cpe_struct (host_cpe, &target); + if (cpe_struct_match (source, target)) + *vulnerable = TRUE; + cpe_struct_free (&source); + cpe_struct_free (&target); + } + g_free (range_uri_product); + g_free (range_fs_cpe); + g_free (vsi); + g_free (vse); + g_free (vei); + g_free (vee); + if (strcmp (operator, "AND") == 0 && !(*match)) + return; + if (strcmp (operator, "OR") == 0 && (*match) && (*vulnerable)) + return; + } +} + /** * @brief Perform a CVE "scan" on a host. * @@ -3161,89 +3296,218 @@ cve_scan_host (task_t task, report_t report, gvm_host_t *gvm_host) results = g_array_new (TRUE, TRUE, sizeof (result_t)); start_time = time (NULL); prognosis_report_host = 0; - init_host_prognosis_iterator (&prognosis, report_host); - while (next (&prognosis)) + + gboolean use_json = FALSE; + if (sql_int64_0 ("SELECT count(*) FROM scap.cpe_match_nodes") > 0) + use_json = TRUE; + + if (use_json) { - const char *app, *cve; - double severity; - gchar *desc; - iterator_t locations_iter; - GString *locations; - result_t result; + iterator_t host_details_cpe; + init_host_details_cpe_iterator (&host_details_cpe, report_host); + while (next (&host_details_cpe)) + { + iterator_t cpe_match_root_node; + iterator_t locations_iter; + result_t result; + char *cpe_product; + const char *host_cpe; + double severity; + + host_cpe = host_details_cpe_iterator_cpe (&host_details_cpe); + cpe_product = uri_cpe_to_fs_product (host_cpe); + init_cpe_match_nodes_iterator (&cpe_match_root_node, cpe_product); + while (next (&cpe_match_root_node)) + { + result_t root_node; + gboolean match, vulnerable; + const char *app, *cve; + + vulnerable = FALSE; + match = FALSE; + root_node = cpe_match_nodes_iterator_root_id (&cpe_match_root_node); + check_cpe_match_rule (root_node, &match, &vulnerable, report_host, host_cpe); + if (match && vulnerable) + { + GString *locations; + gchar *desc; + + if (prognosis_report_host == 0) + prognosis_report_host = manage_report_host_add (report, + ip, + start_time, + 0); + + severity = sql_double ("SELECT severity FROM scap.cves, scap.cpe_match_nodes" + " WHERE scap.cves.id = scap.cpe_match_nodes.cve_id" + " AND scap.cpe_match_nodes.id = %llu;", + root_node); + + app = host_cpe; + cve = sql_string ("SELECT name FROM scap.cves, scap.cpe_match_nodes" + " WHERE scap.cves.id = cpe_match_nodes.cve_id" + " AND scap.cpe_match_nodes.id = %llu;", + root_node); + locations = g_string_new(""); + + insert_report_host_detail (global_current_report, ip, "cve", cve, + "CVE Scanner", "App", app, NULL); + + init_app_locations_iterator (&locations_iter, report_host, app); + + while (next (&locations_iter)) + { + const char *location; + location = app_locations_iterator_location (&locations_iter); + + if (location == NULL) + { + g_warning ("%s: Location is null for ip %s, app %s", + __func__, ip, app); + continue; + } + + if (locations->len) + { + g_string_append (locations, ", "); + } + g_string_append (locations, location); + + insert_report_host_detail (report, ip, "cve", cve, + "CVE Scanner", app, location, NULL); + + insert_report_host_detail (report, ip, "cve", cve, + "CVE Scanner", "detected_at", + location, NULL); + + insert_report_host_detail (report, ip, "cve", cve, + "CVE Scanner", "detected_by", + /* Detected by itself. */ + cve, NULL); + } + + const char *description; + description = sql_string ("SELECT description FROM scap.cves, scap.cpe_match_nodes" + " WHERE scap.cves.id = scap.cpe_match_nodes.cve_id" + " AND scap.cpe_match_nodes.id = %llu;", + root_node); - if (prognosis_report_host == 0) - prognosis_report_host = manage_report_host_add (report, - ip, - start_time, - 0); + desc = g_strdup_printf ("The host carries the product: %s\n" + "It is vulnerable according to: %s.\n" + "%s%s%s" + "\n" + "%s", + app, + cve, + locations->len + ? "The product was found at: " + : "", + locations->len ? locations->str : "", + locations->len ? ".\n" : "", + description); - severity = prognosis_iterator_cvss_double (&prognosis); + g_debug ("%s: making result with severity %1.1f desc [%s]", + __func__, severity, desc); - app = prognosis_iterator_cpe (&prognosis); - cve = prognosis_iterator_cve (&prognosis); - locations = g_string_new(""); + result = make_cve_result (task, ip, cve, severity, desc); + g_free (desc); - insert_report_host_detail (global_current_report, ip, "cve", cve, - "CVE Scanner", "App", app, NULL); + g_array_append_val (results, result); - init_app_locations_iterator (&locations_iter, report_host, app); + g_string_free (locations, TRUE); - while (next (&locations_iter)) + } + } + g_free (cpe_product); + } + cleanup_iterator (&host_details_cpe); + } + + if (!use_json) + { + init_host_prognosis_iterator (&prognosis, report_host); + while (next (&prognosis)) { - const char *location; - location = app_locations_iterator_location (&locations_iter); + const char *app, *cve; + double severity; + gchar *desc; + iterator_t locations_iter; + GString *locations; + result_t result; + + if (prognosis_report_host == 0) + prognosis_report_host = manage_report_host_add (report, + ip, + start_time, + 0); + + severity = prognosis_iterator_cvss_double (&prognosis); + + app = prognosis_iterator_cpe (&prognosis); + cve = prognosis_iterator_cve (&prognosis); + locations = g_string_new(""); + + insert_report_host_detail (global_current_report, ip, "cve", cve, + "CVE Scanner", "App", app, NULL); - if (location == NULL) + init_app_locations_iterator (&locations_iter, report_host, app); + + while (next (&locations_iter)) { - g_warning ("%s: Location is null for ip %s, app %s", - __func__, ip, app); - continue; - } + const char *location; + location = app_locations_iterator_location (&locations_iter); + + if (location == NULL) + { + g_warning ("%s: Location is null for ip %s, app %s", + __func__, ip, app); + continue; + } - if (locations->len) - g_string_append (locations, ", "); - g_string_append (locations, location); + if (locations->len) + g_string_append (locations, ", "); + g_string_append (locations, location); - insert_report_host_detail (report, ip, "cve", cve, - "CVE Scanner", app, location, NULL); + insert_report_host_detail (report, ip, "cve", cve, + "CVE Scanner", app, location, NULL); - insert_report_host_detail (report, ip, "cve", cve, - "CVE Scanner", "detected_at", - location, NULL); + insert_report_host_detail (report, ip, "cve", cve, + "CVE Scanner", "detected_at", + location, NULL); - insert_report_host_detail (report, ip, "cve", cve, - "CVE Scanner", "detected_by", - /* Detected by itself. */ - cve, NULL); - } + insert_report_host_detail (report, ip, "cve", cve, + "CVE Scanner", "detected_by", + /* Detected by itself. */ + cve, NULL); + } - desc = g_strdup_printf ("The host carries the product: %s\n" - "It is vulnerable according to: %s.\n" - "%s%s%s" - "\n" - "%s", - app, - cve, - locations->len - ? "The product was found at: " - : "", - locations->len ? locations->str : "", - locations->len ? ".\n" : "", - prognosis_iterator_description - (&prognosis)); - - g_debug ("%s: making result with severity %1.1f desc [%s]", - __func__, severity, desc); - - result = make_cve_result (task, ip, cve, severity, desc); - g_free (desc); - - g_array_append_val (results, result); - - g_string_free (locations, TRUE); + desc = g_strdup_printf ("The host carries the product: %s\n" + "It is vulnerable according to: %s.\n" + "%s%s%s" + "\n" + "%s", + app, + cve, + locations->len + ? "The product was found at: " + : "", + locations->len ? locations->str : "", + locations->len ? ".\n" : "", + prognosis_iterator_description + (&prognosis)); + + g_debug ("%s: making result with severity %1.1f desc [%s]", + __func__, severity, desc); + + result = make_cve_result (task, ip, cve, severity, desc); + g_free (desc); + + g_array_append_val (results, result); + + g_string_free (locations, TRUE); + } + cleanup_iterator (&prognosis); } - cleanup_iterator (&prognosis); - report_add_results_array (report, results); g_array_free (results, TRUE); diff --git a/src/manage.h b/src/manage.h index 0ee2e9c23..226f56753 100644 --- a/src/manage.h +++ b/src/manage.h @@ -1688,7 +1688,52 @@ void init_app_locations_iterator (iterator_t*, report_host_t, const gchar *); const char * -app_locations_iterator_location (iterator_t *); +app_locations_iterator_location (iterator_t*); + +void +init_cpe_match_nodes_iterator (iterator_t*, const char *); + +long long int +cpe_match_nodes_iterator_root_id (iterator_t*); + +void +init_host_details_cpe_iterator (iterator_t*, report_host_t); + +const char* +host_details_cpe_iterator_cpe (iterator_t*); + +void +init_cpe_match_node_childs_iterator (iterator_t*, long long int); + +long long int +cpe_match_node_childs_iterator_id (iterator_t*); + +void +init_cpe_match_range_iterator (iterator_t*, long long int); + +const char* +cpe_match_range_iterator_cpe (iterator_t*); + +const char* +cpe_match_range_iterator_version_start_incl (iterator_t*); + +const char* +cpe_match_range_iterator_version_start_excl (iterator_t*); + +const char* +cpe_match_range_iterator_version_end_incl (iterator_t*); + +const char* +cpe_match_range_iterator_version_end_excl (iterator_t*); + +int +cpe_match_range_iterator_vulnerable (iterator_t*); + +void +init_host_details_cpe_product_iterator (iterator_t*, const char *, report_host_t); + +const char* +host_details_cpe_product_iterator_value (iterator_t*); void init_host_prognosis_iterator (iterator_t*, report_host_t); diff --git a/src/manage_pg.c b/src/manage_pg.c index dba91d68f..e4e8ca4b8 100644 --- a/src/manage_pg.c +++ b/src/manage_pg.c @@ -1589,6 +1589,8 @@ manage_create_sql_functions () " THEN $1 = 0" " WHEN 'false'" " THEN $1 = -1" + " WHEN 'error'" + " THEN $1 = -3" " ELSE 0::boolean" " END);" "$$ LANGUAGE SQL" @@ -3533,6 +3535,7 @@ manage_db_init (const gchar *name) sql ("CREATE TABLE scap2.cpe_match_nodes" " (id SERIAL PRIMARY KEY," " parent_id INTEGER DEFAULT 0," + " root_id INTEGER DEFAULT 0," " cve_id INTEGER DEFAULT 0," " operator text);"); @@ -3540,11 +3543,11 @@ manage_db_init (const gchar *name) " (id SERIAL PRIMARY KEY," " node_id INTEGER DEFAULT 0," " vulnerable INTEGER DEFAULT 0," - " cpe text," - " version_start_incl text," - " version_start_excl text," - " version_end_incl text," - " version_end_excl text);"); + " cpe text DEFAULT NULL," + " version_start_incl text DEFAULT NULL," + " version_start_excl text DEFAULT NULL," + " version_end_incl text DEFAULT NULL," + " version_end_excl text DEFAULT NULL);"); sql ("CREATE TABLE scap2.cpe_details" " (id SERIAL PRIMARY KEY," diff --git a/src/manage_sql.c b/src/manage_sql.c index c90a3a882..a3b67eb7b 100644 --- a/src/manage_sql.c +++ b/src/manage_sql.c @@ -20446,6 +20446,198 @@ app_locations_iterator_location (iterator_t *iterator) return iterator_string (iterator, 0); } +/** + * @brief Initialize an iterator of CPEs for a report's host. + * + * @param[in] iterator Iterator. + * @param[in] report_host Report host. + */ +void +init_host_details_cpe_iterator (iterator_t* iterator, report_host_t report_host) +{ + init_iterator (iterator, + "SELECT LOWER (value) FROM report_host_details" + " WHERE name = 'App' and report_host = %llu;", + report_host); +} + +/** + * @brief Get a CPE from an CPE iterator. + * + * @param[in] iterator Iterator. + * + * @return The CPE. + */ +DEF_ACCESS (host_details_cpe_iterator_cpe, 0); + +/** + * @brief Initialize an iterator of CPEs for a product of a report's host. + * + * @param[in] iterator Iterator. + * @param[in] product The product for which to get the CPEs. + * @param[in] report_host Report host. + */ +void +init_host_details_cpe_product_iterator (iterator_t* iterator, const char *product, report_host_t report_host) +{ + gchar *quoted_product; + quoted_product = sql_quote (product); + init_iterator (iterator, + "SELECT DISTINCT LOWER (value) FROM report_host_details" + " WHERE name = 'App' AND report_host = %llu" + " AND value like '%s%s';", + report_host, quoted_product, "%"); + g_free (quoted_product); +} + +/** + * @brief Get a CPE from an CPE product iterator. + * + * @param[in] iterator Iterator. + * + * @return The CPE. + */ +DEF_ACCESS (host_details_cpe_product_iterator_value, 0); + +/** + * @brief Initialize an iterator of root_ids of CPE match nodes. + * + * @param[in] iterator Iterator. + * @param[in] cpe The cpe contained in the match nodes. + */ +void +init_cpe_match_nodes_iterator (iterator_t* iterator, const char *cpe) +{ + gchar *quoted_cpe; + quoted_cpe = sql_quote (cpe); + init_iterator (iterator, + "SELECT DISTINCT root_id" + " FROM scap.cpe_match_nodes, scap.cpe_match_range" + " WHERE cpe like '%s%s' AND scap.cpe_match_nodes.id = node_id;", + quoted_cpe, "%"); + g_free (quoted_cpe); +} + +/** + * @brief Get a root id from an CPE match node iterator. + * + * @param[in] iterator Iterator. + * + * @return The root id. + */ +long long int +cpe_match_nodes_iterator_root_id (iterator_t* iterator) +{ + return iterator_int64 (iterator, 0); +} + +/** + * @brief Initialize an iterator of childs of an CPE match node. + * + * @param[in] iterator Iterator. + * @param[in] node The match node with the childs. + */ +void +init_cpe_match_node_childs_iterator (iterator_t* iterator, long long int node) +{ + init_iterator (iterator, + "SELECT id FROM scap.cpe_match_nodes" + " WHERE parent_id = %llu;", + node); +} + +/** + * @brief Get a child from an CPE match node childs iterator. + * + * @param[in] iterator Iterator. + * + * @return The id of the child node. + */ +long long int +cpe_match_node_childs_iterator_id (iterator_t* iterator) +{ + return iterator_int64 (iterator, 0); +} + +/** + * @brief Initialize an iterator of match ranges of an CPE match node. + * + * @param[in] iterator Iterator. + * @param[in] node The match node with match ranges. + */ +void +init_cpe_match_range_iterator (iterator_t* iterator, long long int node) +{ + init_iterator (iterator, + "SELECT vulnerable, cpe, version_start_incl," + " version_start_excl, version_end_incl, version_end_excl" + " FROM scap.cpe_match_range" + " WHERE node_id = %llu;", + node); +} + +/** + * @brief Return if the CPE of the actual match node is vulnerable. + * + * @param[in] iterator Iterator. + * + * @return 1 if the match node is vulnerable, 0 otherwise. + */ +int +cpe_match_range_iterator_vulnerable (iterator_t* iterator) +{ + return iterator_int64 (iterator, 0); +} + +/** + * @brief Return the CPE of the actual match node. + * + * @param[in] iterator Iterator. + * + * @return The CPE of the actual match node. + */ +DEF_ACCESS (cpe_match_range_iterator_cpe, 1); + +/** + * @brief Return the start included version of the actual match node. + * + * @param[in] iterator Iterator. + * + * @return The start included version of the actual match node, if any. + * NULL otherwise. + */ +DEF_ACCESS (cpe_match_range_iterator_version_start_incl, 2); + +/** + * @brief Return the start excluded version of the actual match node. + * + * @param[in] iterator Iterator. + * + * @return The start excluded version of the actual match node, if any. + * NULL otherwise. + */ +DEF_ACCESS (cpe_match_range_iterator_version_start_excl, 3); + +/** + * @brief Return the end included version of the actual match node. + * + * @param[in] iterator Iterator. + * + * @return The end included version of the actual match node, if any. + * NULL otherwise. + */ +DEF_ACCESS (cpe_match_range_iterator_version_end_incl, 4); + +/** + * @brief Return the end excluded version of the actual match node. + * + * @param[in] iterator Iterator. + * + * @return The end excluded version of the actual match node, if any. + * NULL otherwise. + */ +DEF_ACCESS (cpe_match_range_iterator_version_end_excl, 5); + /** * @brief Initialise a report host prognosis iterator. * @@ -22357,6 +22549,16 @@ where_levels_auto (const char *levels, const char *new_severity_sql) g_string_append (levels_sql, ", 'false'"); count++; } + if (strchr (levels, 'e')) + { + g_string_append (levels_sql, ", 'error'"); + count++; + } + if (strchr (levels, 'd')) + { + g_string_append (levels_sql, ", 'error'"); + count++; + } if (count == 0) { @@ -23086,6 +23288,8 @@ results_extra_where (int trash, report_t report, const gchar* host, min_qod_clause = where_qod (min_qod); + g_message ("PROTO1: %s", levels); + fflush (NULL); levels_clause = where_levels_auto (levels ? levels : "hmlgdf", given_new_severity_sql ? given_new_severity_sql @@ -23099,7 +23303,7 @@ results_extra_where (int trash, report_t report, const gchar* host, extra_where = g_strdup_printf("%s%s%s%s%s", report_clause ? report_clause : "", host_clause ? host_clause : "", - levels_clause->str, + (levels_clause && levels_clause->str) ? levels_clause->str : "", min_qod_clause ? min_qod_clause : "", compliance_levels_clause ?: ""); diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index d189d83bb..e592f4606 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -2703,7 +2703,9 @@ json_object_item_double (cJSON *object, char *key, double fallback) * @brief Save the node of a cve match rule tree. * * @param[in] parent_id The parent_id of the node. If this value is 0, - * this node is the root of the tree. + * the node is the root of the tree. + * @param[in] root_id The id of the root of the tree. If this value + * is 0, the node is the root of the tree. * @param[in] cve_id The id of the CVE to which the tree belongs. * @param[in] operator The operator for the match rules. * @@ -2737,7 +2739,6 @@ add_cpe_match_rules (result_t id, cJSON *match_rules) cJSON *cpe_js; gboolean vulnerable = FALSE; - char * cpe = NULL; char * version_start_incl = NULL; char * version_start_excl = NULL; char * version_end_incl = NULL; @@ -2745,9 +2746,9 @@ add_cpe_match_rules (result_t id, cJSON *match_rules) cJSON_ArrayForEach(match_rule, match_rules) { - char *quoted_cpe; + char *quoted_cpe = NULL; + char *sql_cpe = NULL; vulnerable = FALSE; - cpe = NULL; version_start_incl = NULL; version_start_excl = NULL; version_end_incl = NULL; @@ -2757,22 +2758,40 @@ add_cpe_match_rules (result_t id, cJSON *match_rules) vulnerable = TRUE; else vulnerable = FALSE; + cpe_js = cJSON_GetObjectItemCaseSensitive(match_rule, "cpe23Uri"); - if (cpe_js != NULL) - cpe = cpe_js->valuestring; - quoted_cpe = sql_quote (cpe); + if (cpe_js != NULL && strcmp (cpe_js->valuestring, "(null)")) + { + quoted_cpe = sql_quote (cpe_js->valuestring); + sql_cpe = g_strdup_printf ("'%s'", quoted_cpe); + g_free (quoted_cpe); + } + else + sql_cpe = g_strdup ("NULL"); + ver_se = cJSON_GetObjectItemCaseSensitive(match_rule, "versionStartIncluding"); - if (ver_se != NULL) - version_start_incl = ver_se->valuestring; + if (ver_se != NULL && strcmp (ver_se->valuestring, "(null)")) + version_start_incl = g_strdup_printf ("'%s'", ver_se->valuestring); + else + version_start_incl = g_strdup ("NULL"); + ver_se = cJSON_GetObjectItemCaseSensitive(match_rule, "versionStartExcluding"); - if (ver_se != NULL) - version_start_excl = ver_se->valuestring; + if (ver_se != NULL && strcmp (ver_se->valuestring, "(null)")) + version_start_excl = g_strdup_printf ("'%s'", ver_se->valuestring); + else + version_start_excl = g_strdup ("NULL"); + ver_se = cJSON_GetObjectItemCaseSensitive(match_rule, "versionEndIncluding"); - if (ver_se != NULL) - version_end_incl = ver_se->valuestring; + if (ver_se != NULL && strcmp (ver_se->valuestring, "(null)")) + version_end_incl = g_strdup_printf ("'%s'", ver_se->valuestring); + else + version_end_incl = g_strdup ("NULL"); + ver_se = cJSON_GetObjectItemCaseSensitive(match_rule, "versionEndExcluding"); - if (ver_se != NULL) - version_end_excl = ver_se->valuestring; + if (ver_se != NULL && strcmp (ver_se->valuestring, "(null)")) + version_end_excl = g_strdup_printf ("'%s'", ver_se->valuestring); + else + version_end_excl = g_strdup ("NULL"); sql ("INSERT INTO scap2.cpe_match_range" @@ -2780,30 +2799,51 @@ add_cpe_match_rules (result_t id, cJSON *match_rules) " version_start_incl, version_start_excl," " version_end_incl, version_end_excl)" " VALUES" - " (%llu, %d, '%s', '%s', '%s', '%s', '%s')", + " (%llu, %d, %s, %s, %s, %s, %s)", id, vulnerable ? 1 : 0, - quoted_cpe, - version_start_incl, - version_start_excl, - version_end_incl, - version_end_excl); - g_free (quoted_cpe); + sql_cpe ? sql_cpe : "", + version_start_incl ? version_start_incl : "", + version_start_excl ? version_start_excl : "", + version_end_incl ? version_end_incl : "", + version_end_excl ? version_end_excl : ""); + + g_free (sql_cpe); + g_free (version_start_incl); + g_free (version_start_excl); + g_free (version_end_incl); + g_free (version_end_excl); } } +/** + * @brief Set the root id for a node of a cve match rule tree. + * + * @param[in] id The id of the node for which the root id is to be set. + * @param[in] root_id The id of the root of the tree this node belongs to. + */ +static void +set_root_id (long int id, long int root_id) +{ + sql ("UPDATE scap2.cpe_match_nodes set root_id = %i" + " WHERE id = %i;", + root_id, + id); +} + /** * @brief Load and add recursively all nodes of a match rule tree for a * specific CVE. Build a match rule tree. * - * @param[in] parent_id The parent_id of the nodes to insert + * @param[in] parent_id The parent id of the nodes to insert * (0 for the root node). + * @param[in] root_id The root id of the nodes to insert * @param[in] cveid The id of the CVE the tree belongs to. * @param[in] nodes The JSON object that contains the rules for a * specific tree level. */ static void -load_nodes (resource_t parent_id, resource_t cveid, cJSON *nodes) +load_nodes (resource_t parent_id, resource_t cveid, resource_t root_id, cJSON *nodes) { cJSON *node; resource_t id; @@ -2823,13 +2863,22 @@ load_nodes (resource_t parent_id, resource_t cveid, cJSON *nodes) cJSON_ArrayForEach(node, nodes) { operator = cJSON_GetObjectItemCaseSensitive(node, "operator"); - if (operator) - id = save_node (parent_id, cveid, operator->valuestring); + if (operator && operator->valuestring) + { + id = save_node (parent_id, cveid, operator->valuestring); + } + else + return; + + if (parent_id == 0) + root_id = id; + set_root_id (id, root_id); + cpe_match_rules = cJSON_GetObjectItemCaseSensitive(node, "cpe_match"); if (cpe_match_rules) add_cpe_match_rules (id, cpe_match_rules); child_nodes = cJSON_GetObjectItemCaseSensitive(node, "children"); - load_nodes (id, cveid, child_nodes); + load_nodes (id, cveid, root_id, child_nodes); } } @@ -3007,7 +3056,7 @@ handle_json_cve_item (cJSON *item) g_warning("%s: nodes missing for %s.", __func__, cve_id); return -1; } - load_nodes (0, cve_db_id, nodes_json); + load_nodes (0, cve_db_id, 0, nodes_json); return 0; } From d0c6ffd8c925476de5a50bc89bcd171896f0f85f Mon Sep 17 00:00:00 2001 From: Johannes Helmold Date: Tue, 24 Sep 2024 10:14:33 +0200 Subject: [PATCH 02/26] Small amendment. --- src/manage.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/manage.c b/src/manage.c index 97688b391..54b549754 100644 --- a/src/manage.c +++ b/src/manage.c @@ -3298,7 +3298,9 @@ cve_scan_host (task_t task, report_t report, gvm_host_t *gvm_host) prognosis_report_host = 0; gboolean use_json = FALSE; - if (sql_int64_0 ("SELECT count(*) FROM scap.cpe_match_nodes") > 0) + if (sql_int64_0 ("SELECT count(1) FROM information_schema.tables" + " WHERE table_schema = 'scap'" + " AND table_name = 'cpe_match_nodes';") > 0) use_json = TRUE; if (use_json) From 271bd1e7ad20292afb2c5c4aaee0d1eba3775343 Mon Sep 17 00:00:00 2001 From: Johannes Helmold Date: Wed, 25 Sep 2024 13:24:27 +0200 Subject: [PATCH 03/26] Small amendment. --- src/manage_sql.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/manage_sql.c b/src/manage_sql.c index 5b19f2237..c468b3b5e 100644 --- a/src/manage_sql.c +++ b/src/manage_sql.c @@ -20453,10 +20453,10 @@ app_locations_iterator_location (iterator_t *iterator) * @param[in] report_host Report host. */ void -init_host_details_cpe_iterator (iterator_t* iterator, report_host_t report_host) +init_host_details_cpe_iterator (iterator_t *iterator, report_host_t report_host) { init_iterator (iterator, - "SELECT LOWER (value) FROM report_host_details" + "SELECT DISTINCT LOWER (value) FROM report_host_details" " WHERE name = 'App' and report_host = %llu;", report_host); } From ed5123e29077dd2a5e73fc81326f3e2cab33ad31 Mon Sep 17 00:00:00 2001 From: Johannes Helmold Date: Fri, 18 Oct 2024 16:06:09 +0200 Subject: [PATCH 04/26] Some amendments. --- src/manage.c | 316 +++++++++++++++++++++------------------ src/manage_sql.c | 11 +- src/manage_sql_secinfo.c | 10 +- 3 files changed, 176 insertions(+), 161 deletions(-) diff --git a/src/manage.c b/src/manage.c index 54b549754..159d5fa60 100644 --- a/src/manage.c +++ b/src/manage.c @@ -3165,6 +3165,8 @@ check_cpe_match_rule (long long int node, gboolean *match, gboolean *vulnerable, { iterator_t cpe_match_node_childs; gchar *operator; + iterator_t cpe_match_ranges; + operator = sql_string ("SELECT operator FROM scap.cpe_match_nodes WHERE id = %llu", node); init_cpe_match_node_childs_iterator (&cpe_match_node_childs, node); while (next (&cpe_match_node_childs)) @@ -3177,7 +3179,7 @@ check_cpe_match_rule (long long int node, gboolean *match, gboolean *vulnerable, if (strcmp (operator, "OR") == 0 && (*match) && (*vulnerable)) return; } - iterator_t cpe_match_ranges; + init_cpe_match_range_iterator (&cpe_match_ranges, node); while (next (&cpe_match_ranges)) { @@ -3187,12 +3189,10 @@ check_cpe_match_rule (long long int node, gboolean *match, gboolean *vulnerable, gchar *vsi, *vse, *vei, *vee; range_fs_cpe = vsi = vse = vei = vee = NULL; range_fs_cpe = g_strdup (cpe_match_range_iterator_cpe (&cpe_match_ranges)); - vsi = (gchar*) cpe_match_range_iterator_version_start_incl(&cpe_match_ranges); - if (vsi != NULL) - vsi = g_strdup (cpe_match_range_iterator_version_start_incl(&cpe_match_ranges)); - vse = g_strdup (cpe_match_range_iterator_version_start_excl(&cpe_match_ranges)); - vei = g_strdup (cpe_match_range_iterator_version_end_incl(&cpe_match_ranges)); - vee = g_strdup (cpe_match_range_iterator_version_end_excl(&cpe_match_ranges)); + vsi = g_strdup (cpe_match_range_iterator_version_start_incl (&cpe_match_ranges)); + vse = g_strdup (cpe_match_range_iterator_version_start_excl (&cpe_match_ranges)); + vei = g_strdup (cpe_match_range_iterator_version_end_incl (&cpe_match_ranges)); + vee = g_strdup (cpe_match_range_iterator_version_end_excl (&cpe_match_ranges)); range_uri_product = fs_cpe_to_uri_product (range_fs_cpe); init_host_details_cpe_product_iterator (&cpe_host_details_products, range_uri_product, report_host); while (next (&cpe_host_details_products)) @@ -3200,12 +3200,12 @@ check_cpe_match_rule (long long int node, gboolean *match, gboolean *vulnerable, cpe_struct_t source, target; const char *host_details_cpe; gboolean matches; - host_details_cpe = host_details_cpe_product_iterator_value(&cpe_host_details_products); + host_details_cpe = host_details_cpe_product_iterator_value (&cpe_host_details_products); cpe_struct_init (&source); cpe_struct_init (&target); fs_cpe_to_cpe_struct (range_fs_cpe, &source); uri_cpe_to_cpe_struct (host_details_cpe, &target); - matches = cpe_struct_match (source, target); + matches = cpe_struct_match (&source, &target); if (matches) { int result; @@ -3216,18 +3216,18 @@ check_cpe_match_rule (long long int node, gboolean *match, gboolean *vulnerable, cpe_struct_free (&source); cpe_struct_free (&target); } - if (*match && cpe_match_range_iterator_vulnerable(&cpe_match_ranges) == 1) - { - cpe_struct_t source, target; - cpe_struct_init (&source); - cpe_struct_init (&target); - fs_cpe_to_cpe_struct (range_fs_cpe, &source); - uri_cpe_to_cpe_struct (host_cpe, &target); - if (cpe_struct_match (source, target)) - *vulnerable = TRUE; - cpe_struct_free (&source); - cpe_struct_free (&target); - } + if (*match && cpe_match_range_iterator_vulnerable (&cpe_match_ranges) == 1) + { + cpe_struct_t source, target; + cpe_struct_init (&source); + cpe_struct_init (&target); + fs_cpe_to_cpe_struct (range_fs_cpe, &source); + uri_cpe_to_cpe_struct (host_cpe, &target); + if (cpe_struct_match (&source, &target)) + *vulnerable = TRUE; + cpe_struct_free (&source); + cpe_struct_free (&target); + } g_free (range_uri_product); g_free (range_fs_cpe); g_free (vsi); @@ -3241,6 +3241,150 @@ check_cpe_match_rule (long long int node, gboolean *match, gboolean *vulnerable, } } +/** + * @brief Perform the json CVE "scan" for the found report host. + * + * @param[in] task Task. + * @param[in] report The report to add the host, results and details to. + * @param[in] report_host The report host. + * @param[in] ip The ip of the report host. + * @param[in] start_time The start time of the scan. + * + * @param[out] prognosis_report_host The report_host with prognosis results + * and host details. + * @param[out] results The results of the scan. + */ +static void +cve_scan_report_host_json (task_t task, + report_t report, + report_host_t report_host, + gchar *ip, + int start_time, + int *prognosis_report_host, + GArray *results) +{ + iterator_t host_details_cpe; + init_host_details_cpe_iterator (&host_details_cpe, report_host); + while (next (&host_details_cpe)) + { + iterator_t cpe_match_root_node; + iterator_t locations_iter; + result_t result; + char *cpe_product; + const char *host_cpe; + double severity; + + g_message ("PROTO1: WHILE1"); + + host_cpe = host_details_cpe_iterator_cpe (&host_details_cpe); + cpe_product = uri_cpe_to_fs_product (host_cpe); + init_cpe_match_nodes_iterator (&cpe_match_root_node, cpe_product); + while (next (&cpe_match_root_node)) + { + result_t root_node; + gboolean match, vulnerable; + const char *app, *cve; + + vulnerable = FALSE; + match = FALSE; + root_node = cpe_match_nodes_iterator_root_id (&cpe_match_root_node); + check_cpe_match_rule (root_node, &match, &vulnerable, report_host, host_cpe); + if (match && vulnerable) + { + GString *locations; + gchar *desc; + + if (*prognosis_report_host == 0) + *prognosis_report_host = manage_report_host_add (report, + ip, + start_time, + 0); + + severity = sql_double ("SELECT severity FROM scap.cves, scap.cpe_match_nodes" + " WHERE scap.cves.id = scap.cpe_match_nodes.cve_id" + " AND scap.cpe_match_nodes.id = %llu;", + root_node); + + app = host_cpe; + cve = sql_string ("SELECT name FROM scap.cves, scap.cpe_match_nodes" + " WHERE scap.cves.id = cpe_match_nodes.cve_id" + " AND scap.cpe_match_nodes.id = %llu;", + root_node); + locations = g_string_new (""); + + insert_report_host_detail (global_current_report, ip, "cve", cve, + "CVE Scanner", "App", app, NULL); + + init_app_locations_iterator (&locations_iter, report_host, app); + + while (next (&locations_iter)) + { + const char *location; + location = app_locations_iterator_location (&locations_iter); + + if (location == NULL) + { + g_warning ("%s: Location is null for ip %s, app %s", + __func__, ip, app); + continue; + } + + if (locations->len) + { + g_string_append (locations, ", "); + } + g_string_append (locations, location); + + insert_report_host_detail (report, ip, "cve", cve, + "CVE Scanner", app, location, NULL); + + insert_report_host_detail (report, ip, "cve", cve, + "CVE Scanner", "detected_at", + location, NULL); + + insert_report_host_detail (report, ip, "cve", cve, + "CVE Scanner", "detected_by", + /* Detected by itself. */ + cve, NULL); + } + + const char *description; + description = sql_string ("SELECT description FROM scap.cves, scap.cpe_match_nodes" + " WHERE scap.cves.id = scap.cpe_match_nodes.cve_id" + " AND scap.cpe_match_nodes.id = %llu;", + root_node); + + desc = g_strdup_printf ("The host carries the product: %s\n" + "It is vulnerable according to: %s.\n" + "%s%s%s" + "\n" + "%s", + app, + cve, + locations->len + ? "The product was found at: " + : "", + locations->len ? locations->str : "", + locations->len ? ".\n" : "", + description); + + g_debug ("%s: making result with severity %1.1f desc [%s]", + __func__, severity, desc); + + result = make_cve_result (task, ip, cve, severity, desc); + g_free (desc); + + g_array_append_val (results, result); + + g_string_free (locations, TRUE); + + } + } + g_free (cpe_product); + } + cleanup_iterator (&host_details_cpe); +} + /** * @brief Perform a CVE "scan" on a host. * @@ -3297,136 +3441,18 @@ cve_scan_host (task_t task, report_t report, gvm_host_t *gvm_host) start_time = time (NULL); prognosis_report_host = 0; - gboolean use_json = FALSE; if (sql_int64_0 ("SELECT count(1) FROM information_schema.tables" " WHERE table_schema = 'scap'" " AND table_name = 'cpe_match_nodes';") > 0) - use_json = TRUE; - - if (use_json) { - iterator_t host_details_cpe; - init_host_details_cpe_iterator (&host_details_cpe, report_host); - while (next (&host_details_cpe)) - { - iterator_t cpe_match_root_node; - iterator_t locations_iter; - result_t result; - char *cpe_product; - const char *host_cpe; - double severity; - - host_cpe = host_details_cpe_iterator_cpe (&host_details_cpe); - cpe_product = uri_cpe_to_fs_product (host_cpe); - init_cpe_match_nodes_iterator (&cpe_match_root_node, cpe_product); - while (next (&cpe_match_root_node)) - { - result_t root_node; - gboolean match, vulnerable; - const char *app, *cve; - - vulnerable = FALSE; - match = FALSE; - root_node = cpe_match_nodes_iterator_root_id (&cpe_match_root_node); - check_cpe_match_rule (root_node, &match, &vulnerable, report_host, host_cpe); - if (match && vulnerable) - { - GString *locations; - gchar *desc; - - if (prognosis_report_host == 0) - prognosis_report_host = manage_report_host_add (report, - ip, - start_time, - 0); - - severity = sql_double ("SELECT severity FROM scap.cves, scap.cpe_match_nodes" - " WHERE scap.cves.id = scap.cpe_match_nodes.cve_id" - " AND scap.cpe_match_nodes.id = %llu;", - root_node); - - app = host_cpe; - cve = sql_string ("SELECT name FROM scap.cves, scap.cpe_match_nodes" - " WHERE scap.cves.id = cpe_match_nodes.cve_id" - " AND scap.cpe_match_nodes.id = %llu;", - root_node); - locations = g_string_new(""); - - insert_report_host_detail (global_current_report, ip, "cve", cve, - "CVE Scanner", "App", app, NULL); - - init_app_locations_iterator (&locations_iter, report_host, app); - - while (next (&locations_iter)) - { - const char *location; - location = app_locations_iterator_location (&locations_iter); - - if (location == NULL) - { - g_warning ("%s: Location is null for ip %s, app %s", - __func__, ip, app); - continue; - } - - if (locations->len) - { - g_string_append (locations, ", "); - } - g_string_append (locations, location); - - insert_report_host_detail (report, ip, "cve", cve, - "CVE Scanner", app, location, NULL); - - insert_report_host_detail (report, ip, "cve", cve, - "CVE Scanner", "detected_at", - location, NULL); - - insert_report_host_detail (report, ip, "cve", cve, - "CVE Scanner", "detected_by", - /* Detected by itself. */ - cve, NULL); - } - - const char *description; - description = sql_string ("SELECT description FROM scap.cves, scap.cpe_match_nodes" - " WHERE scap.cves.id = scap.cpe_match_nodes.cve_id" - " AND scap.cpe_match_nodes.id = %llu;", - root_node); - - desc = g_strdup_printf ("The host carries the product: %s\n" - "It is vulnerable according to: %s.\n" - "%s%s%s" - "\n" - "%s", - app, - cve, - locations->len - ? "The product was found at: " - : "", - locations->len ? locations->str : "", - locations->len ? ".\n" : "", - description); - - g_debug ("%s: making result with severity %1.1f desc [%s]", - __func__, severity, desc); - - result = make_cve_result (task, ip, cve, severity, desc); - g_free (desc); - - g_array_append_val (results, result); - - g_string_free (locations, TRUE); - - } - } - g_free (cpe_product); - } - cleanup_iterator (&host_details_cpe); + // Use new JSON CVE scan + cve_scan_report_host_json (task, report, report_host, ip, + start_time, &prognosis_report_host, + results); } - - if (!use_json) + else { + // Use XML CVE scan init_host_prognosis_iterator (&prognosis, report_host); while (next (&prognosis)) { diff --git a/src/manage_sql.c b/src/manage_sql.c index c468b3b5e..d0746a186 100644 --- a/src/manage_sql.c +++ b/src/manage_sql.c @@ -20513,8 +20513,8 @@ init_cpe_match_nodes_iterator (iterator_t* iterator, const char *cpe) init_iterator (iterator, "SELECT DISTINCT root_id" " FROM scap.cpe_match_nodes, scap.cpe_match_range" - " WHERE cpe like '%s%s' AND scap.cpe_match_nodes.id = node_id;", - quoted_cpe, "%"); + " WHERE cpe like '%s%%' AND scap.cpe_match_nodes.id = node_id;", + quoted_cpe); g_free (quoted_cpe); } @@ -22554,11 +22554,6 @@ where_levels_auto (const char *levels, const char *new_severity_sql) g_string_append (levels_sql, ", 'error'"); count++; } - if (strchr (levels, 'd')) - { - g_string_append (levels_sql, ", 'error'"); - count++; - } if (count == 0) { @@ -23288,8 +23283,6 @@ results_extra_where (int trash, report_t report, const gchar* host, min_qod_clause = where_qod (min_qod); - g_message ("PROTO1: %s", levels); - fflush (NULL); levels_clause = where_levels_auto (levels ? levels : "hmlgdf", given_new_severity_sql ? given_new_severity_sql diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index 6b552f035..2ecf26ec3 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -2704,8 +2704,6 @@ json_object_item_double (cJSON *object, char *key, double fallback) * * @param[in] parent_id The parent_id of the node. If this value is 0, * the node is the root of the tree. - * @param[in] root_id The id of the root of the tree. If this value - * is 0, the node is the root of the tree. * @param[in] cve_id The id of the CVE to which the tree belongs. * @param[in] operator The operator for the match rules. * @@ -2837,8 +2835,8 @@ set_root_id (long int id, long int root_id) * * @param[in] parent_id The parent id of the nodes to insert * (0 for the root node). - * @param[in] root_id The root id of the nodes to insert * @param[in] cveid The id of the CVE the tree belongs to. + * @param[in] root_id The root id of the nodes to insert. * @param[in] nodes The JSON object that contains the rules for a * specific tree level. */ @@ -2864,14 +2862,12 @@ load_nodes (resource_t parent_id, resource_t cveid, resource_t root_id, cJSON *n { operator = cJSON_GetObjectItemCaseSensitive(node, "operator"); if (operator && operator->valuestring) - { - id = save_node (parent_id, cveid, operator->valuestring); - } + id = save_node (parent_id, cveid, operator->valuestring); else return; if (parent_id == 0) - root_id = id; + root_id = id; set_root_id (id, root_id); cpe_match_rules = cJSON_GetObjectItemCaseSensitive(node, "cpe_match"); From 94377da9dc8e81e49906b38ae0b6246698b022e8 Mon Sep 17 00:00:00 2001 From: Johannes Helmold Date: Fri, 18 Oct 2024 17:18:12 +0200 Subject: [PATCH 05/26] Removed stray (debug) message. --- src/manage.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/manage.c b/src/manage.c index 159d5fa60..fca82209f 100644 --- a/src/manage.c +++ b/src/manage.c @@ -3274,8 +3274,6 @@ cve_scan_report_host_json (task_t task, const char *host_cpe; double severity; - g_message ("PROTO1: WHILE1"); - host_cpe = host_details_cpe_iterator_cpe (&host_details_cpe); cpe_product = uri_cpe_to_fs_product (host_cpe); init_cpe_match_nodes_iterator (&cpe_match_root_node, cpe_product); From 02cb67fd6983ae792946081149b9824187952397 Mon Sep 17 00:00:00 2001 From: Johannes Helmold Date: Mon, 21 Oct 2024 13:20:59 +0200 Subject: [PATCH 06/26] Add: Added reading of gzip files for CVEs and EPSS. --- src/manage_sql_secinfo.c | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index a12e4f256..9496964c1 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -51,6 +51,7 @@ #include #include #include +#include #include #include #include @@ -3046,7 +3047,7 @@ update_cve_json (const gchar *cve_path, GHashTable *hashed_cpes) g_info ("Updating %s", full_path); - cve_file = fdopen (fd, "r"); + cve_file = gvm_gzip_open_file_reader_fd (fd); if (cve_file == NULL) { g_warning ("%s: Failed to open CVE file: %s", @@ -3283,7 +3284,8 @@ update_scap_cves () gboolean read_json = FALSE; while ((cve_path = g_dir_read_name (dir))) { - if (fnmatch ("nvdcve-1.1-*.json", cve_path, 0) == 0) + if (fnmatch ("nvdcve-1.1-*.json.gz", cve_path, 0) == 0 || + fnmatch ("nvdcve-1.1-*.json", cve_path, 0) == 0) { read_json = TRUE; break; @@ -3294,7 +3296,9 @@ update_scap_cves () count = 0; while ((cve_path = g_dir_read_name (dir))) { - if ((fnmatch ("nvdcve-1.1-*.json", cve_path, 0) == 0) && read_json) + if ((fnmatch ("nvdcve-1.1-*.json.gz", cve_path, 0) == 0 || + fnmatch ("nvdcve-1.1-*.json", cve_path, 0) == 0) + && read_json) { if (update_cve_json (cve_path, hashed_cpes)) { @@ -3380,10 +3384,19 @@ update_epss_scores () inserts_t inserts; current_json_path = g_build_filename (GVM_SCAP_DATA_DIR, - "epss-scores-current.json", + "epss-scores-current.json.gz", NULL); int fd = open(current_json_path, O_RDONLY); + if (fd < 0 && errno == ENOENT) + { + g_free (current_json_path); + current_json_path = g_build_filename (GVM_SCAP_DATA_DIR, + "epss-scores-current.json", + NULL); + fd = open(current_json_path, O_RDONLY); + } + if (fd < 0) { int ret; @@ -3403,7 +3416,7 @@ update_epss_scores () return ret; } - epss_scores_file = fdopen(fd, "r"); + epss_scores_file = gvm_gzip_open_file_reader_fd (fd); if (epss_scores_file == NULL) { g_warning ("%s: Failed to convert file descriptor to FILE*: %s", From cfc885af072e6a6e0804116d7022f4141c8320a8 Mon Sep 17 00:00:00 2001 From: Johannes Helmold Date: Tue, 22 Oct 2024 10:13:56 +0200 Subject: [PATCH 07/26] Changed location of initialization of "*quoted_cpe". --- src/manage_sql_secinfo.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index 2ecf26ec3..6230408de 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -2744,7 +2744,6 @@ add_cpe_match_rules (result_t id, cJSON *match_rules) cJSON_ArrayForEach(match_rule, match_rules) { - char *quoted_cpe = NULL; char *sql_cpe = NULL; vulnerable = FALSE; version_start_incl = NULL; @@ -2760,7 +2759,7 @@ add_cpe_match_rules (result_t id, cJSON *match_rules) cpe_js = cJSON_GetObjectItemCaseSensitive(match_rule, "cpe23Uri"); if (cpe_js != NULL && strcmp (cpe_js->valuestring, "(null)")) { - quoted_cpe = sql_quote (cpe_js->valuestring); + char *quoted_cpe = sql_quote (cpe_js->valuestring); sql_cpe = g_strdup_printf ("'%s'", quoted_cpe); g_free (quoted_cpe); } From 2168aa40aca3266fe64e538ee6bed4756b7e1535 Mon Sep 17 00:00:00 2001 From: Johannes Helmold Date: Tue, 22 Oct 2024 10:22:26 +0200 Subject: [PATCH 08/26] Removed the option "e" for the display of errors. --- src/manage_sql.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/manage_sql.c b/src/manage_sql.c index d0746a186..fc1bf3905 100644 --- a/src/manage_sql.c +++ b/src/manage_sql.c @@ -22549,11 +22549,6 @@ where_levels_auto (const char *levels, const char *new_severity_sql) g_string_append (levels_sql, ", 'false'"); count++; } - if (strchr (levels, 'e')) - { - g_string_append (levels_sql, ", 'error'"); - count++; - } if (count == 0) { From d0f409e1f496ccdeb10516a6ae8f5b476d8423f5 Mon Sep 17 00:00:00 2001 From: Timo Pollmeier Date: Mon, 30 Sep 2024 11:36:22 +0200 Subject: [PATCH 09/26] Change: Adjust loading of CPEs to new JSON API CPEs can now be loaded from JSON files based on the NVD API. As some fields in the old XML differ the JSON API, they are replaced by similar fields: "nvd_id" is replaced by "cpe_name_id" and "status" is replaced by "deprecated". The "raw_data" will no longer be available after switching to JSON, so the references are and "deprecated_by" element are handled explictly. These changes are made because the XML-based data feeds have been deprecated by NVD. --- CMakeLists.txt | 2 +- src/gmp.c | 35 +- src/manage.h | 18 +- src/manage_pg.c | 12 +- src/manage_sql_secinfo.c | 619 +++++++++++++++++++++++++++--- src/manage_sql_secinfo.h | 6 +- src/schema_formats/XML/GMP.xml.in | 41 +- 7 files changed, 650 insertions(+), 83 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9d3b36a5a..eb6b215d2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -103,7 +103,7 @@ include (CPack) set (GVMD_DATABASE_VERSION 256) -set (GVMD_SCAP_DATABASE_VERSION 21) +set (GVMD_SCAP_DATABASE_VERSION 22) set (GVMD_CERT_DATABASE_VERSION 8) diff --git a/src/gmp.c b/src/gmp.c index 8ede8d6a9..ac95a6f67 100644 --- a/src/gmp.c +++ b/src/gmp.c @@ -13473,24 +13473,33 @@ handle_get_info (gmp_parser_t *gmp_parser, GError **error) "%s", cpe_info_iterator_title (&info)); xml_string_append (result, - "%s" + "%s" "%s" "%s" - "%s", - cpe_info_iterator_nvd_id (&info) - ? cpe_info_iterator_nvd_id (&info) + "%s", + cpe_info_iterator_cpe_name_id (&info) + ? cpe_info_iterator_cpe_name_id (&info) : "", cpe_info_iterator_severity (&info) ? cpe_info_iterator_severity (&info) : "", cpe_info_iterator_cve_refs (&info), - cpe_info_iterator_status (&info) - ? cpe_info_iterator_status (&info) - : ""); + cpe_info_iterator_deprecated (&info) + ? cpe_info_iterator_deprecated (&info) + : "0"); if (get_info_data->details == 1) { - iterator_t cves; + const char *deprecated_by_id + = cpe_info_iterator_deprecated_by_id (&info); + if (deprecated_by_id && strcmp (deprecated_by_id, "")) + { + xml_string_append (result, + "%s", + deprecated_by_id); + } + + iterator_t cves, refs; g_string_append (result, ""); init_cpe_cve_iterator (&cves, get_iterator_name (&info), 0, NULL); while (next (&cves)) @@ -13518,6 +13527,16 @@ handle_get_info (gmp_parser_t *gmp_parser, GError **error) : ""); cleanup_iterator (&cves); g_string_append (result, ""); + + g_string_append (result, ""); + init_cpe_reference_iterator (&refs, get_iterator_name (&info)); + while (next (&refs)) + xml_string_append (result, + "%s", + cpe_reference_iterator_href (&refs), + cpe_reference_iterator_type (&refs)); + cleanup_iterator (&refs); + g_string_append (result, ""); } } else if (g_strcmp0 ("cve", get_info_data->type) == 0) diff --git a/src/manage.h b/src/manage.h index 0ee2e9c23..6d34f02f9 100644 --- a/src/manage.h +++ b/src/manage.h @@ -3355,23 +3355,33 @@ const char* cpe_info_iterator_title (iterator_t*); const char* -cpe_info_iterator_status (iterator_t*); +cpe_info_iterator_deprecated (iterator_t*); const char * cpe_info_iterator_severity (iterator_t*); const char* -cpe_info_iterator_deprecated_by_id (iterator_t*); +cpe_info_iterator_cve_refs (iterator_t*); const char* -cpe_info_iterator_cve_refs (iterator_t*); +cpe_info_iterator_cpe_name_id (iterator_t*); const char* -cpe_info_iterator_nvd_id (iterator_t*); +cpe_info_iterator_deprecated_by_id (iterator_t*); gchar * cpe_details_xml (const char*); +void +init_cpe_reference_iterator (iterator_t *, const char *); + +const char* +cpe_reference_iterator_href (iterator_t *); + +const char* +cpe_reference_iterator_type (iterator_t *); + + /* CVE. */ const char* diff --git a/src/manage_pg.c b/src/manage_pg.c index dba91d68f..7088a4e9c 100644 --- a/src/manage_pg.c +++ b/src/manage_pg.c @@ -3525,10 +3525,18 @@ manage_db_init (const gchar *name) " modification_time integer," " title text," " status text," - " deprecated_by_id INTEGER," + " deprecated_by_id TEXT," " severity DOUBLE PRECISION DEFAULT 0," " cve_refs INTEGER DEFAULT 0," - " nvd_id text);"); + " nvd_id text," + " deprecated integer," + " cpe_name_id text);"); + + sql ("CREATE TABLE scap2.cpe_refs" + " (id SERIAL PRIMARY KEY," + " cpe INTEGER," + " ref TEXT," + " type TEXT);"); sql ("CREATE TABLE scap2.cpe_match_nodes" " (id SERIAL PRIMARY KEY," diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index a12e4f256..b03413453 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -51,6 +51,7 @@ #include #include #include +#include #include #include #include @@ -377,6 +378,72 @@ inserts_run (inserts_t *inserts, gboolean finalize) inserts_free_statements (inserts); } +/** + * @brief Get the string value for a specified key from a JSON object. + * + * @param[in] object JSON object + * @param[in] key The key of the string in the JSON object. + * + * @return The string out of the JSON object with key "key", if any. + * NULL otherwise. + */ +static char* +json_object_item_string (cJSON *object, char *key) +{ + cJSON *value_json; + + value_json = cJSON_GetObjectItemCaseSensitive(object, key); + if (cJSON_IsString(value_json)) + return value_json->valuestring; + return NULL; +} + +/** + * @brief Get the double value for a specified key from a JSON object. + * + * @param[in] object JSON object + * @param[in] key The key of the double value in the JSON object. + * @param[in] fallback The fallback value if the double value is not + * available. + * + * @return The double value out of the JSON object with key "key", if any. + * The fallback value otherwise. + */ +static double +json_object_item_double (cJSON *object, char *key, double fallback) +{ + cJSON *value_json; + + value_json = cJSON_GetObjectItemCaseSensitive(object, key); + if (cJSON_IsNumber(value_json)) + return value_json->valuedouble; + return fallback; +} + +/** + * @brief Get the boolean value for a specified key from a JSON object. + * + * @param[in] object JSON object + * @param[in] key The key of the double value in the JSON object. + * @param[in] fallback The fallback value if the boolean value is not + * available. + * + * @return The double value out of the JSON object with key "key", if any. + * The fallback value otherwise. + */ +static int +json_object_item_boolean (cJSON *object, char *key, int fallback) +{ + cJSON *value_json; + + value_json = cJSON_GetObjectItemCaseSensitive(object, key); + if (cJSON_IsTrue(value_json)) + return 1; + else if (cJSON_IsFalse(value_json)) + return 0; + return fallback; +} + /* CPE data. */ @@ -495,14 +562,25 @@ init_cpe_info_iterator_all (iterator_t* iterator, get_data_t *get) DEF_ACCESS (cpe_info_iterator_title, GET_ITERATOR_COLUMN_COUNT); /** - * @brief Get the status from a CPE iterator. + * @brief Get the deprecation status from a CPE iterator. * * @param[in] iterator Iterator. * - * @return The Status of the CPE, or NULL if iteration is complete. Freed by - * cleanup_iterator. + * @return The deprecation status of the CPE, or NULL if iteration is complete. + * Freed by cleanup_iterator. + */ +DEF_ACCESS (cpe_info_iterator_deprecated, GET_ITERATOR_COLUMN_COUNT + 1); + +/** + * @brief Get the first CPE the current one is deprecated by + * from a CPE iterator. + * + * @param[in] iterator Iterator. + * + * @return The first CPE the current one is deprecated by, + * or NULL if iteration is complete. Freed by cleanup_iterator. */ -DEF_ACCESS (cpe_info_iterator_status, GET_ITERATOR_COLUMN_COUNT + 1); +DEF_ACCESS (cpe_info_iterator_deprecated_by_id, GET_ITERATOR_COLUMN_COUNT + 2); /** * @brief Get the highest severity Score of all CVE's referencing this cpe. @@ -525,14 +603,14 @@ DEF_ACCESS (cpe_info_iterator_severity, GET_ITERATOR_COLUMN_COUNT + 3); DEF_ACCESS (cpe_info_iterator_cve_refs, GET_ITERATOR_COLUMN_COUNT + 4); /** - * @brief Get the NVD ID for this CPE. + * @brief Get the NVD assigned cpeNameId for this CPE. * * @param[in] iterator Iterator. * * @return The NVD ID of this CPE, or NULL if iteration is * complete. Freed by cleanup_iterator. */ -DEF_ACCESS (cpe_info_iterator_nvd_id, GET_ITERATOR_COLUMN_COUNT + 5); +DEF_ACCESS (cpe_info_iterator_cpe_name_id, GET_ITERATOR_COLUMN_COUNT + 5); /** * @brief Get the XML details / raw data for a given CPE ID. @@ -553,6 +631,44 @@ cpe_details_xml (const char *cpe_id) { return details_xml; } +/** + * @brief Initialise a CPE refrerences iterator. + * + * @param[in] iterator Iterator. + * @param[in] cpe CPE to get references of. + */ +void +init_cpe_reference_iterator (iterator_t *iterator, const char *cpe) +{ + gchar *quoted_cpe; + quoted_cpe = sql_quote (cpe); + init_iterator (iterator, + "SELECT ref, type FROM cpe_refs" + " WHERE cpe = (SELECT id FROM cpes WHERE uuid = '%s');", + quoted_cpe); + g_free (quoted_cpe); +} + +/** + * @brief Get the reference URL from CPE reference iterator. + * + * @param[in] iterator Iterator. + * + * @return The reference URL, or NULL if iteration is complete. Freed by + * cleanup_iterator. + */ +DEF_ACCESS (cpe_reference_iterator_href, 0); + +/** + * @brief Get the reference type from CPE reference iterator. + * + * @param[in] iterator Iterator. + * + * @return The reference type, or NULL if iteration is complete. Freed by + * cleanup_iterator. + */ +DEF_ACCESS (cpe_reference_iterator_type, 1); + /* CVE data. */ @@ -1895,6 +2011,29 @@ update_cert_bund_advisories (int last_cert_update) /* SCAP update: CPEs. */ +/** + * @brief Convert a CPE name from formatted string to URI and SQL quote it. + * + * @param[in] name Name. + * + * @return URI converted uoted name. + */ +static gchar * +fs_to_uri_convert_and_quote_cpe_name (const char *name) +{ + gchar *name_converted, *name_decoded, *name_tilde, *quoted_name; + + name_converted = fs_cpe_to_uri_cpe (name); + name_decoded = g_uri_unescape_string (name_converted, NULL); + name_tilde = string_replace (name_decoded, + "~", "%7E", "%7e", NULL); + g_free (name_decoded); + g_free (name_converted); + quoted_name = sql_quote (name_tilde); + g_free (name_tilde); + return quoted_name; +} + /** * @brief Decode and SQL quote a CPE name. * @@ -1917,7 +2056,7 @@ decode_and_quote_cpe_name (const char *name) } /** - * @brief Insert a SCAP CPE. + * @brief Insert a SCAP CPE from XML. * * @param[in] inserts Pointer to SQL buffer. * @param[in] cpe_item CPE item XML element. @@ -2081,14 +2220,408 @@ insert_scap_cpe_details (inserts_t *inserts, element_t cpe_item) } /** - * @brief Update SCAP CPEs from a file. + * @brief Try to skip to the products list in a CPEs JSON parser. + * + * @param[in] parser Parser to skip elements in. + * @param[in] event Parser event structure. + * + * @return 0 success, -1 error. + */ +static int +scap_cpes_json_skip_to_products (gvm_json_pull_parser_t *parser, + gvm_json_pull_event_t *event) +{ + gvm_json_pull_parser_next (parser, event); + if (event->type == GVM_JSON_PULL_EVENT_ERROR) + { + g_warning ("%s: Parser error: %s", __func__, event->error_message); + return -1; + } + else if (event->type != GVM_JSON_PULL_EVENT_OBJECT_START) + { + g_warning ("%s: CPEs file content is not a JSON object.", __func__); + return -1; + } + + gboolean products_found = FALSE; + while (!products_found) + { + gvm_json_pull_parser_next (parser, event); + gvm_json_path_elem_t *path_tail = g_queue_peek_tail (event->path); + if (event->type == GVM_JSON_PULL_EVENT_ARRAY_START && path_tail && + path_tail->key && strcmp (path_tail->key, "products") == 0) + { + products_found = TRUE; + } + else if (event->type == GVM_JSON_PULL_EVENT_ERROR) + { + g_warning ("%s: Parser error: %s", __func__, event->error_message); + return -1; + } + else if (event->type == GVM_JSON_PULL_EVENT_OBJECT_END) + { + g_warning ("%s: Unexpected json object end.", __func__); + return -1; + } + } + gvm_json_pull_parser_next (parser, event); + + return 0; +} + +/** + * @brief Insert a SCAP CPE from JSON. + * + * @param[in] inserts Pointer to SQL buffer. + * @param[in] product_item JSON object from the products list. + * + * @return 0 success, -1 error. + */ +static int +handle_json_cpe_item (inserts_t *inserts, cJSON *product_item) +{ + cJSON *cpe_item; + char *name, *cpe_name_id, *last_modified, *title_text; + char *deprecated_by; + gchar *quoted_name, *quoted_title, *quoted_cpe_name_id; + gchar *quoted_deprecated_by; + cJSON *titles, *title; + time_t modification_time; + int deprecated; + int first; + + assert (inserts); + + cpe_item = cJSON_GetObjectItemCaseSensitive (product_item, "cpe"); + if (! cJSON_IsObject (cpe_item)) + { + g_warning ("%s: 'cpe' field in product missing or not an object", + __func__); + return -1; + } + + name = json_object_item_string (cpe_item, "cpeName"); + if (name == NULL) + { + g_warning ("%s: 'cpeName' field missing or not a string", __func__); + return -1; + } + + cpe_name_id = json_object_item_string (cpe_item, "cpeNameId"); + if (cpe_name_id == NULL) + { + g_warning ("%s: 'cpeNameId' field missing or not a string", __func__); + return -1; + } + + last_modified = json_object_item_string (cpe_item, "lastModified"); + if (last_modified == NULL) + { + g_warning ("%s: 'lastModified' field missing or not a string", __func__); + return -1; + } + modification_time = parse_iso_time (last_modified); + + titles = cJSON_GetObjectItemCaseSensitive (cpe_item, "titles"); + if (! cJSON_IsArray (titles)) + { + g_warning ("%s: 'titles' field missing or not an array", __func__); + return -1; + } + + title_text = NULL; + cJSON_ArrayForEach (title, titles) + { + gchar *lang = json_object_item_string (title, "lang"); + if (lang && strcmp (lang, "en") == 0) + { + title_text = json_object_item_string (title, "title"); + break; + } + } + + deprecated = json_object_item_boolean (cpe_item, "deprecated", -1); + if (deprecated == -1) + { + g_warning ("%s: 'deprecated' field missing or not a boolean", __func__); + return -1; + } + + deprecated_by = NULL; + if (deprecated) + { + /* CPEs can have multiple deprecatedBy entries, + * but for the GMP field only the first one is used */ + cJSON *deprecated_by_array, *first_deprecated_by; + deprecated_by_array = cJSON_GetObjectItemCaseSensitive (cpe_item, + "deprecatedBy"); + if (! cJSON_IsArray (deprecated_by_array)) + { + g_warning ("%s: 'deprecatedBy' field missing or not an array", + __func__); + return -1; + } + else if (cJSON_GetArraySize (deprecated_by_array) == 0) + { + g_warning ("%s: 'deprecatedBy' array is empty", + __func__); + return -1; + } + + first_deprecated_by = cJSON_GetArrayItem (deprecated_by_array, 0); + deprecated_by = json_object_item_string (first_deprecated_by, "cpeName"); + if (deprecated_by == NULL) + { + g_warning ("%s: Could not get 'cpeName' string from 'deprecatedBy'", + __func__); + return -1; + } + } + + quoted_name = fs_to_uri_convert_and_quote_cpe_name (name); + quoted_cpe_name_id = sql_quote (cpe_name_id); + quoted_title = sql_quote (title_text ? title_text : ""); + quoted_deprecated_by + = deprecated_by ? fs_to_uri_convert_and_quote_cpe_name (deprecated_by) + : NULL; + + first = inserts_check_size (inserts); + + g_string_append_printf (inserts->statement, + "%s ('%s', '%s', '%s', %li, %li, %d, '%s', '%s')", + first ? "" : ",", + quoted_name, + quoted_name, + quoted_title, + modification_time, + modification_time, + deprecated, + quoted_deprecated_by ? quoted_deprecated_by : "", + quoted_cpe_name_id); + + inserts->current_chunk_size++; + + g_free (quoted_title); + g_free (quoted_name); + g_free (quoted_cpe_name_id); + g_free (quoted_deprecated_by); + + return 0; +} + +/** + * @brief Insert a SCAP CPE from JSON. + * + * @param[in] inserts Pointer to SQL buffer. + * @param[in] product_item JSON object from the products list. + * + * @return 0 success, -1 error. + */ +static int +handle_json_cpe_refs (inserts_t *inserts, cJSON *product_item) +{ + cJSON *cpe_item, *refs, *refs_item; + gchar *name, *quoted_name; + + assert (inserts); + + cpe_item = cJSON_GetObjectItemCaseSensitive (product_item, "cpe"); + if (! cJSON_IsObject (cpe_item)) + { + g_warning ("%s: 'cpe' field in product missing or not an object", + __func__); + return -1; + } + + name = json_object_item_string (cpe_item, "cpeName"); + if (name == NULL) + { + g_warning ("%s: 'cpeName' field missing or not a string", __func__); + return -1; + } + + refs = cJSON_GetObjectItemCaseSensitive (cpe_item, "refs"); + if (! cJSON_IsArray (refs)) + { + g_debug ("%s: 'refs' field missing or not an array", __func__); + return 0; + } + + quoted_name = fs_to_uri_convert_and_quote_cpe_name (name); + cJSON_ArrayForEach (refs_item, refs) + { + int first; + char *ref, *type; + gchar *quoted_ref, *quoted_type; + ref = json_object_item_string (refs_item, "ref"); + if (ref == NULL) + { + g_warning ("%s: 'ref' field missing or not a string", __func__); + g_free (quoted_name); + return -1; + } + type = json_object_item_string (refs_item, "type"); + quoted_ref = sql_quote (ref ? ref : ""); + quoted_type = sql_quote (type ? type : ""); + + first = inserts_check_size (inserts); + + g_string_append_printf (inserts->statement, + "%s ('%s', '%s', '%s')", + first ? "" : ",", + quoted_name, + quoted_ref, + quoted_type); + + inserts->current_chunk_size++; + } + g_free (quoted_name); + + return 0; +} + +/** + * @brief Update SCAP CPEs from a JSON file. * * @param[in] path Path to file. * * @return 0 success, -1 error. */ static int -update_scap_cpes_from_file (const gchar *path) +update_scap_cpes_from_json_file (const gchar *path) +{ + inserts_t inserts; + gvm_json_pull_parser_t parser; + gvm_json_pull_event_t event; + FILE *json_stream = fopen (path, "r"); + if (json_stream == NULL) + { + g_warning ("%s: Could not open file '%s': %s", + __func__, path, strerror(errno)); + return -1; + } + + gvm_json_pull_parser_init (&parser, json_stream); + gvm_json_pull_event_init (&event); + if (scap_cpes_json_skip_to_products (&parser, &event)) + { + gvm_json_pull_event_cleanup (&event); + gvm_json_pull_parser_cleanup (&parser); + fclose (json_stream); + return -1; + } + + sql_begin_immediate (); + inserts_init (&inserts, + CPE_MAX_CHUNK_SIZE, + setting_secinfo_sql_buffer_threshold_bytes (), + "INSERT INTO scap2.cpes" + " (uuid, name, title, creation_time," + " modification_time, deprecated, deprecated_by_id," + " cpe_name_id)" + " VALUES", + " ON CONFLICT (uuid) DO UPDATE" + " SET name = EXCLUDED.name," + " title = EXCLUDED.title," + " creation_time = EXCLUDED.creation_time," + " modification_time = EXCLUDED.modification_time," + " deprecated = EXCLUDED.deprecated," + " deprecated_by_id = EXCLUDED.deprecated_by_id," + " cpe_name_id = EXCLUDED.cpe_name_id"); + + while (event.type == GVM_JSON_PULL_EVENT_OBJECT_START) + { + gchar *error_message; + cJSON *entry = gvm_json_pull_expand_container (&parser, &error_message); + if (error_message) + { + g_warning ("%s: Error expanding CVE item: %s", __func__, error_message); + gvm_json_pull_event_cleanup (&event); + gvm_json_pull_parser_cleanup (&parser); + cJSON_Delete (entry); + fclose (json_stream); + sql_commit (); + return -1; + } + if (handle_json_cpe_item (&inserts, entry)) + { + gvm_json_pull_event_cleanup (&event); + gvm_json_pull_parser_cleanup (&parser); + cJSON_Delete (entry); + fclose (json_stream); + sql_commit (); + return -1; + } + cJSON_Delete (entry); + gvm_json_pull_parser_next (&parser, &event); + } + inserts_run (&inserts, TRUE); + sql_commit (); + gvm_json_pull_parser_cleanup (&parser); + + // Reset and insert refs + fseek (json_stream, 0, SEEK_SET); + gvm_json_pull_parser_init (&parser, json_stream); + + if (scap_cpes_json_skip_to_products (&parser, &event)) + { + gvm_json_pull_event_cleanup (&event); + gvm_json_pull_parser_cleanup (&parser); + fclose (json_stream); + return -1; + } + + sql_begin_immediate (); + inserts_init (&inserts, 10, + setting_secinfo_sql_buffer_threshold_bytes (), + "INSERT INTO scap2.cpe_refs (cpe, ref, type)" + " SELECT scap2.cpes.id, new_refs.ref, new_refs.type" + " FROM scap2.cpes JOIN (VALUES ", + ") AS new_refs (cpe_name, ref, type)" + " ON scap2.cpes.name = cpe_name;"); + + while (event.type == GVM_JSON_PULL_EVENT_OBJECT_START) + { + gchar *error_message; + cJSON *entry = gvm_json_pull_expand_container (&parser, &error_message); + if (error_message) + { + g_warning ("%s: Error expanding CVE item: %s", __func__, error_message); + gvm_json_pull_event_cleanup (&event); + gvm_json_pull_parser_cleanup (&parser); + cJSON_Delete (entry); + fclose (json_stream); + sql_commit (); + return -1; + } + if (handle_json_cpe_refs (&inserts, entry)) + { + gvm_json_pull_event_cleanup (&event); + gvm_json_pull_parser_cleanup (&parser); + cJSON_Delete (entry); + fclose (json_stream); + sql_commit (); + return -1; + } + cJSON_Delete (entry); + gvm_json_pull_parser_next (&parser, &event); + } + inserts_run (&inserts, TRUE); + sql_commit (); + gvm_json_pull_parser_cleanup (&parser); + + return 0; +} + +/** + * @brief Update SCAP CPEs from an XML file. + * + * @param[in] path Path to file. + * + * @return 0 success, -1 error. + */ +static int +update_scap_cpes_from_xml_file (const gchar *path) { int ret; element_t cpe_item; @@ -2287,21 +2820,39 @@ update_scap_cpes () int ret; full_path = g_build_filename (GVM_SCAP_DATA_DIR, - "official-cpe-dictionary_v2.2.xml", + "nvd-cpes.json", NULL); if (g_stat (full_path, &state)) { - g_warning ("%s: No CPE dictionary found at %s", + g_warning ("%s: No JSON CPE dictionary found at %s", __func__, full_path); g_free (full_path); - return -1; + + full_path = g_build_filename (GVM_SCAP_DATA_DIR, + "official-cpe-dictionary_v2.2.xml", + NULL); + + if (g_stat (full_path, &state)) + { + g_warning ("%s: No CPE dictionary found at %s", + __func__, + full_path); + g_free (full_path); + return -1; + } + + ret = update_scap_cpes_from_xml_file (full_path); + if (ret) + return -1; + + return 0; } g_info ("Updating CPEs"); - ret = update_scap_cpes_from_file (full_path); + ret = update_scap_cpes_from_json_file (full_path); if (ret) return -1; @@ -2657,48 +3208,6 @@ insert_cve_from_entry (element_t entry, element_t last_modified, return 0; } -/** - * @brief Get the string value for a specified key from a JSON object. - * - * @param[in] object JSON object - * @param[in] key The key of the string in the JSON object. - * - * @return The string out of the JSON object with key "key", if any. - * NULL otherwise. - */ -static char* -json_object_item_string (cJSON *object, char *key) -{ - cJSON *value_json; - - value_json = cJSON_GetObjectItemCaseSensitive(object, key); - if (cJSON_IsString(value_json)) - return value_json->valuestring; - return NULL; -} - -/** - * @brief Get the double value for a specified key from a JSON object. - * - * @param[in] object JSON object - * @param[in] key The key of the double value in the JSON object. - * @param[in] fallback The fallback value if the double value is not - * available. - * - * @return The double value out of the JSON object with key "key", if any. - * The fallback value otherwise. - */ -static double -json_object_item_double (cJSON *object, char *key, double fallback) -{ - cJSON *value_json; - - value_json = cJSON_GetObjectItemCaseSensitive(object, key); - if (cJSON_IsNumber(value_json)) - return value_json->valuedouble; - return fallback; -} - /** * @brief Save the node of a cve match rule tree. * diff --git a/src/manage_sql_secinfo.h b/src/manage_sql_secinfo.h index 5e8ad4761..656f240eb 100644 --- a/src/manage_sql_secinfo.h +++ b/src/manage_sql_secinfo.h @@ -102,7 +102,7 @@ */ #define CPE_INFO_ITERATOR_FILTER_COLUMNS \ { GET_ITERATOR_FILTER_COLUMNS, "title", "status", \ - "deprecated_by_id", "severity", "cves", "nvd_id", \ + "deprecated_by_id", "severity", "cves", "cpe_name_id", \ NULL } /** @@ -114,11 +114,11 @@ { "''", "_owner", KEYWORD_TYPE_STRING }, \ { "0", NULL, KEYWORD_TYPE_INTEGER }, \ { "title", NULL, KEYWORD_TYPE_STRING }, \ - { "status", NULL, KEYWORD_TYPE_STRING }, \ + { "deprecated", NULL, KEYWORD_TYPE_INTEGER }, \ { "deprecated_by_id", NULL, KEYWORD_TYPE_INTEGER }, \ { "severity", NULL, KEYWORD_TYPE_DOUBLE }, \ { "cve_refs", "cves", KEYWORD_TYPE_INTEGER }, \ - { "nvd_id", NULL, KEYWORD_TYPE_INTEGER }, \ + { "cpe_name_id", NULL, KEYWORD_TYPE_STRING }, \ { NULL, NULL, KEYWORD_TYPE_UNKNOWN } \ } diff --git a/src/schema_formats/XML/GMP.xml.in b/src/schema_formats/XML/GMP.xml.in index 3218e237a..7ba7b8b05 100644 --- a/src/schema_formats/XML/GMP.xml.in +++ b/src/schema_formats/XML/GMP.xml.in @@ -12785,9 +12785,9 @@ END:VCALENDAR Number of CVEs referencing this CPE - nvd_id - integer - NVD ID of the CVE + cpe_name_id + uuid + NVD assigned cpeNameId of the CPE @@ -13012,18 +13012,19 @@ END:VCALENDAR cpe - nvd_id + cpe_name_id title severity cve_refs - status + deprecated cves + references raw_data A CPE info element - nvd_id - The NVD ID of the CPE + cpe_name_id + NVD assigned cpeNameId of the CPE text @@ -13050,10 +13051,10 @@ END:VCALENDAR - status - The status of this CPE + deprecated + Whether the CPE is deprecated - text + boolean @@ -13070,6 +13071,26 @@ END:VCALENDAR + + references + References of this CPE. Only when details were requested + + reference + + + reference + Reference of the CPE. Text contains type + + + href + text + Reference URL + 1 + + text + + + raw_data Source representation of the information. Only when details were requested From 1012034fef29b8717a89c5f575979621b2313c49 Mon Sep 17 00:00:00 2001 From: Timo Pollmeier Date: Wed, 2 Oct 2024 11:51:30 +0200 Subject: [PATCH 10/26] Change deprecated_by of CPEs to a list. --- src/gmp.c | 14 +++-- src/manage.h | 6 ++ src/manage_pg.c | 5 ++ src/manage_sql_secinfo.c | 93 +++++++++++++++++++++++-------- src/schema_formats/XML/GMP.xml.in | 13 +++++ 5 files changed, 103 insertions(+), 28 deletions(-) diff --git a/src/gmp.c b/src/gmp.c index ac95a6f67..2b179fa4e 100644 --- a/src/gmp.c +++ b/src/gmp.c @@ -13490,16 +13490,18 @@ handle_get_info (gmp_parser_t *gmp_parser, GError **error) if (get_info_data->details == 1) { - const char *deprecated_by_id - = cpe_info_iterator_deprecated_by_id (&info); - if (deprecated_by_id && strcmp (deprecated_by_id, "")) + iterator_t deprecated_by, cves, refs; + + init_cpe_deprecated_by_iterator (&deprecated_by, + get_iterator_name (&info)); + while (next (&deprecated_by)) { xml_string_append (result, - "%s", - deprecated_by_id); + "", + cpe_deprecated_by_iterator_deprecated_by + (&deprecated_by)); } - iterator_t cves, refs; g_string_append (result, ""); init_cpe_cve_iterator (&cves, get_iterator_name (&info), 0, NULL); while (next (&cves)) diff --git a/src/manage.h b/src/manage.h index 6d34f02f9..9d2434292 100644 --- a/src/manage.h +++ b/src/manage.h @@ -3339,6 +3339,12 @@ manage_scap_update_time (); /* CPE. */ +void +init_cpe_deprecated_by_iterator (iterator_t *, const char *); + +const char * +cpe_deprecated_by_iterator_deprecated_by (iterator_t *); + void init_cpe_cve_iterator (iterator_t *, const char *, int, const char *); diff --git a/src/manage_pg.c b/src/manage_pg.c index 7088a4e9c..0cad6c0af 100644 --- a/src/manage_pg.c +++ b/src/manage_pg.c @@ -3538,6 +3538,11 @@ manage_db_init (const gchar *name) " ref TEXT," " type TEXT);"); + sql ("CREATE TABLE scap2.cpes_deprecated_by" + " (id SERIAL PRIMARY KEY," + " cpe TEXT," + " deprecated_by TEXT);"); + sql ("CREATE TABLE scap2.cpe_match_nodes" " (id SERIAL PRIMARY KEY," " parent_id INTEGER DEFAULT 0," diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index b03413453..680602322 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -696,6 +696,28 @@ cve_info_filter_columns () return filter_columns; } +/** + * @brief Initialise an iterator listing CPEs another CPE is deprecated_by. + * + * @param[in] iterator Iterator. + * @param[in] cpe CPE to get which other CPEs it's deprecated by. + */ +void +init_cpe_deprecated_by_iterator (iterator_t *iterator, const char *cpe) +{ + gchar *quoted_cpe; + assert (cpe); + quoted_cpe = sql_quote (cpe); + init_iterator (iterator, + "SELECT deprecated_by FROM cpes_deprecated_by" + " WHERE cpe = '%s'" + " ORDER BY deprecated_by;", + quoted_cpe); + g_free (quoted_cpe); +} + +DEF_ACCESS (cpe_deprecated_by_iterator_deprecated_by, 0); + /** * @brief Initialise an CVE iterator, for CVEs reported for a certain CPE. * @@ -2272,19 +2294,19 @@ scap_cpes_json_skip_to_products (gvm_json_pull_parser_t *parser, /** * @brief Insert a SCAP CPE from JSON. * - * @param[in] inserts Pointer to SQL buffer. + * @param[in] inserts Pointer to SQL buffer for main CPE entries. + * @param[in] deprecated_by_inserts Pointer to SQL buffer for deprecated_by. * @param[in] product_item JSON object from the products list. * * @return 0 success, -1 error. */ static int -handle_json_cpe_item (inserts_t *inserts, cJSON *product_item) +handle_json_cpe_item (inserts_t *inserts, inserts_t *deprecated_by_inserts, + cJSON *product_item) { cJSON *cpe_item; char *name, *cpe_name_id, *last_modified, *title_text; - char *deprecated_by; gchar *quoted_name, *quoted_title, *quoted_cpe_name_id; - gchar *quoted_deprecated_by; cJSON *titles, *title; time_t modification_time; int deprecated; @@ -2347,48 +2369,70 @@ handle_json_cpe_item (inserts_t *inserts, cJSON *product_item) return -1; } - deprecated_by = NULL; + quoted_name = fs_to_uri_convert_and_quote_cpe_name (name); if (deprecated) { /* CPEs can have multiple deprecatedBy entries, * but for the GMP field only the first one is used */ - cJSON *deprecated_by_array, *first_deprecated_by; + cJSON *deprecated_by_array, *deprecated_by_item; + char *deprecated_by_id; + gchar *quoted_deprecated_by_id; deprecated_by_array = cJSON_GetObjectItemCaseSensitive (cpe_item, "deprecatedBy"); if (! cJSON_IsArray (deprecated_by_array)) { g_warning ("%s: 'deprecatedBy' field missing or not an array", __func__); + g_free (quoted_name); return -1; } else if (cJSON_GetArraySize (deprecated_by_array) == 0) { g_warning ("%s: 'deprecatedBy' array is empty", __func__); + g_free (quoted_name); return -1; } - first_deprecated_by = cJSON_GetArrayItem (deprecated_by_array, 0); - deprecated_by = json_object_item_string (first_deprecated_by, "cpeName"); - if (deprecated_by == NULL) + cJSON_ArrayForEach (deprecated_by_item, deprecated_by_array) { - g_warning ("%s: Could not get 'cpeName' string from 'deprecatedBy'", - __func__); - return -1; + deprecated_by_id = json_object_item_string (deprecated_by_item, + "cpeName"); + if (deprecated_by_id == NULL) + { + g_warning ("%s: 'cpeName' field in 'deprecatedBy' missing or not" + " a string", + __func__); + g_free (quoted_name); + return -1; + } + + quoted_deprecated_by_id + = fs_to_uri_convert_and_quote_cpe_name (deprecated_by_id); + + g_message ("%s deprecated by %s", quoted_name, quoted_deprecated_by_id); + + first = inserts_check_size (deprecated_by_inserts); + + g_string_append_printf (deprecated_by_inserts->statement, + "%s ('%s', '%s')", + first ? "" : ",", + quoted_name, + quoted_deprecated_by_id); + + deprecated_by_inserts->current_chunk_size++; + + g_free (quoted_deprecated_by_id); } } - quoted_name = fs_to_uri_convert_and_quote_cpe_name (name); quoted_cpe_name_id = sql_quote (cpe_name_id); quoted_title = sql_quote (title_text ? title_text : ""); - quoted_deprecated_by - = deprecated_by ? fs_to_uri_convert_and_quote_cpe_name (deprecated_by) - : NULL; first = inserts_check_size (inserts); g_string_append_printf (inserts->statement, - "%s ('%s', '%s', '%s', %li, %li, %d, '%s', '%s')", + "%s ('%s', '%s', '%s', %li, %li, %d, '%s')", first ? "" : ",", quoted_name, quoted_name, @@ -2396,7 +2440,6 @@ handle_json_cpe_item (inserts_t *inserts, cJSON *product_item) modification_time, modification_time, deprecated, - quoted_deprecated_by ? quoted_deprecated_by : "", quoted_cpe_name_id); inserts->current_chunk_size++; @@ -2404,7 +2447,6 @@ handle_json_cpe_item (inserts_t *inserts, cJSON *product_item) g_free (quoted_title); g_free (quoted_name); g_free (quoted_cpe_name_id); - g_free (quoted_deprecated_by); return 0; } @@ -2490,7 +2532,7 @@ handle_json_cpe_refs (inserts_t *inserts, cJSON *product_item) static int update_scap_cpes_from_json_file (const gchar *path) { - inserts_t inserts; + inserts_t inserts, deprecated_by_inserts; gvm_json_pull_parser_t parser; gvm_json_pull_event_t event; FILE *json_stream = fopen (path, "r"); @@ -2517,7 +2559,7 @@ update_scap_cpes_from_json_file (const gchar *path) setting_secinfo_sql_buffer_threshold_bytes (), "INSERT INTO scap2.cpes" " (uuid, name, title, creation_time," - " modification_time, deprecated, deprecated_by_id," + " modification_time, deprecated," " cpe_name_id)" " VALUES", " ON CONFLICT (uuid) DO UPDATE" @@ -2529,6 +2571,12 @@ update_scap_cpes_from_json_file (const gchar *path) " deprecated_by_id = EXCLUDED.deprecated_by_id," " cpe_name_id = EXCLUDED.cpe_name_id"); + inserts_init (&deprecated_by_inserts, 10, + setting_secinfo_sql_buffer_threshold_bytes (), + "INSERT INTO scap2.cpes_deprecated_by (cpe, deprecated_by)" + " VALUES ", + ""); + while (event.type == GVM_JSON_PULL_EVENT_OBJECT_START) { gchar *error_message; @@ -2543,7 +2591,7 @@ update_scap_cpes_from_json_file (const gchar *path) sql_commit (); return -1; } - if (handle_json_cpe_item (&inserts, entry)) + if (handle_json_cpe_item (&inserts, &deprecated_by_inserts, entry)) { gvm_json_pull_event_cleanup (&event); gvm_json_pull_parser_cleanup (&parser); @@ -2556,6 +2604,7 @@ update_scap_cpes_from_json_file (const gchar *path) gvm_json_pull_parser_next (&parser, &event); } inserts_run (&inserts, TRUE); + inserts_run (&deprecated_by_inserts, TRUE); sql_commit (); gvm_json_pull_parser_cleanup (&parser); diff --git a/src/schema_formats/XML/GMP.xml.in b/src/schema_formats/XML/GMP.xml.in index 7ba7b8b05..eb2feaf73 100644 --- a/src/schema_formats/XML/GMP.xml.in +++ b/src/schema_formats/XML/GMP.xml.in @@ -13017,6 +13017,7 @@ END:VCALENDAR severity cve_refs deprecated + deprecated_by cves references raw_data @@ -13057,6 +13058,18 @@ END:VCALENDAR boolean + + deprecated_by + Another CPE the current one is deprecated by + + + cpe_id + uuid + CPE id the current CPE is deprecated by + 1 + + + cves CVEs referring to this CPE. Only when details were requested From 96208aa4724978f7ef1296d87b66a0f94f2372dc Mon Sep 17 00:00:00 2001 From: Timo Pollmeier Date: Fri, 18 Oct 2024 09:44:18 +0200 Subject: [PATCH 11/26] Apply review suggestions for JSON CPEs Add missing cleanup, fix comments and remove leftover test log output. --- src/gmp.c | 1 + src/manage_sql_secinfo.c | 13 +++++-------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/gmp.c b/src/gmp.c index 2b179fa4e..b90c1ff54 100644 --- a/src/gmp.c +++ b/src/gmp.c @@ -13501,6 +13501,7 @@ handle_get_info (gmp_parser_t *gmp_parser, GError **error) cpe_deprecated_by_iterator_deprecated_by (&deprecated_by)); } + cleanup_iterator (&deprecated_by); g_string_append (result, ""); init_cpe_cve_iterator (&cves, get_iterator_name (&info), 0, NULL); diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index 680602322..d8bbe4ec3 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -424,11 +424,11 @@ json_object_item_double (cJSON *object, char *key, double fallback) * @brief Get the boolean value for a specified key from a JSON object. * * @param[in] object JSON object - * @param[in] key The key of the double value in the JSON object. + * @param[in] key The key of the boolean value in the JSON object. * @param[in] fallback The fallback value if the boolean value is not * available. * - * @return The double value out of the JSON object with key "key", if any. + * @return The boolean value out of the JSON object with key "key", if any. * The fallback value otherwise. */ static int @@ -2372,8 +2372,6 @@ handle_json_cpe_item (inserts_t *inserts, inserts_t *deprecated_by_inserts, quoted_name = fs_to_uri_convert_and_quote_cpe_name (name); if (deprecated) { - /* CPEs can have multiple deprecatedBy entries, - * but for the GMP field only the first one is used */ cJSON *deprecated_by_array, *deprecated_by_item; char *deprecated_by_id; gchar *quoted_deprecated_by_id; @@ -2410,8 +2408,6 @@ handle_json_cpe_item (inserts_t *inserts, inserts_t *deprecated_by_inserts, quoted_deprecated_by_id = fs_to_uri_convert_and_quote_cpe_name (deprecated_by_id); - g_message ("%s deprecated by %s", quoted_name, quoted_deprecated_by_id); - first = inserts_check_size (deprecated_by_inserts); g_string_append_printf (deprecated_by_inserts->statement, @@ -2420,8 +2416,7 @@ handle_json_cpe_item (inserts_t *inserts, inserts_t *deprecated_by_inserts, quoted_name, quoted_deprecated_by_id); - deprecated_by_inserts->current_chunk_size++; - + deprecated_by_inserts->current_chunk_size++; g_free (quoted_deprecated_by_id); } } @@ -2516,6 +2511,8 @@ handle_json_cpe_refs (inserts_t *inserts, cJSON *product_item) quoted_type); inserts->current_chunk_size++; + g_free (quoted_ref); + g_free (quoted_type); } g_free (quoted_name); From 8a1c684a36c6ba3d24058295f27f97dd232d88cb Mon Sep 17 00:00:00 2001 From: Timo Pollmeier Date: Fri, 18 Oct 2024 10:52:27 +0200 Subject: [PATCH 12/26] Remove: Drop unused deprecated_by column of CPEs The column has been replaced by a table to allow multiple deprecated_by entries per CPE. --- src/manage.h | 3 --- src/manage_pg.c | 1 - src/manage_sql_secinfo.c | 23 +++++------------------ src/manage_sql_secinfo.h | 5 ++--- 4 files changed, 7 insertions(+), 25 deletions(-) diff --git a/src/manage.h b/src/manage.h index 9d2434292..a2cfc5775 100644 --- a/src/manage.h +++ b/src/manage.h @@ -3372,9 +3372,6 @@ cpe_info_iterator_cve_refs (iterator_t*); const char* cpe_info_iterator_cpe_name_id (iterator_t*); -const char* -cpe_info_iterator_deprecated_by_id (iterator_t*); - gchar * cpe_details_xml (const char*); diff --git a/src/manage_pg.c b/src/manage_pg.c index 0cad6c0af..4bc218524 100644 --- a/src/manage_pg.c +++ b/src/manage_pg.c @@ -3525,7 +3525,6 @@ manage_db_init (const gchar *name) " modification_time integer," " title text," " status text," - " deprecated_by_id TEXT," " severity DOUBLE PRECISION DEFAULT 0," " cve_refs INTEGER DEFAULT 0," " nvd_id text," diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index d8bbe4ec3..f0a87c63c 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -571,17 +571,6 @@ DEF_ACCESS (cpe_info_iterator_title, GET_ITERATOR_COLUMN_COUNT); */ DEF_ACCESS (cpe_info_iterator_deprecated, GET_ITERATOR_COLUMN_COUNT + 1); -/** - * @brief Get the first CPE the current one is deprecated by - * from a CPE iterator. - * - * @param[in] iterator Iterator. - * - * @return The first CPE the current one is deprecated by, - * or NULL if iteration is complete. Freed by cleanup_iterator. - */ -DEF_ACCESS (cpe_info_iterator_deprecated_by_id, GET_ITERATOR_COLUMN_COUNT + 2); - /** * @brief Get the highest severity Score of all CVE's referencing this cpe. * @@ -590,7 +579,7 @@ DEF_ACCESS (cpe_info_iterator_deprecated_by_id, GET_ITERATOR_COLUMN_COUNT + 2); * @return The highest severity score of the CPE, * or NULL if iteration is complete. Freed by cleanup_iterator. */ -DEF_ACCESS (cpe_info_iterator_severity, GET_ITERATOR_COLUMN_COUNT + 3); +DEF_ACCESS (cpe_info_iterator_severity, GET_ITERATOR_COLUMN_COUNT + 2); /** * @brief Get the Number of CVE's referencing this cpe from a CPE iterator. @@ -600,7 +589,7 @@ DEF_ACCESS (cpe_info_iterator_severity, GET_ITERATOR_COLUMN_COUNT + 3); * @return The Number of references to the CPE, or NULL if iteration is * complete. Freed by cleanup_iterator. */ -DEF_ACCESS (cpe_info_iterator_cve_refs, GET_ITERATOR_COLUMN_COUNT + 4); +DEF_ACCESS (cpe_info_iterator_cve_refs, GET_ITERATOR_COLUMN_COUNT + 3); /** * @brief Get the NVD assigned cpeNameId for this CPE. @@ -610,7 +599,7 @@ DEF_ACCESS (cpe_info_iterator_cve_refs, GET_ITERATOR_COLUMN_COUNT + 4); * @return The NVD ID of this CPE, or NULL if iteration is * complete. Freed by cleanup_iterator. */ -DEF_ACCESS (cpe_info_iterator_cpe_name_id, GET_ITERATOR_COLUMN_COUNT + 5); +DEF_ACCESS (cpe_info_iterator_cpe_name_id, GET_ITERATOR_COLUMN_COUNT + 4); /** * @brief Get the XML details / raw data for a given CPE ID. @@ -2373,7 +2362,6 @@ handle_json_cpe_item (inserts_t *inserts, inserts_t *deprecated_by_inserts, if (deprecated) { cJSON *deprecated_by_array, *deprecated_by_item; - char *deprecated_by_id; gchar *quoted_deprecated_by_id; deprecated_by_array = cJSON_GetObjectItemCaseSensitive (cpe_item, "deprecatedBy"); @@ -2394,6 +2382,7 @@ handle_json_cpe_item (inserts_t *inserts, inserts_t *deprecated_by_inserts, cJSON_ArrayForEach (deprecated_by_item, deprecated_by_array) { + char *deprecated_by_id; deprecated_by_id = json_object_item_string (deprecated_by_item, "cpeName"); if (deprecated_by_id == NULL) @@ -2565,7 +2554,6 @@ update_scap_cpes_from_json_file (const gchar *path) " creation_time = EXCLUDED.creation_time," " modification_time = EXCLUDED.modification_time," " deprecated = EXCLUDED.deprecated," - " deprecated_by_id = EXCLUDED.deprecated_by_id," " cpe_name_id = EXCLUDED.cpe_name_id"); inserts_init (&deprecated_by_inserts, 10, @@ -2705,7 +2693,7 @@ update_scap_cpes_from_xml_file (const gchar *path) setting_secinfo_sql_buffer_threshold_bytes (), "INSERT INTO scap2.cpes" " (uuid, name, title, creation_time," - " modification_time, status, deprecated_by_id," + " modification_time, status," " nvd_id)" " VALUES", " ON CONFLICT (uuid) DO UPDATE" @@ -2714,7 +2702,6 @@ update_scap_cpes_from_xml_file (const gchar *path) " creation_time = EXCLUDED.creation_time," " modification_time = EXCLUDED.modification_time," " status = EXCLUDED.status," - " deprecated_by_id = EXCLUDED.deprecated_by_id," " nvd_id = EXCLUDED.nvd_id"); cpe_item = xml_file_iterator_next (file_iterator, &error_message); diff --git a/src/manage_sql_secinfo.h b/src/manage_sql_secinfo.h index 656f240eb..71f00c091 100644 --- a/src/manage_sql_secinfo.h +++ b/src/manage_sql_secinfo.h @@ -101,8 +101,8 @@ * @brief Filter columns for CVE iterator. */ #define CPE_INFO_ITERATOR_FILTER_COLUMNS \ - { GET_ITERATOR_FILTER_COLUMNS, "title", "status", \ - "deprecated_by_id", "severity", "cves", "cpe_name_id", \ + { GET_ITERATOR_FILTER_COLUMNS, "title", "deprecated", \ + "severity", "cves", "cpe_name_id", \ NULL } /** @@ -115,7 +115,6 @@ { "0", NULL, KEYWORD_TYPE_INTEGER }, \ { "title", NULL, KEYWORD_TYPE_STRING }, \ { "deprecated", NULL, KEYWORD_TYPE_INTEGER }, \ - { "deprecated_by_id", NULL, KEYWORD_TYPE_INTEGER }, \ { "severity", NULL, KEYWORD_TYPE_DOUBLE }, \ { "cve_refs", "cves", KEYWORD_TYPE_INTEGER }, \ { "cpe_name_id", NULL, KEYWORD_TYPE_STRING }, \ From 624f1444d87aa26e6b488b86e7ce85a2437558e5 Mon Sep 17 00:00:00 2001 From: Johannes Helmold Date: Thu, 24 Oct 2024 11:56:09 +0200 Subject: [PATCH 13/26] Add: Added reading of gzip files for CPEs. --- src/manage_sql_secinfo.c | 71 ++++++++++++++++++++++++++++++++-------- 1 file changed, 57 insertions(+), 14 deletions(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index 71cc6e055..0f5527da6 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -2522,21 +2522,35 @@ update_scap_cpes_from_json_file (const gchar *path) inserts_t inserts, deprecated_by_inserts; gvm_json_pull_parser_t parser; gvm_json_pull_event_t event; - FILE *json_stream = fopen (path, "r"); - if (json_stream == NULL) + FILE *cpe_file; + + int fd = open (path, O_RDONLY); + + if (fd < 0) { - g_warning ("%s: Could not open file '%s': %s", + g_warning ("%s: Failed to open CPE file '%s': %s", __func__, path, strerror(errno)); return -1; } - gvm_json_pull_parser_init (&parser, json_stream); + g_info ("Updating %s", path); + + cpe_file = gvm_gzip_open_file_reader_fd (fd); + if (cpe_file == NULL) + { + g_warning ("%s: Failed to open CPE file: %s", + __func__, + strerror (errno)); + return -1; + } + + gvm_json_pull_parser_init (&parser, cpe_file); gvm_json_pull_event_init (&event); if (scap_cpes_json_skip_to_products (&parser, &event)) { gvm_json_pull_event_cleanup (&event); gvm_json_pull_parser_cleanup (&parser); - fclose (json_stream); + fclose (cpe_file); return -1; } @@ -2573,7 +2587,7 @@ update_scap_cpes_from_json_file (const gchar *path) gvm_json_pull_event_cleanup (&event); gvm_json_pull_parser_cleanup (&parser); cJSON_Delete (entry); - fclose (json_stream); + fclose (cpe_file); sql_commit (); return -1; } @@ -2582,7 +2596,7 @@ update_scap_cpes_from_json_file (const gchar *path) gvm_json_pull_event_cleanup (&event); gvm_json_pull_parser_cleanup (&parser); cJSON_Delete (entry); - fclose (json_stream); + fclose (cpe_file); sql_commit (); return -1; } @@ -2595,14 +2609,31 @@ update_scap_cpes_from_json_file (const gchar *path) gvm_json_pull_parser_cleanup (&parser); // Reset and insert refs - fseek (json_stream, 0, SEEK_SET); - gvm_json_pull_parser_init (&parser, json_stream); + fclose (cpe_file); + fd = open (path, O_RDONLY); + + if (fd < 0) + { + g_warning ("%s: Failed to open CPE file '%s': %s", + __func__, path, strerror(errno)); + return -1; + } + + cpe_file = gvm_gzip_open_file_reader_fd (fd); + if (cpe_file == NULL) + { + g_warning ("%s: Failed to open CPE file: %s", + __func__, + strerror (errno)); + return -1; + } + gvm_json_pull_parser_init (&parser, cpe_file); if (scap_cpes_json_skip_to_products (&parser, &event)) { gvm_json_pull_event_cleanup (&event); gvm_json_pull_parser_cleanup (&parser); - fclose (json_stream); + fclose (cpe_file); return -1; } @@ -2625,7 +2656,7 @@ update_scap_cpes_from_json_file (const gchar *path) gvm_json_pull_event_cleanup (&event); gvm_json_pull_parser_cleanup (&parser); cJSON_Delete (entry); - fclose (json_stream); + fclose (cpe_file); sql_commit (); return -1; } @@ -2634,7 +2665,7 @@ update_scap_cpes_from_json_file (const gchar *path) gvm_json_pull_event_cleanup (&event); gvm_json_pull_parser_cleanup (&parser); cJSON_Delete (entry); - fclose (json_stream); + fclose (cpe_file); sql_commit (); return -1; } @@ -2645,6 +2676,7 @@ update_scap_cpes_from_json_file (const gchar *path) sql_commit (); gvm_json_pull_parser_cleanup (&parser); + fclose (cpe_file); return 0; } @@ -2854,8 +2886,15 @@ update_scap_cpes () int ret; full_path = g_build_filename (GVM_SCAP_DATA_DIR, - "nvd-cpes.json", + "nvd-cpes.json.gz", NULL); + if (g_stat (full_path, &state)) + { + g_free (full_path); + full_path = g_build_filename (GVM_SCAP_DATA_DIR, + "nvd-cpes.json", + NULL); + } if (g_stat (full_path, &state)) { @@ -2888,8 +2927,12 @@ update_scap_cpes () ret = update_scap_cpes_from_json_file (full_path); if (ret) - return -1; + { + g_free (full_path); + return -1; + } + g_free (full_path); return 0; } From f5f77acec7ba290f146ac7af2ddc8ebee87d88f4 Mon Sep 17 00:00:00 2001 From: Johannes Helmold Date: Mon, 28 Oct 2024 16:01:30 +0100 Subject: [PATCH 14/26] Very small optimization. --- src/manage_sql_secinfo.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index 0f5527da6..998ccf891 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -2926,13 +2926,12 @@ update_scap_cpes () g_info ("Updating CPEs"); ret = update_scap_cpes_from_json_file (full_path); - if (ret) - { - g_free (full_path); - return -1; - } g_free (full_path); + + if (ret) + return -1; + return 0; } From 3d75038b71ece78869244b50017a3f496569492f Mon Sep 17 00:00:00 2001 From: Ahmed Abdelsalam Date: Fri, 4 Oct 2024 17:38:52 +0200 Subject: [PATCH 15/26] Change: Update handling of CVEs for the new JSON API. - Handle references explicitly to remove raw_data. - Add affected software configurations and references to the response of get_info for CVEs when details are enabled. --- src/gmp.c | 195 ++++++++ src/manage.h | 27 + src/manage_pg.c | 39 +- src/manage_sql.c | 160 +++++- src/manage_sql_secinfo.c | 797 ++++++++++++++++++++++-------- src/schema_formats/XML/GMP.xml.in | 157 ++++++ 6 files changed, 1138 insertions(+), 237 deletions(-) diff --git a/src/gmp.c b/src/gmp.c index b90c1ff54..fe766df12 100644 --- a/src/gmp.c +++ b/src/gmp.c @@ -101,6 +101,7 @@ #include "manage_report_configs.h" #include "manage_report_formats.h" #include "manage_tls_certificates.h" +#include "sql.h" #include "utils.h" #include @@ -128,6 +129,7 @@ #include #include #include +#include #undef G_LOG_DOMAIN /** @@ -13252,6 +13254,194 @@ handle_get_groups (gmp_parser_t *gmp_parser, GError **error) set_client_state (CLIENT_AUTHENTIC); } +/** + * @brief Print CPE match node with its matched CPEs. + * + * @param[in] node CPE match node to print. + * @param[in] buffer Buffer into which to print match node. + */ +static void +print_cpe_match_nodes_xml(resource_t node, GString *buffer) +{ + iterator_t cpe_match_nodes, cpe_match_ranges; + const char *operator = NULL; + int negate = 0; + + init_iterator (&cpe_match_nodes, + "SELECT operator, negate FROM scap.cpe_match_nodes WHERE id = %llu;", + node); + while (next (&cpe_match_nodes)) + { + operator = iterator_string (&cpe_match_nodes, 0); + negate = iterator_int (&cpe_match_nodes, 1); + } + cleanup_iterator (&cpe_match_nodes); + + xml_string_append (buffer, "%s", operator?: ""); + xml_string_append (buffer, "%s", negate? "1" : "0"); + + init_cpe_match_range_iterator (&cpe_match_ranges, node); + while (next (&cpe_match_ranges)) + { + const gchar *vsi, *vse, *vei, *vee, *match_criteria_id, *range_uri_product; + + xml_string_append (buffer, ""); + match_criteria_id = cpe_match_range_iterator_match_criteria_id (&cpe_match_ranges); + range_uri_product + = fs_cpe_to_uri_cpe (cpe_match_range_iterator_cpe (&cpe_match_ranges)); + xml_string_append (buffer, "%s", range_uri_product?: ""); + xml_string_append (buffer, "%s", + cpe_match_range_iterator_vulnerable (&cpe_match_ranges) != 0 + ? "1" + : "0"); + vsi = cpe_match_range_iterator_version_start_incl(&cpe_match_ranges); + vse = cpe_match_range_iterator_version_start_excl(&cpe_match_ranges); + vei = cpe_match_range_iterator_version_end_incl(&cpe_match_ranges); + vee = cpe_match_range_iterator_version_end_excl(&cpe_match_ranges); + + xml_string_append (buffer, + "%s", + vsi ?: ""); + xml_string_append (buffer, + "%s", + vse ?: ""); + xml_string_append (buffer, + "%s", + vei ?: ""); + xml_string_append (buffer, + "%s", + vee ?: ""); + + iterator_t cpe_matches; + init_cpe_match_range_matches_iterator (&cpe_matches, match_criteria_id); + xml_string_append (buffer, ""); + + while (next (&cpe_matches)) + { + iterator_t cpes; + + init_iterator (&cpes, + "SELECT name, deprecated FROM scap.cpes" + " WHERE cpe_name_id = '%s';", + cpe_matches_cpe_name_id(&cpe_matches)); + + const char* cpe = NULL; + int deprecated = 0; + while (next (&cpes)) + { + cpe = iterator_string (&cpes, 0); + deprecated = iterator_int (&cpes, 1); + } + cleanup_iterator (&cpes); + + xml_string_append (buffer, ""); + xml_string_append (buffer, "%s", cpe?: ""); + xml_string_append (buffer, + "%s", + deprecated ? "1" : "0"); + if (deprecated) + { + iterator_t deprecated_by; + init_cpe_deprecated_by_iterator (&deprecated_by, cpe); + while (next (&deprecated_by)) + { + xml_string_append (buffer, + "", + cpe_deprecated_by_iterator_deprecated_by + (&deprecated_by)); + } + cleanup_iterator (&deprecated_by); + } + xml_string_append (buffer, ""); + } + xml_string_append (buffer, ""); + xml_string_append (buffer, ""); + cleanup_iterator (&cpe_matches); + } + cleanup_iterator (&cpe_match_ranges); +} +/** + * @brief Print CVE affected software configurations + * + * @param[in] cve_uuid uuid of the CVE. + * @param[out] result Buffer into which to print. + * + */ +static void +print_cve_affected_software_configs_xml (gchar *cve_uuid, GString *result) +{ + iterator_t cpe_match_root_nodes; + xml_string_append (result, ""); + init_cve_cpe_match_nodes_iterator (&cpe_match_root_nodes, cve_uuid); + while (next (&cpe_match_root_nodes)) + { + result_t root_node; + iterator_t cpe_match_node_childs; + root_node = cpe_match_nodes_iterator_root_id (&cpe_match_root_nodes); + xml_string_append (result, ""); + print_cpe_match_nodes_xml(root_node, result); + init_cpe_match_node_childs_iterator (&cpe_match_node_childs, root_node); + while (next (&cpe_match_node_childs)) + { + resource_t child_node; + child_node = cpe_match_node_childs_iterator_id (&cpe_match_node_childs); + xml_string_append (result, ""); + print_cpe_match_nodes_xml(child_node, result); + xml_string_append (result, ""); + } + xml_string_append (result, ""); + cleanup_iterator (&cpe_match_node_childs); + } + xml_string_append (result, ""); + cleanup_iterator (&cpe_match_root_nodes); +} + +/** + * @brief Print CVE references + * + * @param[in] cve_uuid uuid of the CVE. + * @param[out] result Buffer into which to print. + * + */ +static void +print_cve_references_xml (gchar *cve_uuid, GString *result) +{ + iterator_t references; + init_cve_reference_iterator (&references, cve_uuid); + xml_string_append (result, ""); + while (next (&references)) + { + xml_string_append (result, ""); + xml_string_append (result, "%s", cve_reference_iterator_url (&references)); + xml_string_append (result, ""); + const char * tags_array = cve_reference_iterator_tags (&references); + if(tags_array && strlen(tags_array) > 2) + { + char *trimmed_array = g_strndup (tags_array + 1, strlen (tags_array) - 2); + gchar **tags, **current_tag; + tags = g_strsplit (trimmed_array, ",", -1); + current_tag = tags; + while (*current_tag) + { + if (strlen (*current_tag) > 2 && (*current_tag)[0] == '"' && (*current_tag)[strlen (*current_tag) - 1] == '"') + { + char *trimmed_tag = g_strndup (*current_tag + 1, strlen (*current_tag) - 2); + xml_string_append (result, "%s", trimmed_tag); + g_free (trimmed_tag); + } + else + xml_string_append (result, "%s", *current_tag); + current_tag++; + } + g_strfreev (tags); + g_free (trimmed_array); + } + xml_string_append (result, ""); + xml_string_append (result, ""); + } + xml_string_append (result, ""); + cleanup_iterator (&references); +} /** * @brief Handle end of GET_INFO element. * @@ -13627,6 +13817,11 @@ handle_get_info (gmp_parser_t *gmp_parser, GError **error) ""); } g_string_append (result, ""); + + gchar *cve_uuid = g_strdup(get_iterator_uuid (&info)); + print_cve_affected_software_configs_xml (cve_uuid, result); + print_cve_references_xml (cve_uuid, result); + g_free(cve_uuid); } } else if (g_strcmp0 ("cert_bund_adv", get_info_data->type) == 0) diff --git a/src/manage.h b/src/manage.h index df8f144a4..93cbb01e2 100644 --- a/src/manage.h +++ b/src/manage.h @@ -1693,6 +1693,21 @@ app_locations_iterator_location (iterator_t*); void init_cpe_match_nodes_iterator (iterator_t*, const char *); +void +init_cve_cpe_match_nodes_iterator (iterator_t*, const char *); + +void +init_cve_reference_iterator (iterator_t*, const char *); + +const char* +cve_reference_iterator_url (iterator_t*); + +const char* +cve_reference_iterator_tags (iterator_t*); + +const char* +cve_reference_iterator_tags_count (iterator_t*); + long long int cpe_match_nodes_iterator_root_id (iterator_t*); @@ -1714,6 +1729,12 @@ init_cpe_match_range_iterator (iterator_t*, long long int); const char* cpe_match_range_iterator_cpe (iterator_t*); +const char* +cpe_match_range_iterator_match_criteria_id (iterator_t*); + +const char* +cpe_match_range_iterator_status (iterator_t*); + const char* cpe_match_range_iterator_version_start_incl (iterator_t*); @@ -1729,6 +1750,12 @@ cpe_match_range_iterator_version_end_excl (iterator_t*); int cpe_match_range_iterator_vulnerable (iterator_t*); +void +init_cpe_match_range_matches_iterator (iterator_t*, const char *); + +const char* +cpe_matches_cpe_name_id (iterator_t*); + void init_host_details_cpe_product_iterator (iterator_t*, const char *, report_host_t); diff --git a/src/manage_pg.c b/src/manage_pg.c index f53b9f601..893d3e240 100644 --- a/src/manage_pg.c +++ b/src/manage_pg.c @@ -3546,20 +3546,31 @@ manage_db_init (const gchar *name) sql ("CREATE TABLE scap2.cpe_match_nodes" " (id SERIAL PRIMARY KEY," - " parent_id INTEGER DEFAULT 0," - " root_id INTEGER DEFAULT 0," - " cve_id INTEGER DEFAULT 0," - " operator text);"); + " root_id integer DEFAULT 0," + " cve_id integer DEFAULT 0," + " operator text," + " negate integer DEFAULT 0);"); + + sql ("CREATE TABLE scap2.cpe_nodes_match_criteria" + " (id SERIAL PRIMARY KEY," + " node_id integer DEFAULT 0," + " vulnerable integer DEFAULT 0," + " match_criteria_id text);"); sql ("CREATE TABLE scap2.cpe_match_range" " (id SERIAL PRIMARY KEY," - " node_id INTEGER DEFAULT 0," - " vulnerable INTEGER DEFAULT 0," + " match_criteria_id text," " cpe text DEFAULT NULL," " version_start_incl text DEFAULT NULL," " version_start_excl text DEFAULT NULL," " version_end_incl text DEFAULT NULL," - " version_end_excl text DEFAULT NULL);"); + " version_end_excl text DEFAULT NULL," + " status text);"); + + sql ("CREATE TABLE scap2.cpe_matches" + " (id SERIAL PRIMARY KEY," + " match_criteria_id text," + " cpe_name_id text);"); sql ("CREATE TABLE scap2.cpe_details" " (id SERIAL PRIMARY KEY," @@ -3575,6 +3586,11 @@ manage_db_init (const gchar *name) " epss DOUBLE PRECISION," " percentile DOUBLE PRECISION);"); + sql ("CREATE TABLE scap2.cve_references" + " (id SERIAL PRIMARY KEY," + " cve_id INTEGER," + " url text," + " tags text[]);"); /* Init tables. */ @@ -3624,6 +3640,15 @@ manage_db_add_constraints (const gchar *name) sql ("ALTER TABLE scap2.epss_scores" " ALTER cve SET NOT NULL," " ADD UNIQUE (cve);"); + + sql ("ALTER TABLE scap2.cve_references" + " ALTER cve_id SET NOT NULL," + " ALTER url SET NOT NULL," + " ADD UNIQUE (cve_id, url);"); + + sql ("ALTER TABLE scap2.cpe_match_range" + " ADD UNIQUE (match_criteria_id);"); + } else { diff --git a/src/manage_sql.c b/src/manage_sql.c index fc1bf3905..71461dc00 100644 --- a/src/manage_sql.c +++ b/src/manage_sql.c @@ -20511,13 +20511,80 @@ init_cpe_match_nodes_iterator (iterator_t* iterator, const char *cpe) gchar *quoted_cpe; quoted_cpe = sql_quote (cpe); init_iterator (iterator, - "SELECT DISTINCT root_id" - " FROM scap.cpe_match_nodes, scap.cpe_match_range" - " WHERE cpe like '%s%%' AND scap.cpe_match_nodes.id = node_id;", + " SELECT DISTINCT n.root_id" + " FROM scap.cpe_match_nodes n" + " JOIN scap.cpe_nodes_match_criteria c ON n.id = c.node_id" + " JOIN scap.cpe_match_range r ON c.match_criteria = r.match_criteria_id" + " WHERE cpe like '%s%%';", quoted_cpe); g_free (quoted_cpe); } +/** + * @brief Initialize an iterator of CPE match nodes root_ids for a CVE. + * + * @param[in] iterator Iterator. + * @param[in] cve The CVE contained in the match nodes. + */ +void +init_cve_cpe_match_nodes_iterator (iterator_t* iterator, const char *cve) +{ + gchar *quoted_cve; + quoted_cve = sql_quote (cve); + init_iterator (iterator, + "SELECT DISTINCT root_id" + " FROM scap.cpe_match_nodes" + " WHERE cve_id = (SELECT id from scap.cves where uuid = '%s');", + quoted_cve); + g_free (quoted_cve); +} + +/** + * @brief Initialize an iterator of references for a CVE. + * + * @param[in] iterator Iterator. + * @param[in] cve The CVE with the references. + */ +void +init_cve_reference_iterator (iterator_t* iterator, const char *cve) +{ + gchar *quoted_cve; + quoted_cve = sql_quote (cve); + init_iterator (iterator, + "SELECT url, array_length(tags, 1), tags" + " FROM scap.cve_references" + " WHERE cve_id = (SELECT id from scap.cves where uuid = '%s');", + quoted_cve); + g_free (quoted_cve); +} + +/** + * @brief Get a URL from a CVE reference iterator. + * + * @param[in] iterator Iterator. + * + * @return The URL. + */ +DEF_ACCESS (cve_reference_iterator_url, 0); + +/** + * @brief Get the length of the tags array from a CVE reference iterator. + * + * @param[in] iterator Iterator. + * + * @return Length of the tags array. + */ +DEF_ACCESS (cve_reference_iterator_tags_count, 1); + +/** + * @brief Get the tags array from a CVE reference iterator. + * + * @param[in] iterator Iterator. + * + * @return The tags array. + */ +DEF_ACCESS (cve_reference_iterator_tags, 2); + /** * @brief Get a root id from an CPE match node iterator. * @@ -20542,7 +20609,8 @@ init_cpe_match_node_childs_iterator (iterator_t* iterator, long long int node) { init_iterator (iterator, "SELECT id FROM scap.cpe_match_nodes" - " WHERE parent_id = %llu;", + " WHERE root_id = %llu" + " AND root_id <> id;", node); } @@ -20569,10 +20637,13 @@ void init_cpe_match_range_iterator (iterator_t* iterator, long long int node) { init_iterator (iterator, - "SELECT vulnerable, cpe, version_start_incl," - " version_start_excl, version_end_incl, version_end_excl" - " FROM scap.cpe_match_range" - " WHERE node_id = %llu;", + "SELECT n.vulnerable, r.cpe, r.match_criteria_id, r.status," + " r.version_start_incl, r.version_start_excl," + " r.version_end_incl, r.version_end_excl" + " FROM scap.cpe_match_range r" + " JOIN scap.cpe_nodes_match_criteria n" + " ON r.match_criteria_id = n.match_criteria_id" + " WHERE n.node_id = %llu;", node); } @@ -20599,44 +20670,93 @@ cpe_match_range_iterator_vulnerable (iterator_t* iterator) DEF_ACCESS (cpe_match_range_iterator_cpe, 1); /** - * @brief Return the start included version of the actual match node. + * @brief Return the match criteria id of the CPE match range. + * + * @param[in] iterator Iterator. + * + * @return The match criteria id, if any. NULL otherwise. + */ +DEF_ACCESS (cpe_match_range_iterator_match_criteria_id, 2); + +/** + * @brief Return the status of the CPE match range. * * @param[in] iterator Iterator. * - * @return The start included version of the actual match node, if any. + * @return The status of the CPE match range, if any. * NULL otherwise. */ -DEF_ACCESS (cpe_match_range_iterator_version_start_incl, 2); +DEF_ACCESS (cpe_match_range_iterator_status, 3); /** - * @brief Return the start excluded version of the actual match node. + * @brief Return the start included version of the match range. * * @param[in] iterator Iterator. * - * @return The start excluded version of the actual match node, if any. + * @return The start included version of the match range, if any. * NULL otherwise. */ -DEF_ACCESS (cpe_match_range_iterator_version_start_excl, 3); +DEF_ACCESS (cpe_match_range_iterator_version_start_incl, 4); /** - * @brief Return the end included version of the actual match node. + * @brief Return the start excluded version of the match range. * * @param[in] iterator Iterator. * - * @return The end included version of the actual match node, if any. + * @return The start excluded version of the match range, if any. * NULL otherwise. */ -DEF_ACCESS (cpe_match_range_iterator_version_end_incl, 4); +DEF_ACCESS (cpe_match_range_iterator_version_start_excl, 5); /** - * @brief Return the end excluded version of the actual match node. + * @brief Return the end included version of the match range. * * @param[in] iterator Iterator. * - * @return The end excluded version of the actual match node, if any. + * @return The end included version of the match range, if any. * NULL otherwise. */ -DEF_ACCESS (cpe_match_range_iterator_version_end_excl, 5); +DEF_ACCESS (cpe_match_range_iterator_version_end_incl, 6); + +/** + * @brief Return the end excluded version of the match range. + * + * @param[in] iterator Iterator. + * + * @return The end excluded version of the match range, if any. + * NULL otherwise. + */ +DEF_ACCESS (cpe_match_range_iterator_version_end_excl, 7); + +/** + * @brief Initialize an iterator of CPE matches for a match range + * given a match criteria id. + * + * @param[in] iterator Iterator. + * @param[in] match_criteria_id The match criteria id to get the matches for. + */ +void +init_cpe_match_range_matches_iterator (iterator_t* iterator, const char *match_criteria_id) +{ + init_iterator (iterator, + "SELECT cpe_name_id" + " FROM scap.cpe_matches" + " WHERE match_criteria_id = '%s'", + match_criteria_id); +} + +/** + * @brief Get the CPE name id from a CPE match range matches iterator. + * + * @param[in] iterator Iterator. + * + * @return The CPE name id. + */ +const char * +cpe_matches_cpe_name_id (iterator_t* iterator) +{ + return iterator_string (iterator, 0); +} /** * @brief Initialise a report host prognosis iterator. diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index cd95c4aff..3eee935c7 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -3245,115 +3245,24 @@ insert_cve_from_entry (element_t entry, element_t last_modified, /** * @brief Save the node of a cve match rule tree. * - * @param[in] parent_id The parent_id of the node. If this value is 0, - * the node is the root of the tree. * @param[in] cve_id The id of the CVE to which the tree belongs. * @param[in] operator The operator for the match rules. + * @param[in] negate Whether the match rules are negated. * * @return The (database) id of the node. */ static resource_t -save_node (resource_t parent_id, resource_t cve_id, char *operator) +save_node (resource_t cve_id, char *operator, gboolean negate) { return sql_int64_0 ("INSERT INTO scap2.cpe_match_nodes" - " (parent_id, cve_id, operator)" + " (cve_id, operator, negate)" " VALUES" - " (%llu, %llu, '%s')" + " (%llu, '%s', %d)" " RETURNING scap2.cpe_match_nodes.id;", - parent_id, cve_id, - operator); -} - -/** - * @brief Add match rules to a node of a match rule tree* - * - * @param[in] id The id of the node the rules belong to. - * @param[in] match_rules The JSON object that contains the rules. - */ -static void -add_cpe_match_rules (result_t id, cJSON *match_rules) -{ - cJSON *match_rule; - cJSON *ver_se; - cJSON *cpe_js; - - gboolean vulnerable = FALSE; - char * version_start_incl = NULL; - char * version_start_excl = NULL; - char * version_end_incl = NULL; - char * version_end_excl = NULL; - - cJSON_ArrayForEach(match_rule, match_rules) - { - char *sql_cpe = NULL; - vulnerable = FALSE; - version_start_incl = NULL; - version_start_excl = NULL; - version_end_incl = NULL; - version_end_excl = NULL; - - if (cJSON_IsTrue(cJSON_GetObjectItemCaseSensitive(match_rule, "vulnerable"))) - vulnerable = TRUE; - else - vulnerable = FALSE; - - cpe_js = cJSON_GetObjectItemCaseSensitive(match_rule, "cpe23Uri"); - if (cpe_js != NULL && strcmp (cpe_js->valuestring, "(null)")) - { - char *quoted_cpe = sql_quote (cpe_js->valuestring); - sql_cpe = g_strdup_printf ("'%s'", quoted_cpe); - g_free (quoted_cpe); - } - else - sql_cpe = g_strdup ("NULL"); - - ver_se = cJSON_GetObjectItemCaseSensitive(match_rule, "versionStartIncluding"); - if (ver_se != NULL && strcmp (ver_se->valuestring, "(null)")) - version_start_incl = g_strdup_printf ("'%s'", ver_se->valuestring); - else - version_start_incl = g_strdup ("NULL"); - - ver_se = cJSON_GetObjectItemCaseSensitive(match_rule, "versionStartExcluding"); - if (ver_se != NULL && strcmp (ver_se->valuestring, "(null)")) - version_start_excl = g_strdup_printf ("'%s'", ver_se->valuestring); - else - version_start_excl = g_strdup ("NULL"); - - ver_se = cJSON_GetObjectItemCaseSensitive(match_rule, "versionEndIncluding"); - if (ver_se != NULL && strcmp (ver_se->valuestring, "(null)")) - version_end_incl = g_strdup_printf ("'%s'", ver_se->valuestring); - else - version_end_incl = g_strdup ("NULL"); - - ver_se = cJSON_GetObjectItemCaseSensitive(match_rule, "versionEndExcluding"); - if (ver_se != NULL && strcmp (ver_se->valuestring, "(null)")) - version_end_excl = g_strdup_printf ("'%s'", ver_se->valuestring); - else - version_end_excl = g_strdup ("NULL"); - - sql - ("INSERT INTO scap2.cpe_match_range" - " (node_id, vulnerable, cpe," - " version_start_incl, version_start_excl," - " version_end_incl, version_end_excl)" - " VALUES" - " (%llu, %d, %s, %s, %s, %s, %s)", - id, - vulnerable ? 1 : 0, - sql_cpe ? sql_cpe : "", - version_start_incl ? version_start_incl : "", - version_start_excl ? version_start_excl : "", - version_end_incl ? version_end_incl : "", - version_end_excl ? version_end_excl : ""); - - g_free (sql_cpe); - g_free (version_start_incl); - g_free (version_start_excl); - g_free (version_end_incl); - g_free (version_end_excl); - } + operator, + negate ? 1 : 0); } /** @@ -3372,52 +3281,185 @@ set_root_id (long int id, long int root_id) } /** - * @brief Load and add recursively all nodes of a match rule tree for a - * specific CVE. Build a match rule tree. + * @brief Handle the references of a CVE. + * + * @param[in] cve_db_id The id of the CVE the references belong to. + * @param[in] cve_id The id of the CVE. + * @param[in] reference_json JSON array containing the references. * - * @param[in] parent_id The parent id of the nodes to insert - * (0 for the root node). - * @param[in] cveid The id of the CVE the tree belongs to. - * @param[in] root_id The root id of the nodes to insert. - * @param[in] nodes The JSON object that contains the rules for a - * specific tree level. + * @return 0 on success, -1 on error. */ -static void -load_nodes (resource_t parent_id, resource_t cveid, resource_t root_id, cJSON *nodes) +static int +handle_cve_references (resource_t cve_db_id, char * cve_id, cJSON* reference_json) { - cJSON *node; - resource_t id; - cJSON *operator; - cJSON *cpe_match_rules; - cJSON *child_nodes; - - node = NULL; - id = 0; - operator = NULL; - cpe_match_rules = NULL; - child_nodes = NULL; - - if (nodes == NULL) - return; + cJSON *reference_data; + cJSON *tags; - cJSON_ArrayForEach(node, nodes) + cJSON_ArrayForEach(reference_data, reference_json) { - operator = cJSON_GetObjectItemCaseSensitive(node, "operator"); - if (operator && operator->valuestring) - id = save_node (parent_id, cveid, operator->valuestring); - else - return; + GString *tags_string; + char *url_value = json_object_item_string (reference_data, "url"); + if (url_value == NULL) + { + g_warning("%s: url missing in reference for %s.", __func__, cve_id); + return -1; + } + + tags = cJSON_GetObjectItemCaseSensitive(reference_data, "tags"); + if (cJSON_IsArray(tags)) + { + array_t *tags_array = make_array (); + tags_string = g_string_new ("{"); + + for (int i = 0; i < cJSON_GetArraySize(tags); i++) + { + cJSON *tag = cJSON_GetArrayItem(tags, i); + if (!cJSON_IsString(tag)) + { + g_warning("%s: tag for %s is NULL or not a string.", __func__, cve_id); + return -1; + } + if ((strcmp (tag->valuestring, "(null)") == 0) || strlen(tag->valuestring) == 0) + { + g_warning("%s: tag for %s is an empty string or has value (null).", __func__, cve_id); + return -1; + } + array_add (tags_array, tag->valuestring); + } + + for (int i = 0; i < tags_array->len; i++) + { + gchar *tag = g_ptr_array_index (tags_array, i); + gchar *quoted_tag = sql_quote (tag); - if (parent_id == 0) - root_id = id; - set_root_id (id, root_id); + g_string_append (tags_string, quoted_tag); - cpe_match_rules = cJSON_GetObjectItemCaseSensitive(node, "cpe_match"); - if (cpe_match_rules) - add_cpe_match_rules (id, cpe_match_rules); - child_nodes = cJSON_GetObjectItemCaseSensitive(node, "children"); - load_nodes (id, cveid, root_id, child_nodes); + if (i < tags_array->len - 1) + g_string_append (tags_string, ","); + + g_free (quoted_tag); + } + g_string_append (tags_string, "}"); + g_ptr_array_free (tags_array, TRUE); + } + + gchar *quoted_url = sql_quote (url_value); + + sql("INSERT INTO scap2.cve_references" + " (cve_id, url, tags)" + " VALUES" + " (%llu, '%s', '%s')" + " ON CONFLICT (cve_id, url) DO UPDATE" + " SET tags = EXCLUDED.tags;", + cve_db_id, + quoted_url, + tags_string->str ?: "{}"); + + g_free (quoted_url); + if (tags_string) + g_string_free (tags_string, TRUE); + } + return 0; +} + +/** + * @brief Handle the configurations of a CVE. + * + * @param[in] cve_db_id The id of the CVE the configurations belong to. + * @param[in] cve_id The id of the CVE. + * @param[in] configurations_json JSON array containing the configurations. + * + * @return 0 on success, -1 on error. + */ +static int +handle_cve_configurations (resource_t cve_db_id, char * cve_id, cJSON* configurations_json) +{ + cJSON *configuration_item; + + cJSON_ArrayForEach (configuration_item, configurations_json) + { + cJSON *nodes_array, *node_item; + resource_t id, root_id; + char *config_operator; + int negate; + + nodes_array = cJSON_GetObjectItemCaseSensitive (configuration_item, "nodes"); + if (!cJSON_IsArray (nodes_array)) + { + g_warning("%s: 'nodes' field missing or not an array for %s.", + __func__, cve_id); + return -1; + } + + root_id = -1; + config_operator = json_object_item_string (configuration_item, "operator"); + if (config_operator) + { + negate = json_object_item_boolean (configuration_item, "negate", 0); + id = save_node (cve_db_id, config_operator, negate); + set_root_id (id, id); + root_id = id; + } + + char *node_operator; + cJSON_ArrayForEach(node_item, nodes_array) + { + node_operator = json_object_item_string (node_item, "operator"); + if (node_operator == NULL) + { + g_warning("%s: operator missing for %s.", __func__, cve_id); + return -1; + } + + negate = json_object_item_boolean (node_item, "negate", 0); + + cJSON *cpe_matches_array; + cpe_matches_array = cJSON_GetObjectItemCaseSensitive (node_item, "cpeMatch"); + if (!cJSON_IsArray (cpe_matches_array)) + { + g_warning("%s: cpeMatch missing or not an array for %s.", __func__, cve_id); + return -1; + } + + id = save_node (cve_db_id, node_operator, negate); + if(root_id < 0) + root_id = id; + set_root_id (id, root_id); + + cJSON *cpe_match_item; + cJSON_ArrayForEach(cpe_match_item, cpe_matches_array) + { + char *match_criteria_id; + int vulnerable; + gchar *quoted_match_criteria_id; + + vulnerable = json_object_item_boolean (cpe_match_item, "vulnerable", -1); + if (vulnerable == -1) + { + g_warning("%s: vulnerable missing in cpeMatch for %s.", __func__, cve_id); + return -1; + } + match_criteria_id = json_object_item_string (cpe_match_item, "matchCriteriaId"); + if (match_criteria_id == NULL) + { + g_warning("%s: matchCriteriaId missing in cpeMatch for %s.", __func__, cve_id); + return -1; + } + quoted_match_criteria_id = sql_quote (match_criteria_id); + + sql("INSERT INTO scap2.cpe_nodes_match_criteria" + " (node_id, vulnerable, match_criteria_id)" + " VALUES" + " (%llu, %d, lower ('%s'))", + id, + vulnerable ? 1 : 0, + quoted_match_criteria_id); + + g_free (quoted_match_criteria_id); + } + } } + return 0; } /** @@ -3432,23 +3474,26 @@ static int handle_json_cve_item (cJSON *item) { cJSON *cve_json; - cJSON *cve_data_meta_json; - - char *cve_id; + char *cve_id, *vector; + double score_dbl; resource_t cve_db_id; - cve_json = cJSON_GetObjectItemCaseSensitive(item, "cve"); - cve_data_meta_json = cJSON_GetObjectItemCaseSensitive(cve_json, "CVE_data_meta"); - cve_id = json_object_item_string (cve_data_meta_json, "ID"); + cve_json = cJSON_GetObjectItemCaseSensitive (item, "cve"); + if (!cJSON_IsObject (cve_json)) + { + g_warning("%s: 'cve' field is missing or not an object.", __func__); + return -1; + } + cve_id = json_object_item_string (cve_json, "id"); if (cve_id == NULL) { - g_warning("%s: ID missing.", __func__); + g_warning("%s: cve id missing.", __func__); return -1; } char *published; time_t published_time; - published = json_object_item_string (item, "publishedDate"); + published = json_object_item_string (cve_json, "published"); if (published == NULL) { g_warning("%s: publishedDate missing for %s.", __func__, cve_id); @@ -3458,7 +3503,7 @@ handle_json_cve_item (cJSON *item) char *modified; time_t modified_time; - modified = json_object_item_string (item, "lastModifiedDate"); + modified = json_object_item_string (cve_json, "lastModified"); if (modified == NULL) { g_warning("%s: lastModifiedDate missing for %s.", __func__, cve_id); @@ -3466,55 +3511,64 @@ handle_json_cve_item (cJSON *item) } modified_time = parse_iso_time (modified); - cJSON *impact_json; - cJSON *base_metric_json; - char * cvss_key; - cJSON *cvss_json; - char * vector; - double score_dbl; + cJSON *metrics_json; + cJSON *cvss_metric_array; - impact_json = cJSON_GetObjectItemCaseSensitive(item, "impact"); - if (impact_json == NULL) + metrics_json = cJSON_GetObjectItemCaseSensitive(cve_json, "metrics"); + if (!cJSON_IsObject (metrics_json)) { - g_warning("%s: Impact missing for %s.", __func__, cve_id); + g_warning("%s: Metrics missing or not an object for %s.", __func__, cve_id); return -1; } - base_metric_json = cJSON_GetObjectItemCaseSensitive(impact_json, "baseMetricV4"); - if (base_metric_json != NULL) - cvss_key = "cvssV4"; - else + + gboolean cvss_metric_found = FALSE; + + const char *cvss_metric_keys[] = {"cvssMetricV40", "cvssMetricV31", "cvssMetricV30", "cvssMetricV2"}; + for (int i = 0; i < 4; i++) { - base_metric_json = cJSON_GetObjectItemCaseSensitive(impact_json, "baseMetricV3"); - if (base_metric_json != NULL) - cvss_key = "cvssV3"; - else + cvss_metric_array = cJSON_GetObjectItemCaseSensitive(metrics_json, cvss_metric_keys[i]); + if (cJSON_IsArray (cvss_metric_array) && cJSON_GetArraySize(cvss_metric_array) > 0) { - base_metric_json = cJSON_GetObjectItemCaseSensitive(impact_json, "baseMetricV2"); - if (base_metric_json != NULL) - cvss_key = "cvssV2"; - else - cvss_key = NULL; + cvss_metric_found = TRUE; + break; } } - if (cvss_key != NULL) + + if (cvss_metric_found) { - cvss_json = cJSON_GetObjectItemCaseSensitive(base_metric_json, cvss_key); - if (cvss_json == NULL) - { - g_warning("%s: %s missing for %s.", __func__, cvss_key, cve_id); - return -1; - } - score_dbl = json_object_item_double (cvss_json, "baseScore", SEVERITY_MISSING); - if (score_dbl == SEVERITY_MISSING) - { - g_warning("%s: baseScore missing in %s for %s.", __func__, cvss_key, cve_id); - return -1; - } - vector = json_object_item_string (cvss_json, "vectorString"); - if (vector == NULL) + cJSON *cvss_json; + cJSON *cvss_metric_item; + char *source_type; + + cJSON_ArrayForEach (cvss_metric_item, cvss_metric_array) { - g_warning("%s: vectorString missing for %s.", __func__, cve_id); - return -1; + source_type = json_object_item_string (cvss_metric_item, "type"); + if (source_type == NULL) + { + g_warning("%s: type missing in CVSS metric for %s.", __func__, cve_id); + return -1; + } + else if (strcmp (source_type, "Primary")) + continue; + + cvss_json = cJSON_GetObjectItemCaseSensitive(cvss_metric_item, "cvssData"); + if (!cJSON_IsObject (cvss_json)) + { + g_warning("%s: cvssData missing or not an object for %s.", __func__, cve_id); + return -1; + } + score_dbl = json_object_item_double (cvss_json, "baseScore", SEVERITY_MISSING); + if (score_dbl == SEVERITY_MISSING) + { + g_warning("%s: baseScore missing for %s.", __func__, cve_id); + return -1; + } + vector = json_object_item_string (cvss_json, "vectorString"); + if (vector == NULL) + { + g_warning("%s: vectorString missing for %s.", __func__, cve_id); + return -1; + } } } else @@ -3523,25 +3577,17 @@ handle_json_cve_item (cJSON *item) vector = NULL; } - cJSON *description_json; - cJSON *description_data_json; + cJSON *descriptions_json; cJSON *description_item_json; char *description_value; - description_json = cJSON_GetObjectItemCaseSensitive(cve_json, "description"); - if (description_json == NULL) + descriptions_json = cJSON_GetObjectItemCaseSensitive(cve_json, "descriptions"); + if (!cJSON_IsArray(descriptions_json)) { - g_warning("%s: description missing for %s.", __func__, cve_id); + g_warning("%s: descriptions for %s is missing or not an array.", __func__, cve_id); return -1; } - - description_data_json = cJSON_GetObjectItemCaseSensitive(description_json, "description_data"); - if (description_data_json == NULL) - { - g_warning("%s: description_data missing for %s.", __func__, cve_id); - return -1; - } - cJSON_ArrayForEach (description_item_json, description_data_json) + cJSON_ArrayForEach (description_item_json, descriptions_json) { char *lang = json_object_item_string (description_item_json, "lang"); if (lang != NULL && strcmp(lang, "en") == 0) @@ -3577,24 +3623,27 @@ handle_json_cve_item (cJSON *item) g_free (quoted_description); - cJSON *configurations_json; - cJSON *nodes_json; - - configurations_json = - cJSON_GetObjectItemCaseSensitive(item, "configurations"); - if (configurations_json == NULL) + cJSON *configurations_array; + configurations_array = cJSON_GetObjectItemCaseSensitive(cve_json, "configurations"); + if (! cJSON_IsArray(configurations_array)) { - g_warning("%s: configurations missing for %s.", __func__, cve_id); + g_warning("%s: configurations for %s is missing or not an array.", __func__, cve_id); return -1; } - nodes_json = - cJSON_GetObjectItemCaseSensitive(configurations_json, "nodes"); - if (nodes_json == NULL) + + if (handle_cve_configurations (cve_db_id, cve_id, configurations_array)) + return -1; + + cJSON *references_array; + references_array = cJSON_GetObjectItemCaseSensitive(cve_json, "references"); + if (!cJSON_IsArray(references_array)) { - g_warning("%s: nodes missing for %s.", __func__, cve_id); + g_warning("%s: references for %s is missing or not an array.", __func__, cve_id); return -1; } - load_nodes (0, cve_db_id, 0, nodes_json); + + if (handle_cve_references (cve_db_id, cve_id, references_array)) + return -1; return 0; } @@ -3657,7 +3706,7 @@ update_cve_json (const gchar *cve_path, GHashTable *hashed_cpes) gvm_json_pull_parser_next (&parser, &event); gvm_json_path_elem_t *path_tail = g_queue_peek_tail (event.path); if (event.type == GVM_JSON_PULL_EVENT_ARRAY_START && path_tail && - path_tail->key && strcmp (path_tail->key, "CVE_Items") == 0) + path_tail->key && strcmp (path_tail->key, "vulnerabilities") == 0) { cve_items_found = TRUE; } @@ -3685,7 +3734,7 @@ update_cve_json (const gchar *cve_path, GHashTable *hashed_cpes) entry = gvm_json_pull_expand_container (&parser, &error_message); if (error_message) { - g_warning ("%s: Error expanding CVE item: %s", __func__, error_message); + g_warning ("%s: Error expanding vulnerability item: %s", __func__, error_message); gvm_json_pull_event_cleanup (&event); gvm_json_pull_parser_cleanup (&parser); cJSON_Delete (entry); @@ -3917,6 +3966,325 @@ update_scap_cves () return 0; } +/** + * @brief Insert a SCAP CPE match string from JSON. + * + * @param[in] inserts Pointer to SQL buffer for match string entries. + * @param[in] matches_inserts Pointer to SQL buffer for match string matches. + * @param[in] match_string_item JSON object from the matchStrings list. + * + * @return 0 success, -1 error. + */ +static int +handle_json_cpe_match_string (inserts_t *inserts, inserts_t *matches_inserts, + cJSON *match_string_item) +{ + cJSON *match_string, *matches_array; + char *criteria, *match_criteria_id, *status, *ver_se; + gchar *quoted_version_start_incl, *quoted_version_start_excl; + gchar *quoted_version_end_incl, *quoted_version_end_excl; + gchar *quoted_criteria, *quoted_match_criteria_id, *quoted_cpe_name_id; + int first; + + assert (inserts); + assert (matches_inserts); + + match_string = cJSON_GetObjectItemCaseSensitive (match_string_item, "matchString"); + if (!cJSON_IsObject (match_string)) + { + g_warning ("%s: 'matchString' field is missing or not an object", + __func__); + return -1; + } + + criteria = json_object_item_string (match_string, "criteria"); + if (criteria == NULL) + { + g_warning ("%s: 'criteria' field missing or not a string", __func__); + return -1; + } + + match_criteria_id = json_object_item_string (match_string, "matchCriteriaId"); + if (match_criteria_id == NULL) + { + g_warning ("%s: 'matchCriteriaId' field missing or not a string", __func__); + return -1; + } + + status = json_object_item_string (match_string, "status"); + if (status == NULL) + { + g_warning ("%s: 'status' field missing or not a string", __func__); + return -1; + } + + ver_se = json_object_item_string (match_string, "versionStartIncluding"); + if (ver_se == NULL) + quoted_version_start_incl = g_strdup ("NULL"); + else + quoted_version_start_incl = g_strdup_printf ("'%s'", ver_se); + + ver_se = json_object_item_string (match_string, "versionStartExcluding"); + if (ver_se == NULL) + quoted_version_start_excl = g_strdup ("NULL"); + else + quoted_version_start_excl = g_strdup_printf ("'%s'", ver_se); + + ver_se = json_object_item_string (match_string, "versionEndIncluding"); + if (ver_se == NULL) + quoted_version_end_incl = g_strdup ("NULL"); + else + quoted_version_end_incl = g_strdup_printf ("'%s'", ver_se); + + ver_se = json_object_item_string (match_string, "versionEndExcluding"); + if (ver_se == NULL) + quoted_version_end_excl = g_strdup ("NULL"); + else + quoted_version_end_excl = g_strdup_printf ("'%s'", ver_se); + + quoted_match_criteria_id = sql_quote (match_criteria_id); + quoted_criteria = sql_quote (criteria); + + first = inserts_check_size (inserts); + + g_string_append_printf (inserts->statement, + "%s (lower('%s'), '%s', %s, %s, %s, %s, '%s')", + first ? "" : ",", + quoted_match_criteria_id, + quoted_criteria, + quoted_version_start_incl, + quoted_version_start_excl, + quoted_version_end_incl, + quoted_version_end_excl, + status); + + inserts->current_chunk_size++; + + g_free (quoted_criteria); + g_free (quoted_version_start_incl); + g_free (quoted_version_start_excl); + g_free (quoted_version_end_incl); + g_free (quoted_version_end_excl); + + matches_array = cJSON_GetObjectItemCaseSensitive (match_string, "matches"); + + if (cJSON_IsArray (matches_array) && cJSON_GetArraySize (matches_array) > 0) + { + cJSON *match_item; + cJSON_ArrayForEach (match_item, matches_array) + { + char *cpe_name_id; + + cpe_name_id = json_object_item_string (match_item, "cpeNameId"); + if (cpe_name_id == NULL) + { + g_warning ("%s: 'cpeNameId' field missing or not a string", __func__); + g_free (quoted_match_criteria_id); + return -1; + } + + quoted_cpe_name_id = sql_quote (cpe_name_id); + + first = inserts_check_size (matches_inserts); + + g_string_append_printf (matches_inserts->statement, + "%s (lower('%s'), lower('%s'))", + first ? "" : ",", + quoted_match_criteria_id, + quoted_cpe_name_id); + + matches_inserts->current_chunk_size++; + + g_free (quoted_cpe_name_id); + } + } + + g_free (quoted_match_criteria_id); + return 0; +} + +/** + * @brief Updates the CPE match strings in the SCAP database. + * + * @return 0 success, -1 error. + */ +static int +update_scap_cpe_match_strings () +{ + gchar *current_json_path; + FILE *cpe_match_strings_file; + gvm_json_pull_event_t event; + gvm_json_pull_parser_t parser; + inserts_t inserts, matches_inserts; + + current_json_path = g_build_filename (GVM_SCAP_DATA_DIR, + "cpe_match_strings.json.gz", + NULL); + int fd = open(current_json_path, O_RDONLY); + + if (fd < 0 && errno == ENOENT) + { + g_free (current_json_path); + current_json_path = g_build_filename (GVM_SCAP_DATA_DIR, + "cpe_match_strings.json", + NULL); + fd = open(current_json_path, O_RDONLY); + } + + if (fd < 0) + { + int ret; + if (errno == ENOENT) + { + g_info ("%s: CPE match strings file '%s' not found", + __func__, current_json_path); + ret = 0; + } + else + { + g_warning ("%s: Failed to open CPE match strings file: %s", + __func__, strerror (errno)); + ret = -1; + } + g_free (current_json_path); + return ret; + } + + cpe_match_strings_file = gvm_gzip_open_file_reader_fd (fd); + + if (cpe_match_strings_file == NULL) + { + g_warning ("%s: Failed to convert file descriptor to FILE*: %s", + __func__, + strerror (errno)); + g_free (current_json_path); + close (fd); + return -1; + } + + g_info ("Updating CPE match strings from %s", current_json_path); + g_free (current_json_path); + + gvm_json_pull_event_init (&event); + gvm_json_pull_parser_init (&parser, cpe_match_strings_file); + + gvm_json_pull_parser_next (&parser, &event); + + if (event.type == GVM_JSON_PULL_EVENT_OBJECT_START) + { + gboolean cpe_match_strings_found = FALSE; + while (!cpe_match_strings_found) + { + gvm_json_pull_parser_next (&parser, &event); + gvm_json_path_elem_t *path_tail = g_queue_peek_tail (event.path); + if (event.type == GVM_JSON_PULL_EVENT_ARRAY_START + && path_tail && strcmp (path_tail->key, "matchStrings") == 0) + { + cpe_match_strings_found = TRUE; + } + else if (event.type == GVM_JSON_PULL_EVENT_ERROR) + { + g_warning ("%s: Parser error: %s", __func__, event.error_message); + gvm_json_pull_event_cleanup (&event); + gvm_json_pull_parser_cleanup (&parser); + fclose (cpe_match_strings_file); + return -1; + } + else if (event.type == GVM_JSON_PULL_EVENT_OBJECT_END + && g_queue_is_empty (event.path)) + { + g_warning ("%s: Unexpected json object end. Missing CPE matched strings field", __func__); + gvm_json_pull_event_cleanup (&event); + gvm_json_pull_parser_cleanup (&parser); + fclose (cpe_match_strings_file); + return -1; + } + } + + sql_begin_immediate (); + inserts_init (&inserts, + CPE_MAX_CHUNK_SIZE, + setting_secinfo_sql_buffer_threshold_bytes (), + "INSERT INTO scap2.cpe_match_range" + " (match_criteria_id, cpe, version_start_incl," + " version_start_excl, version_end_incl, version_end_excl," + " status)" + " VALUES ", + " ON CONFLICT (match_criteria_id) DO UPDATE" + " SET cpe = EXCLUDED.cpe," + " version_start_incl = EXCLUDED.version_start_incl," + " version_start_excl = EXCLUDED.version_start_excl," + " version_end_incl = EXCLUDED.version_end_incl," + " version_end_excl = EXCLUDED.version_end_excl," + " status = EXCLUDED.status"); + + inserts_init (&matches_inserts, 10, + setting_secinfo_sql_buffer_threshold_bytes (), + "INSERT INTO scap2.cpe_matches" + " (match_criteria_id, cpe_name_id)" + " VALUES ", + ""); + + gvm_json_pull_parser_next (&parser, &event); + while (event.type == GVM_JSON_PULL_EVENT_OBJECT_START) + { + gchar *error_message; + cJSON *cpe_match_string_item = gvm_json_pull_expand_container (&parser, &error_message); + if (error_message) + { + g_warning ("%s: Error expanding match string item: %s", __func__, error_message); + cJSON_Delete (cpe_match_string_item); + inserts_free (&inserts); + inserts_free (&matches_inserts); + sql_commit (); + g_warning ("Update of cpe match strings failed"); + gvm_json_pull_event_cleanup (&event); + gvm_json_pull_parser_cleanup (&parser); + fclose (cpe_match_strings_file); + return -1; + } + if (handle_json_cpe_match_string (&inserts, &matches_inserts, cpe_match_string_item)) + { + cJSON_Delete (cpe_match_string_item); + inserts_free (&inserts); + inserts_free (&matches_inserts); + sql_commit (); + g_warning ("Update of cpe match strings failed"); + gvm_json_pull_event_cleanup (&event); + gvm_json_pull_parser_cleanup (&parser); + fclose (cpe_match_strings_file); + return -1; + } + cJSON_Delete (cpe_match_string_item); + gvm_json_pull_parser_next (&parser, &event); + } + } + else if (event.type == GVM_JSON_PULL_EVENT_ERROR) + { + g_warning ("%s: Parser error: %s", __func__, event.error_message); + gvm_json_pull_event_cleanup (&event); + gvm_json_pull_parser_cleanup (&parser); + fclose (cpe_match_strings_file); + return -1; + } + else + { + g_warning ("%s: CVE affected products file is not a JSON object.", __func__); + gvm_json_pull_event_cleanup (&event); + gvm_json_pull_parser_cleanup (&parser); + fclose (cpe_match_strings_file); + return -1; + } + + inserts_run (&inserts, TRUE); + inserts_run (&matches_inserts, TRUE); + sql_commit (); + gvm_json_pull_event_cleanup (&event); + gvm_json_pull_parser_cleanup (&parser); + fclose (cpe_match_strings_file); + return 0; +} + /** * @brief Adds a EPSS score entry to an SQL inserts buffer. * @@ -5120,6 +5488,15 @@ update_scap (gboolean reset_scap_db) return -1; } + g_debug ("%s: update cpe match strings", __func__); + setproctitle ("Syncing SCAP: Updating CPE Match Strings"); + + if (update_scap_cpe_match_strings () == -1) + { + abort_scap_update (); + return -1; + } + g_debug ("%s: update cves", __func__); setproctitle ("Syncing SCAP: Updating CVEs"); diff --git a/src/schema_formats/XML/GMP.xml.in b/src/schema_formats/XML/GMP.xml.in index eb2feaf73..216f603d5 100644 --- a/src/schema_formats/XML/GMP.xml.in +++ b/src/schema_formats/XML/GMP.xml.in @@ -13129,6 +13129,8 @@ END:VCALENDAR epss nvts cert + configuration_nodes + references raw_data A CVE info element @@ -13249,6 +13251,161 @@ END:VCALENDAR + + configuration_nodes + List of configuration nodes. Only when details were requested + + node + + + node + A configuration node for the CVE + + operator + match_criteria + node + + + operator + The operator for the match criteria + + text + + + + match_criteria + The match criteria for the node + + match_string + vulnerable + version_start_including + version_start_excluding + version_end_including + version_end_excluding + matched_cpes + + + match_string + A CPE Match string + + text + + + + vulnerable + A true or false value, whether matching CPEs are considered vulnerable + + text + + + + version_start_including + From version (including) + + text + + + + version_start_excluding + From version (excluding) + + text + + + + version_end_including + Up to version (including) + + text + + + + version_end_excluding + Up to version (exluding) + + text + + + + matched_cpes + List of matching CPEs + + name + deprecated + deprecated_by + + + name + Name of the matching CPE + + text + + + + deprecated + A true or false value, whether the CPE is deprecated + + text + + + + deprecated_by + CPE ID the current CPE is deprecated by. Only when the CPE is deprecated + + + cpe_id + uuid + CPE ID the current CPE is deprecated by + + + + + + + node + Nodes can contain other nested nodes + + node + + + + + + references + References of this CVE. Only when details were requested + + reference + + + reference + Reference of the CVE + + url + tags + + + url + The URL of the reference + + text + + + + tags + List of tags for the reference + + tag + + + tag + Tag for a reference + + text + + + + + raw_data Source representation of the information. Only when details were requested From ba2dcd533f7d8dca8a56b1587f36ec982fd8ee85 Mon Sep 17 00:00:00 2001 From: Ahmed Abdelsalam Date: Fri, 1 Nov 2024 17:39:00 +0100 Subject: [PATCH 16/26] Address minor issues --- src/gmp.c | 10 ++++------ src/manage_sql_secinfo.c | 8 ++++---- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/gmp.c b/src/gmp.c index fe766df12..9857f9825 100644 --- a/src/gmp.c +++ b/src/gmp.c @@ -13283,13 +13283,12 @@ print_cpe_match_nodes_xml(resource_t node, GString *buffer) init_cpe_match_range_iterator (&cpe_match_ranges, node); while (next (&cpe_match_ranges)) { - const gchar *vsi, *vse, *vei, *vee, *match_criteria_id, *range_uri_product; + const gchar *vsi, *vse, *vei, *vee, *match_criteria_id, *match_string; xml_string_append (buffer, ""); match_criteria_id = cpe_match_range_iterator_match_criteria_id (&cpe_match_ranges); - range_uri_product - = fs_cpe_to_uri_cpe (cpe_match_range_iterator_cpe (&cpe_match_ranges)); - xml_string_append (buffer, "%s", range_uri_product?: ""); + match_string = cpe_match_range_iterator_cpe (&cpe_match_ranges); + xml_string_append (buffer, "%s", match_string?: ""); xml_string_append (buffer, "%s", cpe_match_range_iterator_vulnerable (&cpe_match_ranges) != 0 ? "1" @@ -13334,8 +13333,7 @@ print_cpe_match_nodes_xml(resource_t node, GString *buffer) } cleanup_iterator (&cpes); - xml_string_append (buffer, ""); - xml_string_append (buffer, "%s", cpe?: ""); + xml_string_append (buffer, "", cpe?: ""); xml_string_append (buffer, "%s", deprecated ? "1" : "0"); diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index 8b3cdbde0..2fd61350e 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -3492,7 +3492,7 @@ handle_cve_configurations (resource_t cve_db_id, char * cve_id, cJSON* configura sql("INSERT INTO scap2.cpe_nodes_match_criteria" " (node_id, vulnerable, match_criteria_id)" " VALUES" - " (%llu, %d, lower ('%s'))", + " (%llu, %d, '%s')", id, vulnerable ? 1 : 0, quoted_match_criteria_id); @@ -4085,12 +4085,12 @@ handle_json_cpe_match_string (inserts_t *inserts, inserts_t *matches_inserts, quoted_version_end_excl = g_strdup_printf ("'%s'", ver_se); quoted_match_criteria_id = sql_quote (match_criteria_id); - quoted_criteria = sql_quote (criteria); + quoted_criteria = fs_to_uri_convert_and_quote_cpe_name (criteria); first = inserts_check_size (inserts); g_string_append_printf (inserts->statement, - "%s (lower('%s'), '%s', %s, %s, %s, %s, '%s')", + "%s ('%s', '%s', %s, %s, %s, %s, '%s')", first ? "" : ",", quoted_match_criteria_id, quoted_criteria, @@ -4130,7 +4130,7 @@ handle_json_cpe_match_string (inserts_t *inserts, inserts_t *matches_inserts, first = inserts_check_size (matches_inserts); g_string_append_printf (matches_inserts->statement, - "%s (lower('%s'), lower('%s'))", + "%s ('%s', '%s')", first ? "" : ",", quoted_match_criteria_id, quoted_cpe_name_id); From 03f546ad9b58bd3648fc3e8ab9319d9599c2081f Mon Sep 17 00:00:00 2001 From: Ahmed Abdelsalam Date: Fri, 1 Nov 2024 17:41:53 +0100 Subject: [PATCH 17/26] Update names of CVE JSON feed files --- src/manage_sql_secinfo.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index 2fd61350e..19bea8b35 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -3961,8 +3961,8 @@ update_scap_cves () gboolean read_json = FALSE; while ((cve_path = g_dir_read_name (dir))) { - if (fnmatch ("nvdcve-1.1-*.json.gz", cve_path, 0) == 0 || - fnmatch ("nvdcve-1.1-*.json", cve_path, 0) == 0) + if (fnmatch ("nvdcve-2.0-*.json.gz", cve_path, 0) == 0 || + fnmatch ("nvdcve-2.0-*.json", cve_path, 0) == 0) { read_json = TRUE; break; @@ -3973,8 +3973,8 @@ update_scap_cves () count = 0; while ((cve_path = g_dir_read_name (dir))) { - if ((fnmatch ("nvdcve-1.1-*.json.gz", cve_path, 0) == 0 || - fnmatch ("nvdcve-1.1-*.json", cve_path, 0) == 0) + if ((fnmatch ("nvdcve-2.0-*.json.gz", cve_path, 0) == 0 || + fnmatch ("nvdcve-2.0-*.json", cve_path, 0) == 0) && read_json) { if (update_cve_json (cve_path, hashed_cpes)) From 64983cac2ae8d8cc66234ab078e159c5236b35e6 Mon Sep 17 00:00:00 2001 From: Ahmed Abdelsalam Date: Mon, 4 Nov 2024 14:13:16 +0100 Subject: [PATCH 18/26] Minor code changes and adjusting formatting --- src/gmp.c | 64 ++++--- src/manage_pg.c | 4 + src/manage_sql.c | 17 +- src/manage_sql_secinfo.c | 267 +++++++++++++++++------------- src/schema_formats/XML/GMP.xml.in | 8 + 5 files changed, 218 insertions(+), 142 deletions(-) diff --git a/src/gmp.c b/src/gmp.c index 9857f9825..833434055 100644 --- a/src/gmp.c +++ b/src/gmp.c @@ -13261,15 +13261,17 @@ handle_get_groups (gmp_parser_t *gmp_parser, GError **error) * @param[in] buffer Buffer into which to print match node. */ static void -print_cpe_match_nodes_xml(resource_t node, GString *buffer) +print_cpe_match_nodes_xml (resource_t node, GString *buffer) { iterator_t cpe_match_nodes, cpe_match_ranges; - const char *operator = NULL; - int negate = 0; init_iterator (&cpe_match_nodes, - "SELECT operator, negate FROM scap.cpe_match_nodes WHERE id = %llu;", + "SELECT operator, negate" + " FROM scap.cpe_match_nodes WHERE id = %llu;", node); + + const char *operator = NULL; + int negate = 0; while (next (&cpe_match_nodes)) { operator = iterator_string (&cpe_match_nodes, 0); @@ -13286,17 +13288,23 @@ print_cpe_match_nodes_xml(resource_t node, GString *buffer) const gchar *vsi, *vse, *vei, *vee, *match_criteria_id, *match_string; xml_string_append (buffer, ""); - match_criteria_id = cpe_match_range_iterator_match_criteria_id (&cpe_match_ranges); + match_criteria_id + = cpe_match_range_iterator_match_criteria_id (&cpe_match_ranges); match_string = cpe_match_range_iterator_cpe (&cpe_match_ranges); - xml_string_append (buffer, "%s", match_string?: ""); - xml_string_append (buffer, "%s", + + xml_string_append (buffer, + "%s", + match_string?: ""); + xml_string_append (buffer, + "%s", cpe_match_range_iterator_vulnerable (&cpe_match_ranges) != 0 ? "1" : "0"); - vsi = cpe_match_range_iterator_version_start_incl(&cpe_match_ranges); - vse = cpe_match_range_iterator_version_start_excl(&cpe_match_ranges); - vei = cpe_match_range_iterator_version_end_incl(&cpe_match_ranges); - vee = cpe_match_range_iterator_version_end_excl(&cpe_match_ranges); + + vsi = cpe_match_range_iterator_version_start_incl (&cpe_match_ranges); + vse = cpe_match_range_iterator_version_start_excl (&cpe_match_ranges); + vei = cpe_match_range_iterator_version_end_incl (&cpe_match_ranges); + vee = cpe_match_range_iterator_version_end_excl (&cpe_match_ranges); xml_string_append (buffer, "%s", @@ -13335,7 +13343,7 @@ print_cpe_match_nodes_xml(resource_t node, GString *buffer) xml_string_append (buffer, "", cpe?: ""); xml_string_append (buffer, - "%s", + "%s", deprecated ? "1" : "0"); if (deprecated) { @@ -13366,7 +13374,7 @@ print_cpe_match_nodes_xml(resource_t node, GString *buffer) * */ static void -print_cve_affected_software_configs_xml (gchar *cve_uuid, GString *result) +print_cve_configurations_xml (const gchar *cve_uuid, GString *result) { iterator_t cpe_match_root_nodes; xml_string_append (result, ""); @@ -13377,14 +13385,15 @@ print_cve_affected_software_configs_xml (gchar *cve_uuid, GString *result) iterator_t cpe_match_node_childs; root_node = cpe_match_nodes_iterator_root_id (&cpe_match_root_nodes); xml_string_append (result, ""); - print_cpe_match_nodes_xml(root_node, result); + print_cpe_match_nodes_xml (root_node, result); init_cpe_match_node_childs_iterator (&cpe_match_node_childs, root_node); while (next (&cpe_match_node_childs)) { resource_t child_node; - child_node = cpe_match_node_childs_iterator_id (&cpe_match_node_childs); + child_node = + cpe_match_node_childs_iterator_id (&cpe_match_node_childs); xml_string_append (result, ""); - print_cpe_match_nodes_xml(child_node, result); + print_cpe_match_nodes_xml (child_node, result); xml_string_append (result, ""); } xml_string_append (result, ""); @@ -13402,7 +13411,7 @@ print_cve_affected_software_configs_xml (gchar *cve_uuid, GString *result) * */ static void -print_cve_references_xml (gchar *cve_uuid, GString *result) +print_cve_references_xml (const gchar *cve_uuid, GString *result) { iterator_t references; init_cve_reference_iterator (&references, cve_uuid); @@ -13410,20 +13419,26 @@ print_cve_references_xml (gchar *cve_uuid, GString *result) while (next (&references)) { xml_string_append (result, ""); - xml_string_append (result, "%s", cve_reference_iterator_url (&references)); + xml_string_append (result, + "%s", + cve_reference_iterator_url (&references)); xml_string_append (result, ""); const char * tags_array = cve_reference_iterator_tags (&references); - if(tags_array && strlen(tags_array) > 2) + if(tags_array && strlen (tags_array) > 2) { - char *trimmed_array = g_strndup (tags_array + 1, strlen (tags_array) - 2); + char *trimmed_array + = g_strndup (tags_array + 1, strlen (tags_array) - 2); gchar **tags, **current_tag; tags = g_strsplit (trimmed_array, ",", -1); current_tag = tags; while (*current_tag) { - if (strlen (*current_tag) > 2 && (*current_tag)[0] == '"' && (*current_tag)[strlen (*current_tag) - 1] == '"') + if (strlen (*current_tag) > 2 + && (*current_tag)[0] == '"' + && (*current_tag)[strlen (*current_tag) - 1] == '"') { - char *trimmed_tag = g_strndup (*current_tag + 1, strlen (*current_tag) - 2); + char *trimmed_tag = g_strndup (*current_tag + 1, + strlen (*current_tag) - 2); xml_string_append (result, "%s", trimmed_tag); g_free (trimmed_tag); } @@ -13816,10 +13831,9 @@ handle_get_info (gmp_parser_t *gmp_parser, GError **error) } g_string_append (result, ""); - gchar *cve_uuid = g_strdup(get_iterator_uuid (&info)); - print_cve_affected_software_configs_xml (cve_uuid, result); + const gchar *cve_uuid = get_iterator_uuid (&info); + print_cve_configurations_xml (cve_uuid, result); print_cve_references_xml (cve_uuid, result); - g_free(cve_uuid); } } else if (g_strcmp0 ("cert_bund_adv", get_info_data->type) == 0) diff --git a/src/manage_pg.c b/src/manage_pg.c index 893d3e240..da79adb05 100644 --- a/src/manage_pg.c +++ b/src/manage_pg.c @@ -3649,6 +3649,10 @@ manage_db_add_constraints (const gchar *name) sql ("ALTER TABLE scap2.cpe_match_range" " ADD UNIQUE (match_criteria_id);"); + sql ("ALTER TABLE scap2.cpe_matches" + " ALTER match_criteria_id SET NOT NULL," + " ALTER cpe_name_id SET NOT NULL," + " ADD UNIQUE (match_criteria_id, cpe_name_id);"); } else { diff --git a/src/manage_sql.c b/src/manage_sql.c index 71461dc00..ef9cb24e4 100644 --- a/src/manage_sql.c +++ b/src/manage_sql.c @@ -20511,10 +20511,12 @@ init_cpe_match_nodes_iterator (iterator_t* iterator, const char *cpe) gchar *quoted_cpe; quoted_cpe = sql_quote (cpe); init_iterator (iterator, - " SELECT DISTINCT n.root_id" + "SELECT DISTINCT n.root_id" " FROM scap.cpe_match_nodes n" - " JOIN scap.cpe_nodes_match_criteria c ON n.id = c.node_id" - " JOIN scap.cpe_match_range r ON c.match_criteria = r.match_criteria_id" + " JOIN scap.cpe_nodes_match_criteria c" + " ON n.id = c.node_id" + " JOIN scap.cpe_match_range r" + " ON c.match_criteria = r.match_criteria_id" " WHERE cpe like '%s%%';", quoted_cpe); g_free (quoted_cpe); @@ -20534,7 +20536,8 @@ init_cve_cpe_match_nodes_iterator (iterator_t* iterator, const char *cve) init_iterator (iterator, "SELECT DISTINCT root_id" " FROM scap.cpe_match_nodes" - " WHERE cve_id = (SELECT id from scap.cves where uuid = '%s');", + " WHERE cve_id = (SELECT id FROM scap.cves" + " WHERE uuid = '%s');", quoted_cve); g_free (quoted_cve); } @@ -20553,7 +20556,8 @@ init_cve_reference_iterator (iterator_t* iterator, const char *cve) init_iterator (iterator, "SELECT url, array_length(tags, 1), tags" " FROM scap.cve_references" - " WHERE cve_id = (SELECT id from scap.cves where uuid = '%s');", + " WHERE cve_id = (SELECT id FROM scap.cves" + " WHERE uuid = '%s');", quoted_cve); g_free (quoted_cve); } @@ -20736,7 +20740,8 @@ DEF_ACCESS (cpe_match_range_iterator_version_end_excl, 7); * @param[in] match_criteria_id The match criteria id to get the matches for. */ void -init_cpe_match_range_matches_iterator (iterator_t* iterator, const char *match_criteria_id) +init_cpe_match_range_matches_iterator (iterator_t* iterator, + const char *match_criteria_id) { init_iterator (iterator, "SELECT cpe_name_id" diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index 19bea8b35..b7b421ce6 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -3289,7 +3289,7 @@ insert_cve_from_entry (element_t entry, element_t last_modified, * * @param[in] cve_id The id of the CVE to which the tree belongs. * @param[in] operator The operator for the match rules. - * @param[in] negate Whether the match rules are negated. + * @param[in] negate Whether the operator is negated. * * @return The (database) id of the node. */ @@ -3332,75 +3332,80 @@ set_root_id (long int id, long int root_id) * @return 0 on success, -1 on error. */ static int -handle_cve_references (resource_t cve_db_id, char * cve_id, cJSON* reference_json) +handle_cve_references (resource_t cve_db_id, char * cve_id, + cJSON* reference_json) { cJSON *reference_data; cJSON *tags; - cJSON_ArrayForEach(reference_data, reference_json) - { - GString *tags_string; - char *url_value = json_object_item_string (reference_data, "url"); - if (url_value == NULL) + cJSON_ArrayForEach (reference_data, reference_json) { - g_warning("%s: url missing in reference for %s.", __func__, cve_id); - return -1; - } - - tags = cJSON_GetObjectItemCaseSensitive(reference_data, "tags"); - if (cJSON_IsArray(tags)) + GString *tags_string; + char *url_value = json_object_item_string (reference_data, "url"); + if (url_value == NULL) { - array_t *tags_array = make_array (); - tags_string = g_string_new ("{"); - - for (int i = 0; i < cJSON_GetArraySize(tags); i++) - { - cJSON *tag = cJSON_GetArrayItem(tags, i); - if (!cJSON_IsString(tag)) - { - g_warning("%s: tag for %s is NULL or not a string.", __func__, cve_id); - return -1; - } - if ((strcmp (tag->valuestring, "(null)") == 0) || strlen(tag->valuestring) == 0) - { - g_warning("%s: tag for %s is an empty string or has value (null).", __func__, cve_id); - return -1; - } - array_add (tags_array, tag->valuestring); - } - - for (int i = 0; i < tags_array->len; i++) - { - gchar *tag = g_ptr_array_index (tags_array, i); - gchar *quoted_tag = sql_quote (tag); - - g_string_append (tags_string, quoted_tag); - - if (i < tags_array->len - 1) - g_string_append (tags_string, ","); - - g_free (quoted_tag); - } - g_string_append (tags_string, "}"); - g_ptr_array_free (tags_array, TRUE); + g_warning ("%s: url missing in reference for %s.", + __func__, cve_id); + return -1; } - gchar *quoted_url = sql_quote (url_value); - - sql("INSERT INTO scap2.cve_references" - " (cve_id, url, tags)" - " VALUES" - " (%llu, '%s', '%s')" - " ON CONFLICT (cve_id, url) DO UPDATE" - " SET tags = EXCLUDED.tags;", - cve_db_id, - quoted_url, - tags_string->str ?: "{}"); + tags = cJSON_GetObjectItemCaseSensitive (reference_data, "tags"); + if (cJSON_IsArray (tags)) + { + array_t *tags_array = make_array (); + tags_string = g_string_new ("{"); + + for (int i = 0; i < cJSON_GetArraySize (tags); i++) + { + cJSON *tag = cJSON_GetArrayItem (tags, i); + if (!cJSON_IsString (tag)) + { + g_warning ("%s: tag for %s is NULL or not a string.", + __func__, cve_id); + return -1; + } + if ((strcmp (tag->valuestring, "(null)") == 0) + || strlen (tag->valuestring) == 0) + { + g_warning ("%s: tag for %s is empty string or NULL.", + __func__, cve_id); + return -1; + } + array_add (tags_array, tag->valuestring); + } + + for (int i = 0; i < tags_array->len; i++) + { + gchar *tag = g_ptr_array_index (tags_array, i); + gchar *quoted_tag = sql_quote (tag); + + g_string_append (tags_string, quoted_tag); + + if (i < tags_array->len - 1) + g_string_append (tags_string, ","); + + g_free (quoted_tag); + } + g_string_append (tags_string, "}"); + g_ptr_array_free (tags_array, TRUE); + } - g_free (quoted_url); - if (tags_string) - g_string_free (tags_string, TRUE); - } + gchar *quoted_url = sql_quote (url_value); + + sql ("INSERT INTO scap2.cve_references" + " (cve_id, url, tags)" + " VALUES" + " (%llu, '%s', '%s')" + " ON CONFLICT (cve_id, url) DO UPDATE" + " SET tags = EXCLUDED.tags;", + cve_db_id, + quoted_url, + tags_string->str ?: "{}"); + + g_free (quoted_url); + if (tags_string) + g_string_free (tags_string, TRUE); + } return 0; } @@ -3414,7 +3419,8 @@ handle_cve_references (resource_t cve_db_id, char * cve_id, cJSON* reference_jso * @return 0 on success, -1 on error. */ static int -handle_cve_configurations (resource_t cve_db_id, char * cve_id, cJSON* configurations_json) +handle_cve_configurations (resource_t cve_db_id, char * cve_id, + cJSON* configurations_json) { cJSON *configuration_item; @@ -3425,16 +3431,18 @@ handle_cve_configurations (resource_t cve_db_id, char * cve_id, cJSON* configura char *config_operator; int negate; - nodes_array = cJSON_GetObjectItemCaseSensitive (configuration_item, "nodes"); + nodes_array = cJSON_GetObjectItemCaseSensitive (configuration_item, + "nodes"); if (!cJSON_IsArray (nodes_array)) { - g_warning("%s: 'nodes' field missing or not an array for %s.", + g_warning ("%s: 'nodes' field missing or not an array for %s.", __func__, cve_id); return -1; } root_id = -1; - config_operator = json_object_item_string (configuration_item, "operator"); + config_operator = json_object_item_string (configuration_item, + "operator"); if (config_operator) { negate = json_object_item_boolean (configuration_item, "negate", 0); @@ -3449,47 +3457,55 @@ handle_cve_configurations (resource_t cve_db_id, char * cve_id, cJSON* configura node_operator = json_object_item_string (node_item, "operator"); if (node_operator == NULL) { - g_warning("%s: operator missing for %s.", __func__, cve_id); + g_warning ("%s: operator missing for %s.", __func__, cve_id); return -1; } negate = json_object_item_boolean (node_item, "negate", 0); cJSON *cpe_matches_array; - cpe_matches_array = cJSON_GetObjectItemCaseSensitive (node_item, "cpeMatch"); + cpe_matches_array = cJSON_GetObjectItemCaseSensitive (node_item, + "cpeMatch"); if (!cJSON_IsArray (cpe_matches_array)) { - g_warning("%s: cpeMatch missing or not an array for %s.", __func__, cve_id); + g_warning ("%s: cpeMatch missing or not an array for %s.", + __func__, cve_id); return -1; } id = save_node (cve_db_id, node_operator, negate); - if(root_id < 0) + + if (root_id < 0) root_id = id; + set_root_id (id, root_id); cJSON *cpe_match_item; - cJSON_ArrayForEach(cpe_match_item, cpe_matches_array) + cJSON_ArrayForEach (cpe_match_item, cpe_matches_array) { char *match_criteria_id; int vulnerable; gchar *quoted_match_criteria_id; - vulnerable = json_object_item_boolean (cpe_match_item, "vulnerable", -1); + vulnerable = json_object_item_boolean (cpe_match_item, + "vulnerable", -1); if (vulnerable == -1) { - g_warning("%s: vulnerable missing in cpeMatch for %s.", __func__, cve_id); + g_warning ("%s: vulnerable missing in cpeMatch for %s.", + __func__, cve_id); return -1; } - match_criteria_id = json_object_item_string (cpe_match_item, "matchCriteriaId"); + match_criteria_id = json_object_item_string (cpe_match_item, + "matchCriteriaId"); if (match_criteria_id == NULL) { - g_warning("%s: matchCriteriaId missing in cpeMatch for %s.", __func__, cve_id); + g_warning ("%s: matchCriteriaId missing in cpeMatch for %s.", + __func__, cve_id); return -1; } quoted_match_criteria_id = sql_quote (match_criteria_id); - sql("INSERT INTO scap2.cpe_nodes_match_criteria" + sql ("INSERT INTO scap2.cpe_nodes_match_criteria" " (node_id, vulnerable, match_criteria_id)" " VALUES" " (%llu, %d, '%s')", @@ -3523,13 +3539,13 @@ handle_json_cve_item (cJSON *item) cve_json = cJSON_GetObjectItemCaseSensitive (item, "cve"); if (!cJSON_IsObject (cve_json)) { - g_warning("%s: 'cve' field is missing or not an object.", __func__); + g_warning ("%s: 'cve' field is missing or not an object.", __func__); return -1; } cve_id = json_object_item_string (cve_json, "id"); if (cve_id == NULL) { - g_warning("%s: cve id missing.", __func__); + g_warning ("%s: cve id missing.", __func__); return -1; } @@ -3548,7 +3564,7 @@ handle_json_cve_item (cJSON *item) modified = json_object_item_string (cve_json, "lastModified"); if (modified == NULL) { - g_warning("%s: lastModifiedDate missing for %s.", __func__, cve_id); + g_warning ("%s: lastModifiedDate missing for %s.", __func__, cve_id); return -1; } modified_time = parse_iso_time (modified); @@ -3556,20 +3572,28 @@ handle_json_cve_item (cJSON *item) cJSON *metrics_json; cJSON *cvss_metric_array; - metrics_json = cJSON_GetObjectItemCaseSensitive(cve_json, "metrics"); + metrics_json = cJSON_GetObjectItemCaseSensitive (cve_json, "metrics"); if (!cJSON_IsObject (metrics_json)) { - g_warning("%s: Metrics missing or not an object for %s.", __func__, cve_id); + g_warning ("%s: Metrics missing or not an object for %s.", + __func__, cve_id); return -1; } gboolean cvss_metric_found = FALSE; - const char *cvss_metric_keys[] = {"cvssMetricV40", "cvssMetricV31", "cvssMetricV30", "cvssMetricV2"}; + const char *cvss_metric_keys[] = { + "cvssMetricV40", + "cvssMetricV31", + "cvssMetricV30", + "cvssMetricV2"}; + for (int i = 0; i < 4; i++) { - cvss_metric_array = cJSON_GetObjectItemCaseSensitive(metrics_json, cvss_metric_keys[i]); - if (cJSON_IsArray (cvss_metric_array) && cJSON_GetArraySize(cvss_metric_array) > 0) + cvss_metric_array + = cJSON_GetObjectItemCaseSensitive (metrics_json, cvss_metric_keys[i]); + if (cJSON_IsArray (cvss_metric_array) + && cJSON_GetArraySize (cvss_metric_array) > 0) { cvss_metric_found = TRUE; break; @@ -3587,28 +3611,33 @@ handle_json_cve_item (cJSON *item) source_type = json_object_item_string (cvss_metric_item, "type"); if (source_type == NULL) { - g_warning("%s: type missing in CVSS metric for %s.", __func__, cve_id); + g_warning ("%s: type missing in CVSS metric for %s.", + __func__, cve_id); return -1; } else if (strcmp (source_type, "Primary")) continue; - cvss_json = cJSON_GetObjectItemCaseSensitive(cvss_metric_item, "cvssData"); + cvss_json = cJSON_GetObjectItemCaseSensitive (cvss_metric_item, + "cvssData"); if (!cJSON_IsObject (cvss_json)) { - g_warning("%s: cvssData missing or not an object for %s.", __func__, cve_id); + g_warning ("%s: cvssData missing or not an object for %s.", + __func__, cve_id); return -1; } - score_dbl = json_object_item_double (cvss_json, "baseScore", SEVERITY_MISSING); + score_dbl = json_object_item_double (cvss_json, + "baseScore", + SEVERITY_MISSING); if (score_dbl == SEVERITY_MISSING) { - g_warning("%s: baseScore missing for %s.", __func__, cve_id); + g_warning ("%s: baseScore missing for %s.", __func__, cve_id); return -1; } vector = json_object_item_string (cvss_json, "vectorString"); if (vector == NULL) { - g_warning("%s: vectorString missing for %s.", __func__, cve_id); + g_warning ("%s: vectorString missing for %s.", __func__, cve_id); return -1; } } @@ -3623,17 +3652,20 @@ handle_json_cve_item (cJSON *item) cJSON *description_item_json; char *description_value; - descriptions_json = cJSON_GetObjectItemCaseSensitive(cve_json, "descriptions"); - if (!cJSON_IsArray(descriptions_json)) + descriptions_json = cJSON_GetObjectItemCaseSensitive (cve_json, + "descriptions"); + if (!cJSON_IsArray (descriptions_json)) { - g_warning("%s: descriptions for %s is missing or not an array.", __func__, cve_id); + g_warning ("%s: descriptions for %s is missing or not an array.", + __func__, cve_id); return -1; } cJSON_ArrayForEach (description_item_json, descriptions_json) { char *lang = json_object_item_string (description_item_json, "lang"); - if (lang != NULL && strcmp(lang, "en") == 0) - description_value = json_object_item_string (description_item_json, "value"); + if (lang != NULL && strcmp (lang, "en") == 0) + description_value = json_object_item_string (description_item_json, + "value"); } char *quoted_description = sql_quote (description_value); @@ -3666,10 +3698,12 @@ handle_json_cve_item (cJSON *item) g_free (quoted_description); cJSON *configurations_array; - configurations_array = cJSON_GetObjectItemCaseSensitive(cve_json, "configurations"); - if (! cJSON_IsArray(configurations_array)) + configurations_array = cJSON_GetObjectItemCaseSensitive (cve_json, + "configurations"); + if (!cJSON_IsArray (configurations_array)) { - g_warning("%s: configurations for %s is missing or not an array.", __func__, cve_id); + g_warning ("%s: configurations for %s is missing or not an array.", + __func__, cve_id); return -1; } @@ -3677,10 +3711,11 @@ handle_json_cve_item (cJSON *item) return -1; cJSON *references_array; - references_array = cJSON_GetObjectItemCaseSensitive(cve_json, "references"); - if (!cJSON_IsArray(references_array)) + references_array = cJSON_GetObjectItemCaseSensitive (cve_json, "references"); + if (!cJSON_IsArray (references_array)) { - g_warning("%s: references for %s is missing or not an array.", __func__, cve_id); + g_warning ("%s: references for %s is missing or not an array.", + __func__, cve_id); return -1; } @@ -4031,7 +4066,8 @@ handle_json_cpe_match_string (inserts_t *inserts, inserts_t *matches_inserts, assert (inserts); assert (matches_inserts); - match_string = cJSON_GetObjectItemCaseSensitive (match_string_item, "matchString"); + match_string = cJSON_GetObjectItemCaseSensitive (match_string_item, + "matchString"); if (!cJSON_IsObject (match_string)) { g_warning ("%s: 'matchString' field is missing or not an object", @@ -4046,10 +4082,12 @@ handle_json_cpe_match_string (inserts_t *inserts, inserts_t *matches_inserts, return -1; } - match_criteria_id = json_object_item_string (match_string, "matchCriteriaId"); + match_criteria_id = json_object_item_string (match_string, + "matchCriteriaId"); if (match_criteria_id == NULL) { - g_warning ("%s: 'matchCriteriaId' field missing or not a string", __func__); + g_warning ("%s: 'matchCriteriaId' field missing or not a string", + __func__); return -1; } @@ -4120,7 +4158,8 @@ handle_json_cpe_match_string (inserts_t *inserts, inserts_t *matches_inserts, cpe_name_id = json_object_item_string (match_item, "cpeNameId"); if (cpe_name_id == NULL) { - g_warning ("%s: 'cpeNameId' field missing or not a string", __func__); + g_warning ("%s: 'cpeNameId' field missing or not a string", + __func__); g_free (quoted_match_criteria_id); return -1; } @@ -4235,7 +4274,8 @@ update_scap_cpe_match_strings () else if (event.type == GVM_JSON_PULL_EVENT_OBJECT_END && g_queue_is_empty (event.path)) { - g_warning ("%s: Unexpected json object end. Missing CPE matched strings field", __func__); + g_warning ("%s: Unexpected json object end. Missing matchStrings field", + __func__); gvm_json_pull_event_cleanup (&event); gvm_json_pull_parser_cleanup (&parser); fclose (cpe_match_strings_file); @@ -4271,27 +4311,31 @@ update_scap_cpe_match_strings () while (event.type == GVM_JSON_PULL_EVENT_OBJECT_START) { gchar *error_message; - cJSON *cpe_match_string_item = gvm_json_pull_expand_container (&parser, &error_message); + cJSON *cpe_match_string_item + = gvm_json_pull_expand_container (&parser, &error_message); if (error_message) { - g_warning ("%s: Error expanding match string item: %s", __func__, error_message); + g_warning ("%s: Error expanding match string item: %s", + __func__, error_message); cJSON_Delete (cpe_match_string_item); inserts_free (&inserts); inserts_free (&matches_inserts); sql_commit (); - g_warning ("Update of cpe match strings failed"); + g_warning ("Update of CPE match strings failed"); gvm_json_pull_event_cleanup (&event); gvm_json_pull_parser_cleanup (&parser); fclose (cpe_match_strings_file); return -1; } - if (handle_json_cpe_match_string (&inserts, &matches_inserts, cpe_match_string_item)) + if (handle_json_cpe_match_string (&inserts, + &matches_inserts, + cpe_match_string_item)) { cJSON_Delete (cpe_match_string_item); inserts_free (&inserts); inserts_free (&matches_inserts); sql_commit (); - g_warning ("Update of cpe match strings failed"); + g_warning ("Update of CPE match strings failed"); gvm_json_pull_event_cleanup (&event); gvm_json_pull_parser_cleanup (&parser); fclose (cpe_match_strings_file); @@ -4311,7 +4355,8 @@ update_scap_cpe_match_strings () } else { - g_warning ("%s: CVE affected products file is not a JSON object.", __func__); + g_warning ("%s: CVE match strings file is not a JSON object.", + __func__); gvm_json_pull_event_cleanup (&event); gvm_json_pull_parser_cleanup (&parser); fclose (cpe_match_strings_file); diff --git a/src/schema_formats/XML/GMP.xml.in b/src/schema_formats/XML/GMP.xml.in index 216f603d5..17b1381f5 100644 --- a/src/schema_formats/XML/GMP.xml.in +++ b/src/schema_formats/XML/GMP.xml.in @@ -13262,6 +13262,7 @@ END:VCALENDAR A configuration node for the CVE operator + negate match_criteria node @@ -13272,6 +13273,13 @@ END:VCALENDAR text + + negate + A true or false value, whether the operator is negated + + text + + match_criteria The match criteria for the node From 6ff3d18c121a928b406a02b43ca3159d8641bb85 Mon Sep 17 00:00:00 2001 From: Ahmed Abdelsalam Date: Tue, 5 Nov 2024 11:44:09 +0100 Subject: [PATCH 19/26] Code refactoring - Rename scap.cpe_match_range to scap.cpe_match_strings - Rename cpe in cpe_match_range to criteria - Add cpe_name to scap.cpe_matches - Add status to the configuration nodes in the response of get_info command --- src/gmp.c | 44 ++++++++------- src/manage.c | 14 ++--- src/manage.h | 23 ++++---- src/manage_pg.c | 9 +-- src/manage_sql.c | 93 ++++++++++++++++++------------- src/manage_sql_secinfo.c | 29 +++++++--- src/schema_formats/XML/GMP.xml.in | 12 ++-- 7 files changed, 128 insertions(+), 96 deletions(-) diff --git a/src/gmp.c b/src/gmp.c index 833434055..462a7ed07 100644 --- a/src/gmp.c +++ b/src/gmp.c @@ -13282,29 +13282,31 @@ print_cpe_match_nodes_xml (resource_t node, GString *buffer) xml_string_append (buffer, "%s", operator?: ""); xml_string_append (buffer, "%s", negate? "1" : "0"); - init_cpe_match_range_iterator (&cpe_match_ranges, node); + init_cpe_match_string_iterator (&cpe_match_ranges, node); while (next (&cpe_match_ranges)) { - const gchar *vsi, *vse, *vei, *vee, *match_criteria_id, *match_string; + const gchar *vsi, *vse, *vei, *vee, *match_criteria_id, *criteria, *status; - xml_string_append (buffer, ""); + xml_string_append (buffer, ""); match_criteria_id - = cpe_match_range_iterator_match_criteria_id (&cpe_match_ranges); - match_string = cpe_match_range_iterator_cpe (&cpe_match_ranges); + = cpe_match_string_iterator_match_criteria_id (&cpe_match_ranges); + criteria = cpe_match_string_iterator_criteria (&cpe_match_ranges); + status = cpe_match_string_iterator_status (&cpe_match_ranges); xml_string_append (buffer, - "%s", - match_string?: ""); - xml_string_append (buffer, - "%s", - cpe_match_range_iterator_vulnerable (&cpe_match_ranges) != 0 + "%s" + "%s" + "%s", + criteria?: "", + cpe_match_string_iterator_vulnerable (&cpe_match_ranges) != 0 ? "1" - : "0"); + : "0", + status?: ""); - vsi = cpe_match_range_iterator_version_start_incl (&cpe_match_ranges); - vse = cpe_match_range_iterator_version_start_excl (&cpe_match_ranges); - vei = cpe_match_range_iterator_version_end_incl (&cpe_match_ranges); - vee = cpe_match_range_iterator_version_end_excl (&cpe_match_ranges); + vsi = cpe_match_string_iterator_version_start_incl (&cpe_match_ranges); + vse = cpe_match_string_iterator_version_start_excl (&cpe_match_ranges); + vei = cpe_match_string_iterator_version_end_incl (&cpe_match_ranges); + vee = cpe_match_string_iterator_version_end_excl (&cpe_match_ranges); xml_string_append (buffer, "%s", @@ -13320,7 +13322,7 @@ print_cpe_match_nodes_xml (resource_t node, GString *buffer) vee ?: ""); iterator_t cpe_matches; - init_cpe_match_range_matches_iterator (&cpe_matches, match_criteria_id); + init_cpe_match_string_matches_iterator (&cpe_matches, match_criteria_id); xml_string_append (buffer, ""); while (next (&cpe_matches)) @@ -13328,16 +13330,16 @@ print_cpe_match_nodes_xml (resource_t node, GString *buffer) iterator_t cpes; init_iterator (&cpes, - "SELECT name, deprecated FROM scap.cpes" + "SELECT deprecated FROM scap.cpes" " WHERE cpe_name_id = '%s';", cpe_matches_cpe_name_id(&cpe_matches)); - const char* cpe = NULL; + const char* cpe = cpe_matches_cpe_name (&cpe_matches); + int deprecated = 0; while (next (&cpes)) { - cpe = iterator_string (&cpes, 0); - deprecated = iterator_int (&cpes, 1); + deprecated = iterator_int (&cpes, 0); } cleanup_iterator (&cpes); @@ -13361,7 +13363,7 @@ print_cpe_match_nodes_xml (resource_t node, GString *buffer) xml_string_append (buffer, ""); } xml_string_append (buffer, ""); - xml_string_append (buffer, ""); + xml_string_append (buffer, ""); cleanup_iterator (&cpe_matches); } cleanup_iterator (&cpe_match_ranges); diff --git a/src/manage.c b/src/manage.c index fca82209f..f61426986 100644 --- a/src/manage.c +++ b/src/manage.c @@ -3180,7 +3180,7 @@ check_cpe_match_rule (long long int node, gboolean *match, gboolean *vulnerable, return; } - init_cpe_match_range_iterator (&cpe_match_ranges, node); + init_cpe_match_string_iterator (&cpe_match_ranges, node); while (next (&cpe_match_ranges)) { iterator_t cpe_host_details_products; @@ -3188,11 +3188,11 @@ check_cpe_match_rule (long long int node, gboolean *match, gboolean *vulnerable, gchar *range_uri_product; gchar *vsi, *vse, *vei, *vee; range_fs_cpe = vsi = vse = vei = vee = NULL; - range_fs_cpe = g_strdup (cpe_match_range_iterator_cpe (&cpe_match_ranges)); - vsi = g_strdup (cpe_match_range_iterator_version_start_incl (&cpe_match_ranges)); - vse = g_strdup (cpe_match_range_iterator_version_start_excl (&cpe_match_ranges)); - vei = g_strdup (cpe_match_range_iterator_version_end_incl (&cpe_match_ranges)); - vee = g_strdup (cpe_match_range_iterator_version_end_excl (&cpe_match_ranges)); + range_fs_cpe = g_strdup (cpe_match_string_iterator_criteria (&cpe_match_ranges)); + vsi = g_strdup (cpe_match_string_iterator_version_start_incl (&cpe_match_ranges)); + vse = g_strdup (cpe_match_string_iterator_version_start_excl (&cpe_match_ranges)); + vei = g_strdup (cpe_match_string_iterator_version_end_incl (&cpe_match_ranges)); + vee = g_strdup (cpe_match_string_iterator_version_end_excl (&cpe_match_ranges)); range_uri_product = fs_cpe_to_uri_product (range_fs_cpe); init_host_details_cpe_product_iterator (&cpe_host_details_products, range_uri_product, report_host); while (next (&cpe_host_details_products)) @@ -3216,7 +3216,7 @@ check_cpe_match_rule (long long int node, gboolean *match, gboolean *vulnerable, cpe_struct_free (&source); cpe_struct_free (&target); } - if (*match && cpe_match_range_iterator_vulnerable (&cpe_match_ranges) == 1) + if (*match && cpe_match_string_iterator_vulnerable (&cpe_match_ranges) == 1) { cpe_struct_t source, target; cpe_struct_init (&source); diff --git a/src/manage.h b/src/manage.h index 93cbb01e2..2930789b9 100644 --- a/src/manage.h +++ b/src/manage.h @@ -1724,38 +1724,41 @@ long long int cpe_match_node_childs_iterator_id (iterator_t*); void -init_cpe_match_range_iterator (iterator_t*, long long int); +init_cpe_match_string_iterator (iterator_t*, long long int); const char* -cpe_match_range_iterator_cpe (iterator_t*); +cpe_match_string_iterator_criteria (iterator_t*); const char* -cpe_match_range_iterator_match_criteria_id (iterator_t*); +cpe_match_string_iterator_match_criteria_id (iterator_t*); const char* -cpe_match_range_iterator_status (iterator_t*); +cpe_match_string_iterator_status (iterator_t*); const char* -cpe_match_range_iterator_version_start_incl (iterator_t*); +cpe_match_string_iterator_version_start_incl (iterator_t*); const char* -cpe_match_range_iterator_version_start_excl (iterator_t*); +cpe_match_string_iterator_version_start_excl (iterator_t*); const char* -cpe_match_range_iterator_version_end_incl (iterator_t*); +cpe_match_string_iterator_version_end_incl (iterator_t*); const char* -cpe_match_range_iterator_version_end_excl (iterator_t*); +cpe_match_string_iterator_version_end_excl (iterator_t*); int -cpe_match_range_iterator_vulnerable (iterator_t*); +cpe_match_string_iterator_vulnerable (iterator_t*); void -init_cpe_match_range_matches_iterator (iterator_t*, const char *); +init_cpe_match_string_matches_iterator (iterator_t*, const char *); const char* cpe_matches_cpe_name_id (iterator_t*); +const char* +cpe_matches_cpe_name (iterator_t*); + void init_host_details_cpe_product_iterator (iterator_t*, const char *, report_host_t); diff --git a/src/manage_pg.c b/src/manage_pg.c index da79adb05..43e031b99 100644 --- a/src/manage_pg.c +++ b/src/manage_pg.c @@ -3557,10 +3557,10 @@ manage_db_init (const gchar *name) " vulnerable integer DEFAULT 0," " match_criteria_id text);"); - sql ("CREATE TABLE scap2.cpe_match_range" + sql ("CREATE TABLE scap2.cpe_match_strings" " (id SERIAL PRIMARY KEY," " match_criteria_id text," - " cpe text DEFAULT NULL," + " criteria text DEFAULT NULL," " version_start_incl text DEFAULT NULL," " version_start_excl text DEFAULT NULL," " version_end_incl text DEFAULT NULL," @@ -3570,7 +3570,8 @@ manage_db_init (const gchar *name) sql ("CREATE TABLE scap2.cpe_matches" " (id SERIAL PRIMARY KEY," " match_criteria_id text," - " cpe_name_id text);"); + " cpe_name_id text," + " cpe_name text);"); sql ("CREATE TABLE scap2.cpe_details" " (id SERIAL PRIMARY KEY," @@ -3646,7 +3647,7 @@ manage_db_add_constraints (const gchar *name) " ALTER url SET NOT NULL," " ADD UNIQUE (cve_id, url);"); - sql ("ALTER TABLE scap2.cpe_match_range" + sql ("ALTER TABLE scap2.cpe_match_strings" " ADD UNIQUE (match_criteria_id);"); sql ("ALTER TABLE scap2.cpe_matches" diff --git a/src/manage_sql.c b/src/manage_sql.c index ef9cb24e4..3f47b535a 100644 --- a/src/manage_sql.c +++ b/src/manage_sql.c @@ -20503,23 +20503,23 @@ DEF_ACCESS (host_details_cpe_product_iterator_value, 0); * @brief Initialize an iterator of root_ids of CPE match nodes. * * @param[in] iterator Iterator. - * @param[in] cpe The cpe contained in the match nodes. + * @param[in] criteria The criteria for the match nodes. */ void -init_cpe_match_nodes_iterator (iterator_t* iterator, const char *cpe) +init_cpe_match_nodes_iterator (iterator_t* iterator, const char *criteria) { - gchar *quoted_cpe; - quoted_cpe = sql_quote (cpe); + gchar *quoted_criteria; + quoted_criteria = sql_quote (criteria); init_iterator (iterator, "SELECT DISTINCT n.root_id" " FROM scap.cpe_match_nodes n" " JOIN scap.cpe_nodes_match_criteria c" " ON n.id = c.node_id" - " JOIN scap.cpe_match_range r" + " JOIN scap.cpe_match_strings r" " ON c.match_criteria = r.match_criteria_id" - " WHERE cpe like '%s%%';", - quoted_cpe); - g_free (quoted_cpe); + " WHERE criteria like '%s%%';", + quoted_criteria); + g_free (quoted_criteria); } /** @@ -20632,19 +20632,19 @@ cpe_match_node_childs_iterator_id (iterator_t* iterator) } /** - * @brief Initialize an iterator of match ranges of an CPE match node. + * @brief Initialize an iterator of match strings of an CPE match node. * * @param[in] iterator Iterator. - * @param[in] node The match node with match ranges. + * @param[in] node The match node with match strings. */ void -init_cpe_match_range_iterator (iterator_t* iterator, long long int node) +init_cpe_match_string_iterator (iterator_t* iterator, long long int node) { init_iterator (iterator, - "SELECT n.vulnerable, r.cpe, r.match_criteria_id, r.status," + "SELECT n.vulnerable, r.criteria, r.match_criteria_id, r.status," " r.version_start_incl, r.version_start_excl," " r.version_end_incl, r.version_end_excl" - " FROM scap.cpe_match_range r" + " FROM scap.cpe_match_strings r" " JOIN scap.cpe_nodes_match_criteria n" " ON r.match_criteria_id = n.match_criteria_id" " WHERE n.node_id = %llu;", @@ -20652,106 +20652,106 @@ init_cpe_match_range_iterator (iterator_t* iterator, long long int node) } /** - * @brief Return if the CPE of the actual match node is vulnerable. + * @brief Return if the match criteria is vulnerable. * * @param[in] iterator Iterator. * - * @return 1 if the match node is vulnerable, 0 otherwise. + * @return 1 if the match criteria is vulnerable, 0 otherwise. */ int -cpe_match_range_iterator_vulnerable (iterator_t* iterator) +cpe_match_string_iterator_vulnerable (iterator_t* iterator) { return iterator_int64 (iterator, 0); } /** - * @brief Return the CPE of the actual match node. + * @brief Return the criteria of the CPE match string. * * @param[in] iterator Iterator. * - * @return The CPE of the actual match node. + * @return The criteria of the match string. */ -DEF_ACCESS (cpe_match_range_iterator_cpe, 1); +DEF_ACCESS (cpe_match_string_iterator_criteria, 1); /** - * @brief Return the match criteria id of the CPE match range. + * @brief Return the match criteria id of the CPE match string. * * @param[in] iterator Iterator. * * @return The match criteria id, if any. NULL otherwise. */ -DEF_ACCESS (cpe_match_range_iterator_match_criteria_id, 2); +DEF_ACCESS (cpe_match_string_iterator_match_criteria_id, 2); /** - * @brief Return the status of the CPE match range. + * @brief Return the status of the CPE match criteria. * * @param[in] iterator Iterator. * - * @return The status of the CPE match range, if any. + * @return The status of the CPE match criteria, if any. * NULL otherwise. */ -DEF_ACCESS (cpe_match_range_iterator_status, 3); +DEF_ACCESS (cpe_match_string_iterator_status, 3); /** - * @brief Return the start included version of the match range. + * @brief Return the start included version of the match criteria. * * @param[in] iterator Iterator. * - * @return The start included version of the match range, if any. + * @return The start included version of the match criteria, if any. * NULL otherwise. */ -DEF_ACCESS (cpe_match_range_iterator_version_start_incl, 4); +DEF_ACCESS (cpe_match_string_iterator_version_start_incl, 4); /** - * @brief Return the start excluded version of the match range. + * @brief Return the start excluded version of the match criteria. * * @param[in] iterator Iterator. * - * @return The start excluded version of the match range, if any. + * @return The start excluded version of the match criteria, if any. * NULL otherwise. */ -DEF_ACCESS (cpe_match_range_iterator_version_start_excl, 5); +DEF_ACCESS (cpe_match_string_iterator_version_start_excl, 5); /** - * @brief Return the end included version of the match range. + * @brief Return the end included version of the match criteria. * * @param[in] iterator Iterator. * - * @return The end included version of the match range, if any. + * @return The end included version of the match criteria, if any. * NULL otherwise. */ -DEF_ACCESS (cpe_match_range_iterator_version_end_incl, 6); +DEF_ACCESS (cpe_match_string_iterator_version_end_incl, 6); /** - * @brief Return the end excluded version of the match range. + * @brief Return the end excluded version of the match criteria. * * @param[in] iterator Iterator. * - * @return The end excluded version of the match range, if any. + * @return The end excluded version of the match criteria, if any. * NULL otherwise. */ -DEF_ACCESS (cpe_match_range_iterator_version_end_excl, 7); +DEF_ACCESS (cpe_match_string_iterator_version_end_excl, 7); /** - * @brief Initialize an iterator of CPE matches for a match range + * @brief Initialize an iterator of CPE matches for a match string * given a match criteria id. * * @param[in] iterator Iterator. * @param[in] match_criteria_id The match criteria id to get the matches for. */ void -init_cpe_match_range_matches_iterator (iterator_t* iterator, +init_cpe_match_string_matches_iterator (iterator_t* iterator, const char *match_criteria_id) { init_iterator (iterator, - "SELECT cpe_name_id" + "SELECT cpe_name_id, cpe_name" " FROM scap.cpe_matches" " WHERE match_criteria_id = '%s'", match_criteria_id); } /** - * @brief Get the CPE name id from a CPE match range matches iterator. + * @brief Get the CPE name id from a CPE match string matches iterator. * * @param[in] iterator Iterator. * @@ -20763,6 +20763,19 @@ cpe_matches_cpe_name_id (iterator_t* iterator) return iterator_string (iterator, 0); } +/** + * @brief Get the CPE name from a CPE match string matches iterator. + * + * @param[in] iterator Iterator. + * + * @return The CPE name id. + */ +const char * +cpe_matches_cpe_name (iterator_t* iterator) +{ + return iterator_string (iterator, 1); +} + /** * @brief Initialise a report host prognosis iterator. * diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index b7b421ce6..4e6d3d6fc 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -4060,7 +4060,7 @@ handle_json_cpe_match_string (inserts_t *inserts, inserts_t *matches_inserts, char *criteria, *match_criteria_id, *status, *ver_se; gchar *quoted_version_start_incl, *quoted_version_start_excl; gchar *quoted_version_end_incl, *quoted_version_end_excl; - gchar *quoted_criteria, *quoted_match_criteria_id, *quoted_cpe_name_id; + gchar *quoted_criteria, *quoted_match_criteria_id; int first; assert (inserts); @@ -4153,7 +4153,8 @@ handle_json_cpe_match_string (inserts_t *inserts, inserts_t *matches_inserts, cJSON *match_item; cJSON_ArrayForEach (match_item, matches_array) { - char *cpe_name_id; + char *cpe_name_id, *cpe_name; + gchar *quoted_cpe_name_id, *quoted_cpe_name; cpe_name_id = json_object_item_string (match_item, "cpeNameId"); if (cpe_name_id == NULL) @@ -4164,19 +4165,31 @@ handle_json_cpe_match_string (inserts_t *inserts, inserts_t *matches_inserts, return -1; } + cpe_name = json_object_item_string (match_item, "cpeName"); + if (cpe_name == NULL) + { + g_warning ("%s: 'cpe_name' field missing or not a string", + __func__); + g_free (quoted_match_criteria_id); + return -1; + } + quoted_cpe_name_id = sql_quote (cpe_name_id); + quoted_cpe_name = fs_to_uri_convert_and_quote_cpe_name (cpe_name); first = inserts_check_size (matches_inserts); g_string_append_printf (matches_inserts->statement, - "%s ('%s', '%s')", + "%s ('%s', '%s', '%s')", first ? "" : ",", quoted_match_criteria_id, - quoted_cpe_name_id); + quoted_cpe_name_id, + quoted_cpe_name); matches_inserts->current_chunk_size++; g_free (quoted_cpe_name_id); + g_free (quoted_cpe_name); } } @@ -4287,13 +4300,13 @@ update_scap_cpe_match_strings () inserts_init (&inserts, CPE_MAX_CHUNK_SIZE, setting_secinfo_sql_buffer_threshold_bytes (), - "INSERT INTO scap2.cpe_match_range" - " (match_criteria_id, cpe, version_start_incl," + "INSERT INTO scap2.cpe_match_strings" + " (match_criteria_id, criteria, version_start_incl," " version_start_excl, version_end_incl, version_end_excl," " status)" " VALUES ", " ON CONFLICT (match_criteria_id) DO UPDATE" - " SET cpe = EXCLUDED.cpe," + " SET criteria = EXCLUDED.criteria," " version_start_incl = EXCLUDED.version_start_incl," " version_start_excl = EXCLUDED.version_start_excl," " version_end_incl = EXCLUDED.version_end_incl," @@ -4303,7 +4316,7 @@ update_scap_cpe_match_strings () inserts_init (&matches_inserts, 10, setting_secinfo_sql_buffer_threshold_bytes (), "INSERT INTO scap2.cpe_matches" - " (match_criteria_id, cpe_name_id)" + " (match_criteria_id, cpe_name_id, cpe_name)" " VALUES ", ""); diff --git a/src/schema_formats/XML/GMP.xml.in b/src/schema_formats/XML/GMP.xml.in index 17b1381f5..5bb6e166d 100644 --- a/src/schema_formats/XML/GMP.xml.in +++ b/src/schema_formats/XML/GMP.xml.in @@ -13263,7 +13263,7 @@ END:VCALENDAR operator negate - match_criteria + match_string node @@ -13281,10 +13281,10 @@ END:VCALENDAR - match_criteria - The match criteria for the node + match_string + The match string for the node - match_string + criteria vulnerable version_start_including version_start_excluding @@ -13293,8 +13293,8 @@ END:VCALENDAR matched_cpes - match_string - A CPE Match string + criteria + A CPE match criteria text From 9b43edec2328de3b6b9f8ac564bc64d6c1f8b343 Mon Sep 17 00:00:00 2001 From: Johannes Helmold Date: Mon, 28 Oct 2024 09:49:26 +0100 Subject: [PATCH 20/26] Remove: Removed feature toggle for compliance report views. Removed the feature toggle for the dedicated compliance report views. --- CMakeLists.txt | 13 - src/gmp.c | 49 +-- src/gvmd.c | 2 - src/manage_sql.c | 910 +++++++++++++++++++++++------------------------ 4 files changed, 456 insertions(+), 518 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index eb6b215d2..d5c958a34 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -247,25 +247,12 @@ if (NOT CVSS3_RATINGS) endif (NOT CVSS3_RATINGS) add_definitions (-DCVSS3_RATINGS=${CVSS3_RATINGS}) -if (NOT COMPLIANCE_REPORTS) - set (COMPLIANCE_REPORTS 1) -endif (NOT COMPLIANCE_REPORTS) -add_definitions (-DCOMPLIANCE_REPORTS=${COMPLIANCE_REPORTS}) - message ("-- Install prefix: ${CMAKE_INSTALL_PREFIX}") ## Version set (GVMD_VERSION "${PROJECT_VERSION_STRING}") -if (COMPLIANCE_REPORTS EQUAL 1) - set(IF_COMPLIANCE_REPORTS "") - set(ENDIF_COMPLIANCE_REPORTS "") -elseif (COMPLIANCE_REPORTS EQUAL 0) - set(IF_COMPLIANCE_REPORTS "") -endif() - # Configure Doxyfile with version number configure_file (doc/Doxyfile.in doc/Doxyfile) configure_file (doc/Doxyfile_full.in doc/Doxyfile_full) diff --git a/src/gmp.c b/src/gmp.c index b90c1ff54..ae84d3e58 100644 --- a/src/gmp.c +++ b/src/gmp.c @@ -12956,11 +12956,6 @@ handle_get_features (gmp_parser_t *gmp_parser, GError **error) " status=\"" STATUS_OK "\"" " status_text=\"" STATUS_OK_TEXT "\">"); - SENDF_TO_CLIENT_OR_FAIL ("" - "COMPLIANCE_REPORTS" - "", - COMPLIANCE_REPORTS ? 1 : 0); - SENDF_TO_CLIENT_OR_FAIL ("" "CVSS3_RATINGS" "", @@ -14925,31 +14920,23 @@ handle_get_reports (gmp_parser_t *gmp_parser, GError **error) overrides = filter_term_apply_overrides (filter ? filter : get->filter); min_qod = filter_term_min_qod (filter ? filter : get->filter); levels = filter_term_value (filter ? filter : get->filter, "levels"); - #if COMPLIANCE_REPORTS == 1 - gchar *compliance_levels; - compliance_levels = filter_term_value (filter - ? filter - : get->filter, - "compliance_levels"); - - /* Setup result filter from overrides. */ - get_reports_data->get.filter - = g_strdup_printf - ("apply_overrides=%i min_qod=%i levels=%s compliance_levels=%s", - overrides, - min_qod, - levels ? levels : "hmlgdf", - compliance_levels ? compliance_levels : "yniu"); - g_free (compliance_levels); - #else - /* Setup result filter from overrides. */ - get_reports_data->get.filter - = g_strdup_printf - ("apply_overrides=%i min_qod=%i levels=%s", - overrides, - min_qod, - levels ? levels : "hmlgdf"); - #endif + + gchar *compliance_levels; + compliance_levels = filter_term_value (filter + ? filter + : get->filter, + "compliance_levels"); + + /* Setup result filter from overrides. */ + get_reports_data->get.filter + = g_strdup_printf + ("apply_overrides=%i min_qod=%i levels=%s compliance_levels=%s", + overrides, + min_qod, + levels ? levels : "hmlgdf", + compliance_levels ? compliance_levels : "yniu"); + g_free (compliance_levels); + g_free (filter); g_free (levels); } @@ -15997,7 +15984,6 @@ select_resource_iterator (get_resource_names_data_t *resource_names_data, else if (g_strcmp0 ("report", resource_names_data->type) == 0) { *iterator = (int (*) (iterator_t*, get_data_t *))init_report_iterator; -#if COMPLIANCE_REPORTS == 1 get_data_set_extra (&resource_names_data->get, "usage_type", g_strdup ("scan")); @@ -16008,7 +15994,6 @@ select_resource_iterator (get_resource_names_data_t *resource_names_data, get_data_set_extra (&resource_names_data->get, "usage_type", g_strdup ("audit")); -#endif } else if (g_strcmp0 ("report_config", resource_names_data->type) == 0) { diff --git a/src/gvmd.c b/src/gvmd.c index 0fbe8051b..722ea2022 100644 --- a/src/gvmd.c +++ b/src/gvmd.c @@ -2325,9 +2325,7 @@ gvmd (int argc, char** argv, char *env[]) #if CVSS3_RATINGS == 1 printf ("CVSS3 severity ratings enabled\n"); #endif -#if COMPLIANCE_REPORTS == 1 printf ("Compliance reports enabled\n"); -#endif printf ("Copyright (C) 2009-2021 Greenbone AG\n"); printf ("License: AGPL-3.0-or-later\n"); printf diff --git a/src/manage_sql.c b/src/manage_sql.c index fc1bf3905..9e33f3277 100644 --- a/src/manage_sql.c +++ b/src/manage_sql.c @@ -2221,7 +2221,7 @@ manage_report_filter_controls (const gchar *filter, int *first, int *max, else *apply_overrides = val; } - + if (compliance_levels) { if (filter_control_str ((keyword_t **) split->pdata, @@ -3980,14 +3980,9 @@ valid_type (const char* type) int valid_subtype (const char* type) { - #if COMPLIANCE_REPORTS == 1 return (strcasecmp (type, "audit_report") == 0) || (strcasecmp (type, "audit") == 0) || (strcasecmp (type, "policy") == 0); - #else - return (strcasecmp (type, "audit") == 0) - || (strcasecmp (type, "policy") == 0); - #endif } /** @@ -14406,7 +14401,7 @@ manage_test_alert (const char *alert_id, gchar **script_message) NULL); if (result) report_add_result (report, result); - + result = make_result ( task, "127.0.0.1", "localhost", "general/tcp", @@ -18335,7 +18330,7 @@ task_usage_type (task_t task, char ** usage_type) task); if (usage_type == NULL) return -1; - + return 0; } @@ -22230,7 +22225,6 @@ report_iterator_opts_table (int override, int min_qod) min_qod); } -#if COMPLIANCE_REPORTS == 1 /** * @brief Return SQL WHERE for restricting a SELECT to compliance statuses. * @@ -22288,7 +22282,6 @@ where_compliance_status (const char *compliance) return g_string_free (compliance_sql, FALSE);; } -#endif /** * @brief Generate an extra WHERE clause for selecting reports @@ -22323,32 +22316,30 @@ reports_extra_where (int trash, const gchar *filter, const char *usage_type) g_string_append_printf(extra_where, "%s", trash_clause); g_free (trash_clause); - #if COMPLIANCE_REPORTS == 1 - gchar *usage_type_clause, *compliance_clause = NULL; - gchar *compliance_filter = NULL; - if (usage_type && strcmp (usage_type, "")) - { - gchar *quoted_usage_type; - quoted_usage_type = sql_quote (usage_type); - usage_type_clause = g_strdup_printf (" AND task in (SELECT id from tasks" - " WHERE usage_type='%s')", - quoted_usage_type); + gchar *usage_type_clause, *compliance_clause = NULL; + gchar *compliance_filter = NULL; + if (usage_type && strcmp (usage_type, "")) + { + gchar *quoted_usage_type; + quoted_usage_type = sql_quote (usage_type); + usage_type_clause = g_strdup_printf (" AND task in (SELECT id from tasks" + " WHERE usage_type='%s')", + quoted_usage_type); - g_free (quoted_usage_type); - } - else - usage_type_clause = NULL; + g_free (quoted_usage_type); + } + else + usage_type_clause = NULL; - if (filter) - compliance_filter = filter_term_value(filter, "report_compliance_levels"); + if (filter) + compliance_filter = filter_term_value(filter, "report_compliance_levels"); - compliance_clause = where_compliance_status (compliance_filter ?: "yniu"); + compliance_clause = where_compliance_status (compliance_filter ?: "yniu"); - g_string_append_printf (extra_where, "%s%s", usage_type_clause ?: "", compliance_clause ?: ""); - g_free (compliance_filter); - g_free (compliance_clause); - g_free (usage_type_clause); - #endif + g_string_append_printf (extra_where, "%s%s", usage_type_clause ?: "", compliance_clause ?: ""); + g_free (compliance_filter); + g_free (compliance_clause); + g_free (usage_type_clause); return g_string_free (extra_where, FALSE); } @@ -22370,7 +22361,7 @@ report_count (const get_data_t *get) int ret; extra_tables = report_iterator_opts_table (0, MIN_QOD_DEFAULT); - + const gchar *usage_type = get_data_get_extra (get, "usage_type"); extra_where = reports_extra_where(get->trash, get->filter, usage_type); @@ -23797,7 +23788,7 @@ result_count (const get_data_t *get, report_t report, const char* host) dynamic_severity, "results", "nvts"); - + extra_tables = g_strdup_printf (" LEFT OUTER JOIN result_vt_epss" " ON results.nvt = result_vt_epss.vt_id" " LEFT OUTER JOIN nvts" @@ -26167,7 +26158,6 @@ report_counts_id_full (report_t report, int* holes, int* infos, return 0; } -#if COMPLIANCE_REPORTS == 1 /** * @brief Get the compliance state from compliance counts. * @@ -26312,8 +26302,6 @@ report_compliance_counts (report_t report, return 0; } -#endif - /** * @brief Get only the filtered message counts for a report. @@ -28052,7 +28040,7 @@ print_report_host_tls_certificates_xml (report_host_t report_host, " AND (source_description = 'SSL/TLS Certificate'" " OR source_description = 'SSL Certificate')", report_host); - + while (next (&tls_certs)) { const char *certificate_prefixed, *certificate_b64; @@ -28636,54 +28624,53 @@ print_report_host_xml (FILE *stream, host_iterator_asset_uuid (hosts)); else if (lean == 0) PRINT (stream, - ""); + ""); - #if COMPLIANCE_REPORTS == 1 - if (strcmp (usage_type, "audit") == 0) - { - int yes_count, no_count, incomplete_count, undefined_count; - - yes_count - = GPOINTER_TO_INT - (g_hash_table_lookup (f_host_compliant, current_host)); - no_count - = GPOINTER_TO_INT - (g_hash_table_lookup (f_host_notcompliant, current_host)); - incomplete_count - = GPOINTER_TO_INT - (g_hash_table_lookup (f_host_incomplete, current_host)); - undefined_count - = GPOINTER_TO_INT - (g_hash_table_lookup (f_host_undefined, current_host)); - - PRINT (stream, - "%s" - "%s" - "%d" - "" - "%d" - "%d" - "%d" - "%d" - "%d" - "" - "%s", - host_iterator_start_time (hosts), - host_iterator_end_time (hosts) - ? host_iterator_end_time (hosts) - : "", - ports_count, - (yes_count + no_count + incomplete_count + undefined_count), - yes_count, - no_count, - incomplete_count, - undefined_count, - report_compliance_from_counts (&yes_count, - &no_count, - &incomplete_count, - &undefined_count)); - } else - #endif + if (strcmp (usage_type, "audit") == 0) + { + int yes_count, no_count, incomplete_count, undefined_count; + + yes_count + = GPOINTER_TO_INT + (g_hash_table_lookup (f_host_compliant, current_host)); + no_count + = GPOINTER_TO_INT + (g_hash_table_lookup (f_host_notcompliant, current_host)); + incomplete_count + = GPOINTER_TO_INT + (g_hash_table_lookup (f_host_incomplete, current_host)); + undefined_count + = GPOINTER_TO_INT + (g_hash_table_lookup (f_host_undefined, current_host)); + + PRINT (stream, + "%s" + "%s" + "%d" + "" + "%d" + "%d" + "%d" + "%d" + "%d" + "" + "%s", + host_iterator_start_time (hosts), + host_iterator_end_time (hosts) + ? host_iterator_end_time (hosts) + : "", + ports_count, + (yes_count + no_count + incomplete_count + undefined_count), + yes_count, + no_count, + incomplete_count, + undefined_count, + report_compliance_from_counts (&yes_count, + &no_count, + &incomplete_count, + &undefined_count)); + } + else { int holes_count, warnings_count, infos_count; int logs_count, false_positives_count; @@ -28703,7 +28690,7 @@ print_report_host_xml (FILE *stream, false_positives_count = GPOINTER_TO_INT (g_hash_table_lookup ( f_host_false_positives, - current_host)); + current_host)); PRINT (stream, "%s" @@ -29855,30 +29842,29 @@ print_v2_report_delta_xml (FILE *out, iterator_t *results, if (strchr (delta_states, state[0]) == NULL) continue; - #if COMPLIANCE_REPORTS == 1 - if (strcmp (usage_type, "audit") == 0) - { - const char* compliance; - compliance = result_iterator_compliance (results); - (*f_compliance_count)++; - if (strcasecmp (compliance, "yes") == 0) - { - (*f_compliance_yes)++; - } - else if (strcasecmp (compliance, "no") == 0) - { - (*f_compliance_no)++; - } - else if (strcasecmp (compliance, "incomplete") == 0) - { - (*f_compliance_incomplete)++; - } - else if (strcasecmp (compliance, "undefined") == 0) - { - (*f_compliance_undefined)++; - } - } else - #endif + if (strcmp (usage_type, "audit") == 0) + { + const char* compliance; + compliance = result_iterator_compliance (results); + (*f_compliance_count)++; + if (strcasecmp (compliance, "yes") == 0) + { + (*f_compliance_yes)++; + } + else if (strcasecmp (compliance, "no") == 0) + { + (*f_compliance_no)++; + } + else if (strcasecmp (compliance, "incomplete") == 0) + { + (*f_compliance_incomplete)++; + } + else if (strcasecmp (compliance, "undefined") == 0) + { + (*f_compliance_undefined)++; + } + } + else { const char *level; /* Increase the result count. */ @@ -29943,7 +29929,7 @@ print_v2_report_delta_xml (FILE *out, iterator_t *results, } g_string_free (buffer, TRUE); g_free (usage_type); - + if (fprintf (out, "") < 0) { g_tree_destroy (ports); @@ -30260,9 +30246,7 @@ print_report_xml_start (report_t report, report_t delta, task_t task, if (report) { /* Get total counts of full results. */ - #if COMPLIANCE_REPORTS == 1 - if (strcmp (tsk_usage_type, "audit")) - #endif + if (strcmp (tsk_usage_type, "audit")) { if (delta == 0) { @@ -30400,22 +30384,20 @@ print_report_xml_start (report_t report, report_t delta, task_t task, filters_extra_buffer = g_string_new (""); - #if COMPLIANCE_REPORTS == 1 - if (strcmp (tsk_usage_type, "audit") == 0) - { - compliance_levels = compliance_levels ? compliance_levels : g_strdup ("yniu"); - - if (strchr (compliance_levels, 'y')) - g_string_append (filters_extra_buffer, "Yes"); - if (strchr (compliance_levels, 'n')) - g_string_append (filters_extra_buffer, "No"); - if (strchr (compliance_levels, 'i')) - g_string_append (filters_extra_buffer, "Incomplete"); - if (strchr (compliance_levels, 'u')) - g_string_append (filters_extra_buffer, "Undefined"); - } - else - #endif + if (strcmp (tsk_usage_type, "audit") == 0) + { + compliance_levels = compliance_levels ? compliance_levels : g_strdup ("yniu"); + + if (strchr (compliance_levels, 'y')) + g_string_append (filters_extra_buffer, "Yes"); + if (strchr (compliance_levels, 'n')) + g_string_append (filters_extra_buffer, "No"); + if (strchr (compliance_levels, 'i')) + g_string_append (filters_extra_buffer, "Incomplete"); + if (strchr (compliance_levels, 'u')) + g_string_append (filters_extra_buffer, "Undefined"); + } + else { if (strchr (levels, 'h')) g_string_append (filters_extra_buffer, "High"); @@ -30692,41 +30674,40 @@ print_report_xml_start (report_t report, report_t delta, task_t task, } /* Prepare result counts. */ - #if COMPLIANCE_REPORTS == 1 - int compliance_yes, compliance_no; - int compliance_incomplete, compliance_undefined; - int total_compliance_count = 0; + int compliance_yes, compliance_no; + int compliance_incomplete, compliance_undefined; + int total_compliance_count = 0; - if (strcmp (tsk_usage_type, "audit") == 0) - { - report_compliance_counts (report, get, &compliance_yes, &compliance_no, - &compliance_incomplete, &compliance_undefined); + if (strcmp (tsk_usage_type, "audit") == 0) + { + report_compliance_counts (report, get, &compliance_yes, &compliance_no, + &compliance_incomplete, &compliance_undefined); - total_compliance_count = compliance_yes - + compliance_no - + compliance_incomplete - + compliance_undefined; + total_compliance_count = compliance_yes + + compliance_no + + compliance_incomplete + + compliance_undefined; - f_compliance_yes = f_compliance_no = 0; - f_compliance_incomplete = f_compliance_undefined = 0; + f_compliance_yes = f_compliance_no = 0; + f_compliance_incomplete = f_compliance_undefined = 0; - if (count_filtered == 0) - { - report_compliance_f_counts (report, - get, - &f_compliance_yes, - &f_compliance_no, - &f_compliance_incomplete, - &f_compliance_undefined); + if (count_filtered == 0) + { + report_compliance_f_counts (report, + get, + &f_compliance_yes, + &f_compliance_no, + &f_compliance_incomplete, + &f_compliance_undefined); - f_compliance_count = f_compliance_yes - + f_compliance_no - + f_compliance_incomplete - + f_compliance_undefined; - } - } else - #endif - { + f_compliance_count = f_compliance_yes + + f_compliance_no + + f_compliance_incomplete + + f_compliance_undefined; + } + } + else + { if (count_filtered) { /* We're getting all the filtered results, so we can count them as we @@ -30740,12 +30721,12 @@ print_report_xml_start (report_t report, report_t delta, task_t task, f_false_positives = f_severity = 0; } else - report_counts_id_full (report, &holes, &infos, &logs, - &warnings, &false_positives, &severity, - get, NULL, - &f_holes, &f_infos, &f_logs, &f_warnings, - &f_false_positives, &f_severity); - } + report_counts_id_full (report, &holes, &infos, &logs, + &warnings, &false_positives, &severity, + get, NULL, + &f_holes, &f_infos, &f_logs, &f_warnings, + &f_false_positives, &f_severity); + } /* Results. */ @@ -30810,31 +30791,30 @@ print_report_xml_start (report_t report, report_t delta, task_t task, /* Quiet erroneous compiler warning. */ result_hosts = NULL; - #if COMPLIANCE_REPORTS == 1 - if (strcmp (tsk_usage_type, "audit") == 0) - { - f_host_compliant = g_hash_table_new_full (g_str_hash, g_str_equal, - g_free, NULL); - f_host_notcompliant = g_hash_table_new_full (g_str_hash, g_str_equal, - g_free, NULL); - f_host_incomplete = g_hash_table_new_full (g_str_hash, g_str_equal, + if (strcmp (tsk_usage_type, "audit") == 0) + { + f_host_compliant = g_hash_table_new_full (g_str_hash, g_str_equal, + g_free, NULL); + f_host_notcompliant = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); - f_host_undefined = g_hash_table_new_full (g_str_hash, g_str_equal, - g_free, NULL); - } else - #endif - { - f_host_holes = g_hash_table_new_full (g_str_hash, g_str_equal, - g_free, NULL); - f_host_warnings = g_hash_table_new_full (g_str_hash, g_str_equal, + f_host_incomplete = g_hash_table_new_full (g_str_hash, g_str_equal, + g_free, NULL); + f_host_undefined = g_hash_table_new_full (g_str_hash, g_str_equal, + g_free, NULL); + } + else + { + f_host_holes = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); - f_host_infos = g_hash_table_new_full (g_str_hash, g_str_equal, - g_free, NULL); - f_host_logs = g_hash_table_new_full (g_str_hash, g_str_equal, - g_free, NULL); - f_host_false_positives = g_hash_table_new_full (g_str_hash, g_str_equal, - g_free, NULL); - } + f_host_warnings = g_hash_table_new_full (g_str_hash, g_str_equal, + g_free, NULL); + f_host_infos = g_hash_table_new_full (g_str_hash, g_str_equal, + g_free, NULL); + f_host_logs = g_hash_table_new_full (g_str_hash, g_str_equal, + g_free, NULL); + f_host_false_positives = g_hash_table_new_full (g_str_hash, g_str_equal, + g_free, NULL); + } if (delta && get->details) { @@ -30918,109 +30898,107 @@ print_report_xml_start (report_t report, report_t delta, task_t task, array_add_new_string (result_hosts, result_iterator_host (&results)); - #if COMPLIANCE_REPORTS == 1 - if (strcmp (tsk_usage_type, "audit") == 0) - { - const char* compliance; - compliance = result_iterator_compliance (&results); - - if (strcasecmp (compliance, "yes") == 0) - { - f_host_result_counts = f_host_compliant; - if (count_filtered) - f_compliance_yes++; - } - else if (strcasecmp (compliance, "no") == 0) - { - f_host_result_counts = f_host_notcompliant; - if (count_filtered) - f_compliance_no++; - } - else if (strcasecmp (compliance, "incomplete") == 0) - { - f_host_result_counts = f_host_incomplete; - if (count_filtered) - f_compliance_incomplete++; - } - else if (strcasecmp (compliance, "undefined") == 0) - { - f_host_result_counts = f_host_undefined; - if (count_filtered) - f_compliance_undefined++; - } - else - { - f_host_result_counts = NULL; - } + if (strcmp (tsk_usage_type, "audit") == 0) + { + const char* compliance; + compliance = result_iterator_compliance (&results); - if (f_host_result_counts) - { - const char *result_host = result_iterator_host (&results); - int result_count - = GPOINTER_TO_INT - (g_hash_table_lookup (f_host_result_counts, - result_host)); - - g_hash_table_replace (f_host_result_counts, - g_strdup (result_host), - GINT_TO_POINTER (result_count + 1)); + if (strcasecmp (compliance, "yes") == 0) + { + f_host_result_counts = f_host_compliant; + if (count_filtered) + f_compliance_yes++; + } + else if (strcasecmp (compliance, "no") == 0) + { + f_host_result_counts = f_host_notcompliant; + if (count_filtered) + f_compliance_no++; + } + else if (strcasecmp (compliance, "incomplete") == 0) + { + f_host_result_counts = f_host_incomplete; + if (count_filtered) + f_compliance_incomplete++; + } + else if (strcasecmp (compliance, "undefined") == 0) + { + f_host_result_counts = f_host_undefined; + if (count_filtered) + f_compliance_undefined++; + } + else + { + f_host_result_counts = NULL; } - } else - #endif - { - double result_severity; - result_severity = result_iterator_severity_double (&results); - if (result_severity > f_severity) - f_severity = result_severity; - - level = result_iterator_level (&results); - if (strcasecmp (level, "log") == 0) - { - f_host_result_counts = f_host_logs; - if (count_filtered) - f_logs++; - } - else if (strcasecmp (level, "high") == 0) - { - f_host_result_counts = f_host_holes; - if (count_filtered) - f_holes++; - } - else if (strcasecmp (level, "medium") == 0) - { - f_host_result_counts = f_host_warnings; - if (count_filtered) - f_warnings++; - } - else if (strcasecmp (level, "low") == 0) - { - f_host_result_counts = f_host_infos; - if (count_filtered) - f_infos++; - } - else if (strcasecmp (level, "false positive") == 0) - { - f_host_result_counts = f_host_false_positives; - if (count_filtered) - f_false_positives++; + if (f_host_result_counts) + { + const char *result_host = result_iterator_host (&results); + int result_count + = GPOINTER_TO_INT + (g_hash_table_lookup (f_host_result_counts, + result_host)); + + g_hash_table_replace (f_host_result_counts, + g_strdup (result_host), + GINT_TO_POINTER (result_count + 1)); } - else - f_host_result_counts = NULL; + } + else + { + double result_severity; + result_severity = result_iterator_severity_double (&results); + if (result_severity > f_severity) + f_severity = result_severity; - if (f_host_result_counts) - { - const char *result_host = result_iterator_host (&results); - int result_count - = GPOINTER_TO_INT - (g_hash_table_lookup (f_host_result_counts, result_host)); - - g_hash_table_replace (f_host_result_counts, - g_strdup (result_host), - GINT_TO_POINTER (result_count + 1)); - } - } + level = result_iterator_level (&results); + if (strcasecmp (level, "log") == 0) + { + f_host_result_counts = f_host_logs; + if (count_filtered) + f_logs++; + } + else if (strcasecmp (level, "high") == 0) + { + f_host_result_counts = f_host_holes; + if (count_filtered) + f_holes++; + } + else if (strcasecmp (level, "medium") == 0) + { + f_host_result_counts = f_host_warnings; + if (count_filtered) + f_warnings++; + } + else if (strcasecmp (level, "low") == 0) + { + f_host_result_counts = f_host_infos; + if (count_filtered) + f_infos++; + } + else if (strcasecmp (level, "false positive") == 0) + { + f_host_result_counts = f_host_false_positives; + if (count_filtered) + f_false_positives++; + } + else + f_host_result_counts = NULL; + + if (f_host_result_counts) + { + const char *result_host = result_iterator_host (&results); + int result_count + = GPOINTER_TO_INT + (g_hash_table_lookup (f_host_result_counts, result_host)); + + g_hash_table_replace (f_host_result_counts, + g_strdup (result_host), + GINT_TO_POINTER (result_count + 1)); + } + } } PRINT (out, ""); } @@ -31031,131 +31009,130 @@ print_report_xml_start (report_t report, report_t delta, task_t task, /* Print result counts and severity. */ - #if COMPLIANCE_REPORTS == 1 - if (strcmp (tsk_usage_type, "audit") == 0) - { - if (delta) + if (strcmp (tsk_usage_type, "audit") == 0) + { + if (delta) + PRINT (out, + "" + "%i" + "%i" + "%i" + "%i" + "%i" + "", + f_compliance_count, + (strchr (compliance_levels, 'y') ? f_compliance_yes : 0), + (strchr (compliance_levels, 'n') ? f_compliance_no : 0), + (strchr (compliance_levels, 'i') ? f_compliance_incomplete : 0), + (strchr (compliance_levels, 'u') ? f_compliance_undefined : 0)); + else + { + if (count_filtered) + f_compliance_count = f_compliance_yes + + f_compliance_no + + f_compliance_incomplete + + f_compliance_undefined; PRINT (out, - "" - "%i" - "%i" - "%i" - "%i" - "%i" - "", - f_compliance_count, - (strchr (compliance_levels, 'y') ? f_compliance_yes : 0), - (strchr (compliance_levels, 'n') ? f_compliance_no : 0), - (strchr (compliance_levels, 'i') ? f_compliance_incomplete : 0), - (strchr (compliance_levels, 'u') ? f_compliance_undefined : 0)); - else - { - if (count_filtered) - f_compliance_count = f_compliance_yes - + f_compliance_no - + f_compliance_incomplete - + f_compliance_undefined; - PRINT (out, - "" - "%i" - "%i" - "%i" - "%i%i" - "%i%i" - "%i%i" - "%i%i" - "", - total_compliance_count, - total_compliance_count, - f_compliance_count, - compliance_yes, - (strchr (compliance_levels, 'y') ? f_compliance_yes : 0), - compliance_no, - (strchr (compliance_levels, 'n') ? f_compliance_no : 0), - compliance_incomplete, - (strchr (compliance_levels, 'i') ? f_compliance_incomplete : 0), - compliance_undefined, - (strchr (compliance_levels, 'i') ? f_compliance_undefined : 0)); - - PRINT (out, - "" - "%s" - "%s" - "", - report_compliance_from_counts (&compliance_yes, - &compliance_no, - &compliance_incomplete, - &compliance_undefined), - report_compliance_from_counts (&f_compliance_yes, - &f_compliance_no, - &f_compliance_incomplete, - &f_compliance_undefined)); - } - } else - #endif - { - if (delta) - /** @todo The f_holes, etc. vars are setup to give the page count. */ - PRINT (out, - "" - "%i" - "%i" - "%i" - "%i" - "%i" - "" - "%i" - "" - "", - orig_filtered_result_count, - (strchr (levels, 'h') ? orig_f_holes : 0), - (strchr (levels, 'l') ? orig_f_infos : 0), - (strchr (levels, 'g') ? orig_f_logs : 0), - (strchr (levels, 'm') ? orig_f_warnings : 0), - (strchr (levels, 'f') ? orig_f_false_positives : 0)); - else - { - if (count_filtered) - filtered_result_count = f_holes + f_infos + f_logs - + f_warnings + false_positives; + "" + "%i" + "%i" + "%i" + "%i%i" + "%i%i" + "%i%i" + "%i%i" + "", + total_compliance_count, + total_compliance_count, + f_compliance_count, + compliance_yes, + (strchr (compliance_levels, 'y') ? f_compliance_yes : 0), + compliance_no, + (strchr (compliance_levels, 'n') ? f_compliance_no : 0), + compliance_incomplete, + (strchr (compliance_levels, 'i') ? f_compliance_incomplete : 0), + compliance_undefined, + (strchr (compliance_levels, 'i') ? f_compliance_undefined : 0)); + PRINT (out, + "" + "%s" + "%s" + "", + report_compliance_from_counts (&compliance_yes, + &compliance_no, + &compliance_incomplete, + &compliance_undefined), + report_compliance_from_counts (&f_compliance_yes, + &f_compliance_no, + &f_compliance_incomplete, + &f_compliance_undefined)); + } + } + else + { + if (delta) + /** @todo The f_holes, etc. vars are setup to give the page count. */ PRINT (out, "" - "%i" - "%i" "%i" - "%i%i" - "%i%i" - "%i%i" - "%i%i" + "%i" + "%i" + "%i" + "%i" "" - "%i" "%i" "" "", - total_result_count, - total_result_count, - filtered_result_count, - holes, - (strchr (levels, 'h') ? f_holes : 0), - infos, - (strchr (levels, 'l') ? f_infos : 0), - logs, - (strchr (levels, 'g') ? f_logs : 0), - warnings, - (strchr (levels, 'm') ? f_warnings : 0), - false_positives, - (strchr (levels, 'f') ? f_false_positives : 0)); + orig_filtered_result_count, + (strchr (levels, 'h') ? orig_f_holes : 0), + (strchr (levels, 'l') ? orig_f_infos : 0), + (strchr (levels, 'g') ? orig_f_logs : 0), + (strchr (levels, 'm') ? orig_f_warnings : 0), + (strchr (levels, 'f') ? orig_f_false_positives : 0)); + else + { + if (count_filtered) + filtered_result_count = f_holes + f_infos + f_logs + + f_warnings + false_positives; - PRINT (out, - "" - "%1.1f" - "%1.1f" - "", - severity, - f_severity); - } - } + PRINT (out, + "" + "%i" + "%i" + "%i" + "%i%i" + "%i%i" + "%i%i" + "%i%i" + "" + "%i" + "%i" + "" + "", + total_result_count, + total_result_count, + filtered_result_count, + holes, + (strchr (levels, 'h') ? f_holes : 0), + infos, + (strchr (levels, 'l') ? f_infos : 0), + logs, + (strchr (levels, 'g') ? f_logs : 0), + warnings, + (strchr (levels, 'm') ? f_warnings : 0), + false_positives, + (strchr (levels, 'f') ? f_false_positives : 0)); + + PRINT (out, + "" + "%1.1f" + "%1.1f" + "", + severity, + f_severity); + } + } if (host_summary) { @@ -31236,22 +31213,21 @@ print_report_xml_start (report_t report, report_t delta, task_t task, } cleanup_iterator (&hosts); } - #if COMPLIANCE_REPORTS == 1 - if (strcmp (tsk_usage_type, "audit") == 0) - { - g_hash_table_destroy (f_host_compliant); - g_hash_table_destroy (f_host_notcompliant); - g_hash_table_destroy (f_host_incomplete); - g_hash_table_destroy (f_host_undefined); - } else - #endif - { - g_hash_table_destroy (f_host_holes); - g_hash_table_destroy (f_host_warnings); - g_hash_table_destroy (f_host_infos); - g_hash_table_destroy (f_host_logs); - g_hash_table_destroy (f_host_false_positives); - } + if (strcmp (tsk_usage_type, "audit") == 0) + { + g_hash_table_destroy (f_host_compliant); + g_hash_table_destroy (f_host_notcompliant); + g_hash_table_destroy (f_host_incomplete); + g_hash_table_destroy (f_host_undefined); + } + else + { + g_hash_table_destroy (f_host_holes); + g_hash_table_destroy (f_host_warnings); + g_hash_table_destroy (f_host_infos); + g_hash_table_destroy (f_host_logs); + g_hash_table_destroy (f_host_false_positives); + } g_hash_table_destroy (f_host_ports); /* Print TLS certificates */ @@ -31349,36 +31325,36 @@ print_report_xml_start (report_t report, report_t delta, task_t task, return 0; failed_delta_report: - fclose (out); - g_free (sort_field); - g_free (levels); - g_free (search_phrase); - g_free (min_qod); - g_free (delta_states); - cleanup_iterator (&results); - cleanup_iterator (&delta_results); + fclose (out); + g_free (sort_field); + g_free (levels); + g_free (search_phrase); + g_free (min_qod); + g_free (delta_states); + cleanup_iterator (&results); + cleanup_iterator (&delta_results); failed_print_report_host: if (host_summary_buffer) g_string_free (host_summary_buffer, TRUE); tz_revert (zone, tz, old_tz_override); g_hash_table_destroy (f_host_ports); - #if COMPLIANCE_REPORTS == 1 - g_free (compliance_levels); - if (strcmp (tsk_usage_type, "audit") == 0) + + g_free (compliance_levels); + if (strcmp (tsk_usage_type, "audit") == 0) { g_hash_table_destroy (f_host_compliant); g_hash_table_destroy (f_host_notcompliant); g_hash_table_destroy (f_host_incomplete); g_hash_table_destroy (f_host_undefined); - } else - #endif - { - g_hash_table_destroy (f_host_holes); - g_hash_table_destroy (f_host_warnings); - g_hash_table_destroy (f_host_infos); - g_hash_table_destroy (f_host_logs); - g_hash_table_destroy (f_host_false_positives); - } + } + else + { + g_hash_table_destroy (f_host_holes); + g_hash_table_destroy (f_host_warnings); + g_hash_table_destroy (f_host_infos); + g_hash_table_destroy (f_host_logs); + g_hash_table_destroy (f_host_false_positives); + } return -1; } @@ -53686,10 +53662,8 @@ modify_setting (const gchar *uuid, const gchar *name, setting_name = g_strdup ("Alerts Filter"); else if (strcmp (uuid, "0f040d06-abf9-43a2-8f94-9de178b0e978") == 0) setting_name = g_strdup ("Assets Filter"); - #if COMPLIANCE_REPORTS == 1 - else if (strcmp (uuid, "45414da7-55f0-44c1-abbb-6b7d1126fbdf") == 0) - setting_name = g_strdup ("Audit Reports Filter"); - #endif + else if (strcmp (uuid, "45414da7-55f0-44c1-abbb-6b7d1126fbdf") == 0) + setting_name = g_strdup ("Audit Reports Filter"); else if (strcmp (uuid, "1a9fbd91-0182-44cd-bc88-a13a9b3b1bef") == 0) setting_name = g_strdup ("Configs Filter"); else if (strcmp (uuid, "186a5ac8-fe5a-4fb1-aa22-44031fb339f3") == 0) @@ -53813,10 +53787,8 @@ modify_setting (const gchar *uuid, const gchar *name, setting_name = g_strdup ("Reports Top Dashboard Configuration"); /* Audit Reports dashboard settings */ - #if COMPLIANCE_REPORTS == 1 - else if (strcmp (uuid, "8083d77b-05bb-4b17-ab39-c81175cb512c") == 0) - setting_name = g_strdup ("Audit Reports Top Dashboard Configuration"); - #endif + else if (strcmp (uuid, "8083d77b-05bb-4b17-ab39-c81175cb512c") == 0) + setting_name = g_strdup ("Audit Reports Top Dashboard Configuration"); /* Results dashboard settings */ else if (strcmp (uuid, "0b8ae70d-d8fc-4418-8a72-e65ac8d2828e") == 0) setting_name = g_strdup ("Results Top Dashboard Configuration"); @@ -57579,18 +57551,16 @@ tag_add_resources_filter (tag_t tag, const char *type, const char *filter) { get_data_set_extra (&resources_get, "usage_type", g_strdup ("scan")); } - #if COMPLIANCE_REPORTS == 1 - else if (strcasecmp (type, "audit_report") == 0) - { - type = g_strdup ("report"); - resources_get.type = g_strdup (type); - get_data_set_extra (&resources_get, "usage_type", g_strdup ("audit")); - } - else if (strcasecmp (type, "report") == 0) - { - get_data_set_extra (&resources_get, "usage_type", g_strdup ("scan")); - } - #endif + else if (strcasecmp (type, "audit_report") == 0) + { + type = g_strdup ("report"); + resources_get.type = g_strdup (type); + get_data_set_extra (&resources_get, "usage_type", g_strdup ("audit")); + } + else if (strcasecmp (type, "report") == 0) + { + get_data_set_extra (&resources_get, "usage_type", g_strdup ("scan")); + } gchar *columns; @@ -57762,20 +57732,18 @@ tag_remove_resources_filter (tag_t tag, const char *type, const char *filter) { get_data_set_extra (&resources_get, "usage_type", g_strdup ("scan")); } - #if COMPLIANCE_REPORTS == 1 - else if (strcasecmp (type, "audit_report") == 0) - { - type = g_strdup ("report"); - resources_get.type = g_strdup (type); - get_data_set_extra (&resources_get, - "usage_type", - g_strdup ("audit")); - } - else if (strcasecmp (type, "report") == 0) - { - get_data_set_extra (&resources_get, "usage_type", g_strdup ("scan")); - } - #endif + else if (strcasecmp (type, "audit_report") == 0) + { + type = g_strdup ("report"); + resources_get.type = g_strdup (type); + get_data_set_extra (&resources_get, + "usage_type", + g_strdup ("audit")); + } + else if (strcasecmp (type, "report") == 0) + { + get_data_set_extra (&resources_get, "usage_type", g_strdup ("scan")); + } gchar *columns; From 4e12e124e8a5d81e4a378d6db788f9f225bb1c1c Mon Sep 17 00:00:00 2001 From: Johannes Helmold Date: Mon, 4 Nov 2024 17:48:21 +0100 Subject: [PATCH 21/26] Dropped the dump that compliance reports are enabled. Removed the dump that compliance reports are enabled, because that is alway the case from now on. --- src/gvmd.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/gvmd.c b/src/gvmd.c index 722ea2022..99f8e1fa1 100644 --- a/src/gvmd.c +++ b/src/gvmd.c @@ -2325,7 +2325,6 @@ gvmd (int argc, char** argv, char *env[]) #if CVSS3_RATINGS == 1 printf ("CVSS3 severity ratings enabled\n"); #endif - printf ("Compliance reports enabled\n"); printf ("Copyright (C) 2009-2021 Greenbone AG\n"); printf ("License: AGPL-3.0-or-later\n"); printf From fcb7397ab49be6026d37a7349f30a2ba99312d5e Mon Sep 17 00:00:00 2001 From: Johannes Helmold Date: Tue, 5 Nov 2024 10:37:54 +0100 Subject: [PATCH 22/26] Removed variables @IF_COMPLIANCE_REPORTS@, @ENDIF_COMPLIANCE_REPORTS@. --- src/schema_formats/XML/GMP.xml.in | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/src/schema_formats/XML/GMP.xml.in b/src/schema_formats/XML/GMP.xml.in index eb2feaf73..5f2ca6573 100644 --- a/src/schema_formats/XML/GMP.xml.in +++ b/src/schema_formats/XML/GMP.xml.in @@ -68,7 +68,6 @@ along with this program. If not, see . xsd:token { pattern = "y?n?i?u?" } - @IF_COMPLIANCE_REPORTS@ compliance_status A compliance status @@ -76,7 +75,6 @@ along with this program. If not, see . xsd:token { pattern = "yes|no|incomplete|undefined" } - @ENDIF_COMPLIANCE_REPORTS@ ctime A date and time, in the C `ctime' format @@ -2025,16 +2023,12 @@ along with this program. If not, see . permissions user_tags scan_run_status - @IF_COMPLIANCE_REPORTS@ - @ENDIF_COMPLIANCE_REPORTS@ result_count severity - @IF_COMPLIANCE_REPORTS@ compliance_count compliance - @ENDIF_COMPLIANCE_REPORTS@ task ports results @@ -2468,7 +2462,6 @@ along with this program. If not, see . - @IF_COMPLIANCE_REPORTS@ compliance_count Counts of compliance results. Only for reports of an audit task. @@ -2577,7 +2570,6 @@ along with this program. If not, see . - @ENDIF_COMPLIANCE_REPORTS@ severity @@ -2595,7 +2587,6 @@ along with this program. If not, see . Maximum severity of the report after filtering - @IF_COMPLIANCE_REPORTS@ compliance @@ -2613,7 +2604,6 @@ along with this program. If not, see . Compliance of the report after filtering ("yes", "no", "incomplete" or "undefined") - @ENDIF_COMPLIANCE_REPORTS@ task @@ -2875,15 +2865,11 @@ along with this program. If not, see . start end port_count - @IF_COMPLIANCE_REPORTS@ - @ENDIF_COMPLIANCE_REPORTS@ result_count - @IF_COMPLIANCE_REPORTS@ compliance_count host_compliance - @ENDIF_COMPLIANCE_REPORTS@ detail @@ -3001,7 +2987,6 @@ along with this program. If not, see . - @IF_COMPLIANCE_REPORTS@ compliance_count Only for audit reports @@ -3071,7 +3056,6 @@ along with this program. If not, see . Only for audit reports. Host compliance compliance_status - @ENDIF_COMPLIANCE_REPORTS@ detail A detail associated with the host @@ -18029,13 +18013,11 @@ END:VCALENDAR integer Minimum QoD of the results - @IF_COMPLIANCE_REPORTS@ - @ENDIF_COMPLIANCE_REPORTS@ tag text @@ -18181,7 +18163,6 @@ END:VCALENDAR iso_time Scan end time - @IF_COMPLIANCE_REPORTS@ compliance_yes integer @@ -18202,7 +18183,6 @@ END:VCALENDAR compliance_status Compliance state of the report. Can be yes, no, incomplete or undefined - @ENDIF_COMPLIANCE_REPORTS@ @@ -18272,7 +18252,6 @@ END:VCALENDAR boolean - @IF_COMPLIANCE_REPORTS@ usage_type Optional usage type to limit the reports to. Affects total count unlike filter @@ -18284,7 +18263,6 @@ END:VCALENDAR - @ENDIF_COMPLIANCE_REPORTS@ @@ -23005,15 +22983,11 @@ END:VCALENDAR timestamp scan_end - @IF_COMPLIANCE_REPORTS@ - @ENDIF_COMPLIANCE_REPORTS@ result_count severity - @IF_COMPLIANCE_REPORTS@ compliance_count - @ENDIF_COMPLIANCE_REPORTS@ timestamp @@ -23059,7 +23033,6 @@ END:VCALENDAR severity Maximum severity of the report - @IF_COMPLIANCE_REPORTS@ compliance_count Complaince counts. Only for audit tasks @@ -23086,7 +23059,6 @@ END:VCALENDAR integer - @ENDIF_COMPLIANCE_REPORTS@ From d22536008eb33e9f7b0a8a09209cfbf313c278f2 Mon Sep 17 00:00:00 2001 From: Johannes Helmold Date: Thu, 7 Nov 2024 16:23:10 +0100 Subject: [PATCH 23/26] Add: The table scap.affected_products is filled for the new JSON feed. This commit contains the filling of the table scap.affected_products and a small bug-fix for the CVE scan. --- src/manage_sql.c | 2 +- src/manage_sql_secinfo.c | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/manage_sql.c b/src/manage_sql.c index be266c79a..613775b10 100644 --- a/src/manage_sql.c +++ b/src/manage_sql.c @@ -20511,7 +20511,7 @@ init_cpe_match_nodes_iterator (iterator_t* iterator, const char *criteria) " JOIN scap.cpe_nodes_match_criteria c" " ON n.id = c.node_id" " JOIN scap.cpe_match_strings r" - " ON c.match_criteria = r.match_criteria_id" + " ON c.match_criteria_id = r.match_criteria_id" " WHERE criteria like '%s%%';", quoted_criteria); g_free (quoted_criteria); diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index 4e6d3d6fc..ed4903771 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -4043,6 +4043,31 @@ update_scap_cves () return 0; } +/** + * @brief Update SCAP affected products. + * + * Assume that the databases are attached. + * + * @return 0 success, -1 error. + */ +static int +update_scap_affected_products () +{ + g_info ("Updating affected products"); + + sql ("INSERT INTO scap2.affected_products" + " SELECT DISTINCT scap2.cpe_match_nodes.cve_id, scap2.cpes.id" + " FROM scap2.cpe_match_nodes, scap2.cpe_nodes_match_criteria," + " scap2.cpe_matches, scap2.cpes" + " WHERE scap2.cpe_match_nodes.id = scap2.cpe_nodes_match_criteria.node_id" + " AND scap2.cpe_nodes_match_criteria.vulnerable = 1" + " AND scap2.cpe_nodes_match_criteria.match_criteria_id =" + " scap2.cpe_matches.match_criteria_id" + " AND scap2.cpe_matches.cpe_name_id = scap2.cpes.cpe_name_id;"); + + return 0; +} + /** * @brief Insert a SCAP CPE match string from JSON. * @@ -5606,6 +5631,15 @@ update_scap (gboolean reset_scap_db) return -1; } + g_debug ("%s: update affected_products", __func__); + setproctitle ("Syncing SCAP: Updating affected products"); + + if (update_scap_affected_products () == -1) + { + abort_scap_update (); + return -1; + } + g_debug ("%s: updating user defined data", __func__); g_debug ("%s: update epss", __func__); From b3ff0ab4d139f99cf8905d3bbaf4089661d1c2dd Mon Sep 17 00:00:00 2001 From: Johannes Helmold Date: Mon, 11 Nov 2024 10:03:19 +0100 Subject: [PATCH 24/26] Small amendment. --- src/manage_sql_secinfo.c | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index ed4903771..fead001e6 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -4047,10 +4047,8 @@ update_scap_cves () * @brief Update SCAP affected products. * * Assume that the databases are attached. - * - * @return 0 success, -1 error. */ -static int +static void update_scap_affected_products () { g_info ("Updating affected products"); @@ -4064,8 +4062,6 @@ update_scap_affected_products () " AND scap2.cpe_nodes_match_criteria.match_criteria_id =" " scap2.cpe_matches.match_criteria_id" " AND scap2.cpe_matches.cpe_name_id = scap2.cpes.cpe_name_id;"); - - return 0; } /** @@ -5634,11 +5630,7 @@ update_scap (gboolean reset_scap_db) g_debug ("%s: update affected_products", __func__); setproctitle ("Syncing SCAP: Updating affected products"); - if (update_scap_affected_products () == -1) - { - abort_scap_update (); - return -1; - } + update_scap_affected_products (); g_debug ("%s: updating user defined data", __func__); From 7cb34b7d3b36f68f3164f20988a6aa5ffc4801f6 Mon Sep 17 00:00:00 2001 From: Ahmed Abdelsalam Date: Fri, 8 Nov 2024 15:18:40 +0100 Subject: [PATCH 25/26] Fix: Remove the insertion of unused deprecated-by-id from XML CPE feed files --- src/manage_sql_secinfo.c | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index fead001e6..70b819944 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -2081,7 +2081,7 @@ static int insert_scap_cpe (inserts_t *inserts, element_t cpe_item, element_t item_metadata, int modification_time) { - gchar *name, *status, *deprecated, *nvd_id; + gchar *name, *status, *nvd_id; gchar *quoted_name, *quoted_title, *quoted_status, *quoted_nvd_id; element_t title; int first; @@ -2103,27 +2103,12 @@ insert_scap_cpe (inserts_t *inserts, element_t cpe_item, element_t item_metadata return -1; } - deprecated = element_attribute (item_metadata, - "deprecated-by-nvd-id"); - if (deprecated - && (g_regex_match_simple ("^[0-9]+$", (gchar *) deprecated, 0, 0) - == 0)) - { - g_warning ("%s: invalid deprecated-by-nvd-id: %s", - __func__, - deprecated); - g_free (name); - g_free (status); - return -1; - } - nvd_id = element_attribute (item_metadata, "nvd-id"); if (nvd_id == NULL) { g_warning ("%s: nvd_id missing", __func__); g_free (name); g_free (status); - g_free (deprecated); return -1; } @@ -2163,7 +2148,7 @@ insert_scap_cpe (inserts_t *inserts, element_t cpe_item, element_t item_metadata first = inserts_check_size (inserts); g_string_append_printf (inserts->statement, - "%s ('%s', '%s', '%s', %i, %i, '%s', %s, '%s')", + "%s ('%s', '%s', '%s', %i, %i, '%s', '%s')", first ? "" : ",", quoted_name, quoted_name, @@ -2171,7 +2156,6 @@ insert_scap_cpe (inserts_t *inserts, element_t cpe_item, element_t item_metadata modification_time, modification_time, quoted_status, - deprecated ? deprecated : "NULL", quoted_nvd_id); inserts->current_chunk_size++; @@ -2180,7 +2164,6 @@ insert_scap_cpe (inserts_t *inserts, element_t cpe_item, element_t item_metadata g_free (quoted_name); g_free (quoted_status); g_free (quoted_nvd_id); - g_free (deprecated); return 0; } From 2ed6a1b7844c45c7dc3cab126994f5b4766d4bb7 Mon Sep 17 00:00:00 2001 From: Ahmed Abdelsalam Date: Mon, 11 Nov 2024 16:53:44 +0100 Subject: [PATCH 26/26] Change: Insert CVEs products from CPE matches table --- src/manage_sql_secinfo.c | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index 70b819944..cf13bf215 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -3406,6 +3406,7 @@ handle_cve_configurations (resource_t cve_db_id, char * cve_id, cJSON* configurations_json) { cJSON *configuration_item; + GString *software = g_string_new (""); cJSON_ArrayForEach (configuration_item, configurations_json) { @@ -3495,11 +3496,30 @@ handle_cve_configurations (resource_t cve_db_id, char * cve_id, id, vulnerable ? 1 : 0, quoted_match_criteria_id); - + + if (vulnerable) + { + iterator_t cpe_matches; + init_cpe_match_string_matches_iterator (&cpe_matches, quoted_match_criteria_id); + while (next (&cpe_matches)) + g_string_append_printf (software, "%s ", cpe_matches_cpe_name (&cpe_matches)); + cleanup_iterator (&cpe_matches); + } g_free (quoted_match_criteria_id); } } } + if (software->len > 0) + { + gchar *quoted_software = sql_quote (software->str); + sql ("UPDATE scap2.cves" + " SET products = '%s'" + " WHERE id = %llu;", + quoted_software, cve_db_id); + g_free (quoted_software); + } + g_string_free (software, TRUE); + return 0; }