From abcac03049fe4014683ba6afdec221ec305025d2 Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 15 Apr 2024 21:27:12 +0700 Subject: [PATCH 1/9] New. Scanner. Analysis stats and estimated time. --- inc/spbc-scanner.php | 57 ++++++++---- inc/spbc-settings.php | 87 +++++++++++++++++-- lib/CleantalkSP/SpbctWP/DB/SQLSchema.php | 1 + .../SpbctWP/Scanner/ScannerQueue.php | 4 +- security-malware-firewall.php | 3 +- 5 files changed, 124 insertions(+), 28 deletions(-) diff --git a/inc/spbc-scanner.php b/inc/spbc-scanner.php index 3997a234f..c95230c46 100644 --- a/inc/spbc-scanner.php +++ b/inc/spbc-scanner.php @@ -361,7 +361,12 @@ function spbc_scanner_file_send($direct_call = false, $file_id = null, $do_resca if ($sql_result !== false) { $output = array('success' => true, 'result' => $api_response); //set new cron to resend unqueued files - \CleantalkSP\SpbctWP\Cron::updateTask('scanner_resend_pscan_files', 'spbc_scanner_resend_pscan_files', SPBC_PSCAN_UPDATE_FILES_STATUS_PERIOD, time() + SPBC_PSCAN_UPDATE_FILES_STATUS_PERIOD); + \CleantalkSP\SpbctWP\Cron::updateTask( + 'scanner_resend_pscan_files', + 'spbc_scanner_resend_pscan_files', + SPBC_PSCAN_RESEND_FILES_STATUS_PERIOD, + time() + SPBC_PSCAN_RESEND_FILES_STATUS_PERIOD + ); //error on fail } else { $output = array('error' => 'DB_COULD_NOT_UPDATE pscan_pending_queue'); @@ -772,6 +777,11 @@ function spbc_scanner_pscan_check_analysis_status($direct_call = false, $file_id $file_info['pscan_file_id'] ); + //todo Probably remove on backend ready + $api_response['estimated_execution_time'] = isset($api_response['estimated_execution_time']) + ? (int)$api_response['estimated_execution_time'] + : 100; + // Validate API response try { $api_response = spbc_scanner_validate_pscan_status_response($api_response); @@ -789,22 +799,14 @@ function spbc_scanner_pscan_check_analysis_status($direct_call = false, $file_id /* * If file process is not finished, update data */ - // Set old processing status to compare with next - $old_processing_status = !empty($file_info['pscan_processing_status']) ? $file_info['pscan_processing_status'] : null; - // Update processing status - if ( $api_response['processing_status'] !== $old_processing_status ) { - // Keep update result - $update_result = $wpdb->query( - 'UPDATE ' . SPBC_TBL_SCAN_FILES - . ' SET ' - . ' pscan_pending_queue = 0, ' - . ' pscan_processing_status = "' . $api_response['processing_status'] . '"' - . ' WHERE pscan_file_id = "' . $file_info['pscan_file_id'] . '"' - ); - } else { - // Status have not been changed, however status process is succesfull - $update_result = true; - } + $update_result = $wpdb->query( + 'UPDATE ' . SPBC_TBL_SCAN_FILES + . ' SET ' + . ' pscan_pending_queue = 0, ' + . ' pscan_processing_status = "' . $api_response['processing_status'] . '",' + . ' pscan_estimated_execution_time = "' . $api_response['estimated_execution_time'] . '"' + . ' WHERE pscan_file_id = "' . $file_info['pscan_file_id'] . '"' + ); } else { if ( $api_response['file_status'] === 'SAFE' ) { /* @@ -818,7 +820,8 @@ function spbc_scanner_pscan_check_analysis_status($direct_call = false, $file_id . ' pscan_pending_queue = 0, ' . ' pscan_status = "SAFE",' . ' pscan_balls = %s,' - . ' status = "APPROVED_BY_CLOUD" ' + . ' status = "APPROVED_BY_CLOUD",' + . ' pscan_estimated_execution_time = NULL' . ' WHERE pscan_file_id = %s', isset($api_response['file_balls']) ? $api_response['file_balls'] : '{SAFE:0}', $file_info['pscan_file_id'] @@ -836,7 +839,8 @@ function spbc_scanner_pscan_check_analysis_status($direct_call = false, $file_id . ' pscan_status = %s ,' . ' severity = "CRITICAL",' . ' pscan_balls = %s,' - . ' status = "DENIED_BY_CLOUD"' + . ' status = "DENIED_BY_CLOUD",' + . ' pscan_estimated_execution_time = NULL' . ' WHERE pscan_file_id = %s', $api_response['file_status'], isset($api_response['file_balls']) ? $api_response['file_balls'] : '{DANGEROUS:0}', @@ -954,6 +958,7 @@ function spbc_scanner_pscan_update_check_exclusions(array $file_info) /** * @param array $response API Response + * @param bool $await_estimated_data Do await estimated data set on undone files check * @return mixed API Response * @throws Exception if validation failed */ @@ -999,6 +1004,20 @@ function spbc_scanner_validate_pscan_status_response($response) } } + //estimated time validation + if ( $response['processing_status'] !== 'DONE') { + if ( ! isset($response['estimated_execution_time'])) { + throw new Exception('response provided no estimated scan time'); + } + //todo remove on business decision + //if ( ! isset($response['number_of_files'])) { + // throw new Exception('response provided no number of estimated files'); + //} + //if ( ! isset($response['number_of_files_scanned'])) { + // throw new Exception('response provided no number of already scanned files'); + //} + } + return $response; } diff --git a/inc/spbc-settings.php b/inc/spbc-settings.php index 786c38a17..31977234f 100644 --- a/inc/spbc-settings.php +++ b/inc/spbc-settings.php @@ -2642,6 +2642,12 @@ function spbc_field_scanner__prepare_data__analysis_log(&$table) $analysis_comment = __('Processing: queue is full. File will be resent in 5 minutes.', 'security-malware-firewall'); } + if ( !is_null($row->pscan_estimated_execution_time) ) { + $estimated_execution_time = $row->pscan_estimated_execution_time . ' ' . __('second(s)', 'security-malware-firewall'); + } else { + $estimated_execution_time = $row->pscan_processing_status === 'DONE' ? 'Done' : 'Wait for assessing'; + } + // Filter actions for approved files if ( in_array($row->pscan_status, array('SAFE','DANGEROUS')) || $curr_time - $row->last_sent < 500 ) { unset($row->actions['check_analysis_status']); @@ -2662,6 +2668,7 @@ function spbc_field_scanner__prepare_data__analysis_log(&$table) 'last_sent' => is_numeric($row->last_sent) ? date('M j, Y, H:i:s', $row->last_sent) : null, 'pscan_status' => $pscan_status, 'analysis_comment' => $analysis_comment, + 'pscan_estimated_execution_time' => $estimated_execution_time, 'actions' => $row->actions, ); } @@ -3069,6 +3076,74 @@ function spbc_field_scanner() echo ''; } +/** + * Retrieves HTML code block to layout files counters stats in the analysis accordion. + * @return string + */ +function spbc__analysyis_files_stats__get_html() +{ + $out = ' +
+

%s

+

%s: %d / %d / %d

+ %s +
+ '; + $caption = __('List of files sent for the Cloud analysis, it takes up to 10 minutes to process a file. Refresh the page to have the results.', 'security-malware-firewall'); + $files_stats_string = __('Files sent/checked/unchecked', 'security-malware-firewall'); + $data = spbc__analysyis_files_stats__get_data(); + $last_updated_chunk = __('Files statuses updates every', 'security-malware-firewall') . ' ' . SPBC_PSCAN_UPDATE_FILES_STATUS_PERIOD . ' seconds'; + $last_updated_chunk .= $data['last_updated'] && is_int($data['last_updated']) + ? ', ' . __('last update time', 'security-malware-firewall') . ': ' . date("Y-m-d H:i:s", $data['last_updated']) + : '.'; + $out = sprintf( + $out, + $caption, + $files_stats_string, + $data['files_sent_count'], + $data['files_checked_count'], + $data['files_unchecked_count'], + $last_updated_chunk + ); + return $out; +} + +/** + * Retrieves the data for analysis stats block. + * @return array + */ +function spbc__analysyis_files_stats__get_data() +{ + global $wpdb, $spbc; + $out = array( + 'files_sent_count' => 'N/D', + 'files_checked_count' => 'N/D', + 'files_unchecked_count' => 'N/D', + 'last_updated' => false, + ); + $files_sent_count = $wpdb->get_var(' + SELECT COUNT(*) from ' . SPBC_TBL_SCAN_FILES . ' + WHERE last_sent IS NOT NULL; + '); + $files_checked_count = $wpdb->get_var(' + SELECT COUNT(*) from ' . SPBC_TBL_SCAN_FILES . ' + WHERE last_sent IS NOT NULL AND pscan_processing_status = \'DONE\'; + '); + $files_unchecked_count = !is_null($files_sent_count) && !is_null($files_checked_count) + ? (int)$files_sent_count - (int)$files_checked_count + : false; + $last_updated = \CleantalkSP\SpbctWP\Cron::getTask('scanner_update_pscan_files_status'); + //next call checking is a trick - the last_call key does not work properly + $last_updated = $last_updated && !empty($last_updated['next_call']) + ? $last_updated['next_call'] - $last_updated['period'] + $spbc->data['site_utc_offset_in_seconds'] + : false; + $out['files_sent_count'] = !is_null($files_sent_count) ? (int)$files_sent_count : $out['files_sent_count']; + $out['files_checked_count'] = !is_null($files_checked_count) ? (int)$files_checked_count : $out['files_checked_count']; + $out['files_unchecked_count'] = $files_unchecked_count ? : $out['files_unchecked_count']; + $out['last_updated'] = $last_updated ? : $out['last_updated']; + return $out; +} + function spbc_field_scanner__show_accordion($direct_call = false) { if ( ! $direct_call) { @@ -3083,8 +3158,7 @@ function spbc_field_scanner__show_accordion($direct_call = false) '', '' ) : ''; - $analysis_log_description = '
' . - __('List of files sent for the Cloud analysis, it takes up to 10 minutes to process a file. Refresh the page to have the results.', 'security-malware-firewall') . + $analysis_log_description = spbc__analysyis_files_stats__get_html() . '
' . '

' . 'attention' . @@ -3092,7 +3166,6 @@ function spbc_field_scanner__show_accordion($direct_call = false) __('If you feel that the Cloud verdict is incorrect, please click the link "Copy file info" near the file name and contact us', 'security-malware-firewall') . ' ' . $dashboard_link . '

' . - '
' . '
'; if ($spbc->data['display_scanner_warnings']['analysis'] && !$spbc->data['wl_mode_enabled']) { $analysis_log_description .= spbc__get_accordion_tab_info_block_html('analysis'); @@ -3598,21 +3671,23 @@ function spbc_list_table__get_args_by_type($table_type) 'pscan_processing_status', 'fast_hash', 'pscan_status', - 'pscan_pending_queue' + 'pscan_pending_queue', + 'pscan_estimated_execution_time' ), 'where' => spbc_get_sql_where_addiction_for_table_of_category('analysis_log'), ), 'order_by' => array('pscan_status' => 'desc'), - 'sortable' => array('path', 'last_sent', 'pscan_status'), + 'sortable' => array('path', 'last_sent', 'pscan_status', 'pscan_estimated_execution_time'), ) ); $args['columns'] = array( 'cb' => array('heading' => '', 'class' => 'check-column', 'width_percent' => 2), - 'path' => array('heading' => 'Path', 'primary' => true, 'width_percent' => 38), + 'path' => array('heading' => 'Path', 'primary' => true, 'width_percent' => 28), 'detected_at' => array('heading' => 'Detected at', 'width_percent' => 15), 'last_sent' => array('heading' => 'Sent for analysis at', 'width_percent' => 15), 'pscan_status' => array('heading' => 'Cloud verdict', 'width_percent' => 10), + 'pscan_estimated_execution_time' => array('heading' => 'Estimated time', 'width_percent' => 10), //'analysis_comment' => array('heading' => 'Comment', 'width_percent' => 20), ); diff --git a/lib/CleantalkSP/SpbctWP/DB/SQLSchema.php b/lib/CleantalkSP/SpbctWP/DB/SQLSchema.php index 5a4bfae44..a00a17040 100644 --- a/lib/CleantalkSP/SpbctWP/DB/SQLSchema.php +++ b/lib/CleantalkSP/SpbctWP/DB/SQLSchema.php @@ -111,6 +111,7 @@ class SQLSchema extends \CleantalkSP\Common\DB\SQLSchema array('field' => 'pscan_pending_queue', 'type' => 'SMALLINT', 'null' => 'yes', 'default' => 'NULL',), array('field' => 'pscan_balls', 'type' => 'varchar(1024)', 'null' => 'yes', 'default' => 'NULL',), array('field' => 'pscan_file_id', 'type' => 'varchar(32)', 'null' => 'yes', 'default' => 'NULL',), + array('field' => 'pscan_estimated_execution_time', 'type' => 'int', 'null' => 'yes', 'default' => 'NULL',), ), 'indexes' => array( diff --git a/lib/CleantalkSP/SpbctWP/Scanner/ScannerQueue.php b/lib/CleantalkSP/SpbctWP/Scanner/ScannerQueue.php index a0c597362..a2f1ed612 100755 --- a/lib/CleantalkSP/SpbctWP/Scanner/ScannerQueue.php +++ b/lib/CleantalkSP/SpbctWP/Scanner/ScannerQueue.php @@ -1627,8 +1627,8 @@ public function schedule_send_heuristic_suspicious_files() // phpcs:ignore PSR1. \CleantalkSP\SpbctWP\Cron::updateTask( 'scanner_resend_pscan_files', 'spbc_scanner_resend_pscan_files', - SPBC_PSCAN_UPDATE_FILES_STATUS_PERIOD, - time() + SPBC_PSCAN_UPDATE_FILES_STATUS_PERIOD, + SPBC_PSCAN_RESEND_FILES_STATUS_PERIOD, + time() + SPBC_PSCAN_RESEND_FILES_STATUS_PERIOD, array('do_rescan' => false) ); \CleantalkSP\SpbctWP\Cron::updateTask( diff --git a/security-malware-firewall.php b/security-malware-firewall.php index a982016b8..d75f2d0d5 100644 --- a/security-malware-firewall.php +++ b/security-malware-firewall.php @@ -132,7 +132,8 @@ define('SPBC_SCAN_LINKS_AMOUNT', 10); // Links scan amount for 1 iteration define('SPBC_SCAN_FRONTEND_AMOUNT', 10); // Links scan amount for 1 iteration define('SPBC_SCAN_LINKS_PERIOD', 30); // Links scan call period -define('SPBC_PSCAN_UPDATE_FILES_STATUS_PERIOD', 300); // Check cloud analysis files status period +define('SPBC_PSCAN_UPDATE_FILES_STATUS_PERIOD', 60); // Check cloud analysis files status period +define('SPBC_PSCAN_RESEND_FILES_STATUS_PERIOD', 300); // Resend files // brief data limits define('SPBC_BRIEF_DATA_DAYS_LIMIT', 7); // how many days will be logs looked for From 991aeb94f2d916e253f2fc4eb318c435f297c13c Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 15 Apr 2024 21:28:18 +0700 Subject: [PATCH 2/9] Fix. Date format fixed. --- inc/spbc-settings.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inc/spbc-settings.php b/inc/spbc-settings.php index 31977234f..daddc68e4 100644 --- a/inc/spbc-settings.php +++ b/inc/spbc-settings.php @@ -3094,7 +3094,7 @@ function spbc__analysyis_files_stats__get_html() $data = spbc__analysyis_files_stats__get_data(); $last_updated_chunk = __('Files statuses updates every', 'security-malware-firewall') . ' ' . SPBC_PSCAN_UPDATE_FILES_STATUS_PERIOD . ' seconds'; $last_updated_chunk .= $data['last_updated'] && is_int($data['last_updated']) - ? ', ' . __('last update time', 'security-malware-firewall') . ': ' . date("Y-m-d H:i:s", $data['last_updated']) + ? ', ' . __('last update time', 'security-malware-firewall') . ': ' . date("M d Y H:i:s", $data['last_updated']) : '.'; $out = sprintf( $out, From d2d70b154edb49e61bcfa3a11ca5fa83b51945d4 Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 15 Apr 2024 21:37:51 +0700 Subject: [PATCH 3/9] New. Update front estimates time --- js/spbc-table.min.js | 2 +- js/spbc-table.min.js.map | 2 +- js/src/spbc-table.js | 47 +++++++++++++++++++++++++++++++++++ security-malware-firewall.php | 2 +- 4 files changed, 50 insertions(+), 3 deletions(-) diff --git a/js/spbc-table.min.js b/js/spbc-table.min.js index 9cf4650fd..446c0cf04 100644 --- a/js/spbc-table.min.js +++ b/js/spbc-table.min.js @@ -1,2 +1,2 @@ -function spbc_reload_accordion(){spbc_sendAJAXRequest({action:"spbc_scanner_tab__reload_accordion"},{notJson:!0,callback:function(t,e,a,n){jQuery(n).accordion("destroy"),jQuery(n).html(t),jQuery(n).accordion({header:"h3",heightStyle:"content",collapsible:!0,active:!1}),spbc_tbl__bulk_actions__listen(),spbc_tbl__row_actions__listen(),spbc_tbl__pagination__listen(),spbc_tbl__sort__listen(),spbcStartShowHide()}},jQuery("#spbc_scan_accordion"))}function spbc_tbl__bulk_actions__listen(){jQuery(".tbl-bulk_actions-all--apply").off("click").on("click",function(){if(spbc_scanner.active)alert(spbcTableLocalize.scannerIsActive);else{var t=jQuery(this),a=t.siblings("select").children()[t.siblings("select").first()[0].selectedIndex].value;if(-1!==["approve","disapprove","send","check_analysis_status","approve_page","disapprove_page","cure","delete_from_analysis_log"].indexOf(a)){if(confirm(spbcTable.warning_bulk)){var e={action:"spbc_tbl-action--bulk",add_action:a,status:t.parents(".tbl-root").attr("type")};if("cure"===a){var n=t.closest("#spbc_tbl__scanner_cure_log").find(".cb-select");let a=[];n.each(function(t,e){e=jQuery(e).val();a.push(e)}),e.selectedIds=a}if("delete_from_analysis_log"===a){n=t.closest("#spbc_tbl__scanner_analysis_log").find(".cb-select");let a=[];n.each(function(t,e){e=jQuery(e).val();a.push(e)}),e.file_ids=a}n={button:this,spinner:t.children(".tbl-preloader--small"),callback:function(t){var e;spbc_reload_accordion(),"check_analysis_status"===a&&void 0!==t&&(e="

Analysis status updated. Total: "+t.counters.total+", updated: "+t.counters.updated+", skipped: "+t.counters.skipped+", failed: "+t.counters.failed+", queued: "+t.counters.queued+"

",spbcModal.open().put(e)),"send"===a&&void 0!==t&&(e="

Files have been sent: "+t.files_sent_counter+"

",spbcModal.open().put(e))},errorOutput:function(t,e){e&&(t=t+"
Additional information:
"+e),spbcModal.open().putError(t)},timeout:6e4};spbc_sendAJAXRequest(e,n)}}else alert("This action is not supported for all files yet =(")}}),jQuery(".tbl-bulk_actions--apply").off("click").on("click",function(t){if(spbc_bulk_action||confirm(spbcTable.warning_bulk)){var a=spbc_bulk_action||jQuery(this),n=(spbc_bulk_action=a).siblings("select").children()[a.siblings("select").first()[0].selectedIndex].value;if("cure"===n)spbc_scanner__cure_selected(jQuery(this));else if("delete_from_analysis_log"===n)spbc_scanner_analysis_log_delete_from_log(jQuery(this));else if("allow"===n||"ban"===n){let e="ban"===n?"deny":n;var s=a.parents(".tbl-root").find(".cb-select:checked"),s=(s&&(a.children(".tbl-preloader--small").show(),s.each(function(){var t=jQuery(this).parents("tr").find(".tbl-row_action--"+n).data("ip");spbc_sec_logs__filter_ip(t,e,!0),jQuery(this).prop("checked",!1)})),a.children(".tbl-preloader--small").hide(),"allow"===n?"Allowed":"Banned"),s="

Success. Selected IPs have been "+s+". Changes will be applied within 10 minutes.

";spbcModal.open().put(s),void setTimeout(spbc_reload_accordion,1900)}else"-1"!==n&&a.parents(".tbl-root").find(".cb-select").is(":checked")?(a.children(".tbl-preloader--small").show(),a.parents(".tbl-root").find(".cb-select:checked").first().parents("tr").find(".tbl-row_action--"+n)[0]?(a.parents(".tbl-root").find(".cb-select:checked").first().parents("tr").find(".tbl-row_action--"+n).click(),a.parents(".tbl-root").find(".cb-select:checked").first().prop("checked",!1)):(a.parents(".tbl-root").find(".cb-select:checked").first().prop("checked",!1),a.click())):(a.children(".tbl-preloader--small").hide(),spbc_bulk_action=null,"check_analysis_status"!==n&&"disapprove"!==n||spbcModal.open().put("

All available files are updated.

"),"send"===n&&spbcModal.open().put("

All available files are sent.

"),setTimeout(spbc_reload_accordion,1900))}})}function spbc_scanner__cure_selected(e){var t=e.closest("#spbc_tbl__scanner_cure_log").find(".cb-select:checked");let a=[];0===t.length&&alert("Please, select elements."),t.each(function(t,e){e=jQuery(e).val();a.push(e)});t={action:"spbc_cure_selected",security:spbcSettings.ajax_nonce,selectedIds:a};jQuery.ajax({type:"POST",url:spbcSettings.ajaxurl,data:t,beforeSend:function(){e.closest("#spbc_tbl__scanner_cure_log").find(".tbl-button---white_blue .tbl-preloader--in_button").show()},success:function(t){e.closest("#spbc_tbl__scanner_cure_log").find(".tbl-button---white_blue .tbl-preloader--in_button").hide(),t.success?(spbcModal.open().put(t.data),document.addEventListener("spbcModalClosed",function(t){document.location.reload()})):spbcModal.open().putError(t.data)}})}function spbc_tbl__row_actions__listen(){jQuery(".tbl-row_action--ajax").off("click").on("click",function(){var t,e,a,n;console.log("spbc_tbl__row_actions__listen click"),spbc_scanner.active?alert(spbcTableLocalize.scannerIsActive):(e={action:"spbc_tbl-action--row",add_action:(t=jQuery(this)).attr("row-action"),id:t.parents(".row-actions").attr("uid"),cols:t.parents(".row-actions").attr("cols_amount"),page_url:t.parent().attr("uid"),page_id:t.parent().attr("page_id")},a={callback:spbc_tbl__row_actions__callback,errorOutput:function(t){spbcModal.open().putError(t)},spinner:t.parent().siblings(".tbl-preloader--tiny")},!spbc_bulk_action&&(n=spbcTable["warning_"+t.attr("row-action")]||spbcTable.warning_default,"copy_file_info"===e.add_action||"check_analysis_status"===e.add_action||confirm(n))&&spbc_sendAJAXRequest(e,a,t.parents("tr")),spbc_bulk_action&&spbc_sendAJAXRequest(e,a,t.parents("tr")))})}function spbc_tbl__row_actions__callback(t,e,a,n){if(t.color&&n.css({background:t.background,color:t.color}),t.html&&(n.parent().parent().parent().prepend(t.html),setTimeout(function(){n.fadeOut(300)},1500),spbc_bulk_action||setTimeout(spbc_reload_accordion,1900)),"copy_file_info"===e.add_action&&(!0===t.success&&t.data.file_info?window.prompt("Copy the file info below and send it to support@cleantalk.org: ",t.data.file_info):(c=void 0!==t.data.error?t.data.error:"Unknown copy_file_info error",spbcModal.open().putError(c))),t.temp_html){var s=n.html();if(n.html(t.temp_html),void 0!==t.updated_template){var c=t.updated_template_type,t=t.updated_template,l=jQuery(t).find(".wp-list-table tbody tr").length,i=jQuery('[aria-controls="spbc_scan_accordion_tab_'+c+'"]');if(0a&&jQuery(e).css("outline","1px solid green")}),i.click()}else"analysis_log"===c&&(o='",c='