Issue #445 & #434 : Attemping to fix issue with SC_Email_Body_with_Kn…

…own_Phishing_URL taking a long time to load.

app/models/yara_rule.py

@@ -185,7 +185,8 @@ def to_dict(self, include_yara_rule_string=None, short=False, include_relationsh
                 yara_dict["owner_user"] = self.owner_user.to_dict() if self.owner_user else None
 
         if not short:
-            revisions = Yara_rule_history.query.filter_by(yara_rule_id=self.id).all()
+            revision_limit = int(cfg_settings.Cfg_settings.get_setting("FETCH_REVISION_COUNT_LIMIT") or 25)
+            revisions = Yara_rule_history.query.filter_by(yara_rule_id=self.id).order_by(Yara_rule_history.date_created.desc()).limit(revision_limit).all()
             comments = Comments.query.filter_by(entity_id=self.id).filter_by(
                 entity_type=ENTITY_MAPPING["SIGNATURE"]).all()
             files = Files.query.filter_by(entity_id=self.id).filter_by(entity_type=ENTITY_MAPPING["SIGNATURE"]).all()

app/routes/releases.py

@@ -149,26 +149,19 @@ def create_release():
     db.session.commit()
 
     if not release.is_test_release:
-        history = db.session.query(yara_rule.Yara_rule_history.id, yara_rule.Yara_rule_history.yara_rule_id,
-                                   yara_rule.Yara_rule_history.revision).all()
-        history_mapping = {}
-        for h in history:
-            id_, rule_id, revision = h
-            if not rule_id in history_mapping:
-                history_mapping[rule_id] = {}
-            if not revision in history_mapping[rule_id]:
-                history_mapping[rule_id][revision] = id_
-
         release_data = release.release_data_dict
         release_yara_rule_history_list = []
         for sig_id in release_data["Signatures"]["Signatures"].keys():
             revision = release_data["Signatures"]["Signatures"][sig_id]["revision"]
             sig_id = int(sig_id)
             revision = int(revision)
-            if not sig_id in history_mapping:
-                history_mapping[sig_id] = {}
 
-            if not revision in history_mapping[sig_id]:
+            revision_entity = db.session.query(yara_rule.Yara_rule_history) \
+                .filter_by(yara_rule_id=sig_id) \
+                .filter_by(revision=revision) \
+                .first()
+
+            if not revision_entity:
                 yr = db.session.query(yara_rule.Yara_rule).get(sig_id)
                 latest = yara_rule.Yara_rule_history(
                     date_created=datetime.datetime.now(),
@@ -180,10 +173,10 @@ def create_release():
                 )
                 db.session.add(latest)
                 db.session.flush()
-                history_mapping[sig_id][revision] = latest.id
+                revision_entity = latest
 
             release_yara_rule_history_list.append(
-                {"yara_rules_history_id": history_mapping[sig_id][revision], "release_id": release.id})
+                {"yara_rules_history_id": revision_entity.id, "release_id": release.id})
 
         if release_yara_rule_history_list:
             app.logger.debug(

app/static/js/yara_rule/yara_rule-controller.js

@@ -273,7 +273,7 @@ angular.module('ThreatKB')
                         },
                         {
                             name: 'Actions',
-                            width: '160',
+                            width: '180',
                             enableFiltering: false,
                             enableColumnMenu: false,
                             enableSorting: false,
@@ -290,6 +290,12 @@ angular.module('ThreatKB')
                                 + '</small>'
                                 + '</button>'
                                 + '&nbsp;'
+                                + '<button type="button" ng-click="grid.appScope.viewRevision(row.entity.id)"'
+                                + ' class="btn btn-sm">'
+                                + '<small><span class="glyphicon glyphicon-list-alt"></span>'
+                                + '</small>'
+                                + '</button>'
+                                + '&nbsp;'
                                 + '<button type="button" ng-click="grid.appScope.update(row.entity.id)"'
                                 + ' class="btn btn-sm">'
                                 + '<small><span class="glyphicon glyphicon-pencil"></span>'
@@ -416,6 +422,11 @@ angular.module('ThreatKB')
                 $scope.view(id);
             };
 
+            $scope.viewRevision = function (id) {
+                $scope.yara_rule = Yara_rule.resource.get({id: id, include_yara_string: 1});
+                $scope.revision_view(id);
+            };
+
             $scope.activateRule = function (id, name) {
                 Yara_rule.activateRule(id).then(function (success) {
                     growl.info("Successfully activated signature " + name, {ttl: 3000});
@@ -660,6 +671,20 @@ angular.module('ThreatKB')
                 });
             };
 
+            $scope.revision_view = function (id) {
+                const revision_view = $uibModal.open({
+                    templateUrl: 'yara_rule-revision.html',
+                    controller: 'Yara_ruleRevisionViewController',
+                    size: 'lg',
+                    backdrop: 'static',
+                    resolve: {
+                        yara_rule: function () {
+                            return $scope.yara_rule;
+                        }
+                    }
+                });
+            };
+
             if (openModalForId !== null) {
                 if (openModalForId === "add") {
                     $scope.create();
@@ -1115,6 +1140,43 @@ angular.module('ThreatKB')
                 return Tags.loadTags(query);
             };
         }])
+    .controller('Yara_ruleRevisionViewController', ['$scope', '$uibModalInstance', 'yara_rule', '$location', '$window', '$cookies',
+        function ($scope, $uibModalInstance, yara_rule, $location, $window, $cookies) {
+            yara_rule.$promise.then(
+                function (yr) {
+                    $window.document.title = "ThreatKB: " + yr.name;
+                }
+            );
+
+            $scope.selectedRevisions = {
+                main: null,
+                compared: null
+            };
+
+            $scope.edit = function (id) {
+                var location = $location.absUrl();
+                var last_spot = location.split("/")[location.split("/").length - 1];
+                $uibModalInstance.close($scope.yara_rule);
+                if (isNaN(parseInt(last_spot, 10))) {
+                    $window.location.href = $location.absUrl() + "/" + id;
+                    return;
+                } else if (!isNaN(parseInt(last_spot, 10)) && last_spot !== id) {
+                    $window.location.href = $location.absUrl().replace(/\/[0-9]+$/, "/" + id);
+                    return;
+                }
+                $window.location.href = $location.absUrl();
+            };
+
+            $scope.yara_rule = yara_rule;
+
+            $scope.ok = function () {
+                $uibModalInstance.close($scope.yara_rule);
+            };
+
+            $scope.cancel = function () {
+                $uibModalInstance.dismiss('cancel');
+            };
+        }])
     .controller('Yara_revisionController', ['$scope', 'Yara_rule',
         function ($scope, Yara_rule) {
             $scope.revision_diff = null;

app/static/views/yara_rule/yara_rules.html

@@ -132,6 +132,102 @@ <h4 class="modal-title" id="viewSigLabel" style="float: left; margin-right: 10px
             </button>
         </div>
     </script>
+    <script type="text/ng-template" id="yara_rule-revision.html">
+        <a class="btn btn-sm btn-primary modal-fullscreen-button modal-fullscreen-button-abs-pos"
+           ng-click="toggleFullscreen()">Toggle full screen</a>
+        <div class="modal-header">
+            <button type="button" class="close"
+                    ng-click="cancel()">&times;
+            </button>
+            <a class="btn btn-sm  btn-primary" ng-if="yara_rule.id"
+               ng-click="edit(yara_rule.id)">Edit
+            </a>
+            <h4 class="modal-title" id="viewSigLabel" style="float: left; margin-right: 10px;">
+                View Signature Revision(s)
+            </h4>
+        </div>
+        <div class="modal-body">
+            <div ng-if="yara_rule.id && yara_rule.revisions && yara_rule.revisions.length">
+                <style type="text/css">
+                    .yara-revisions {
+                    }
+
+                    .yara-revisions > div.yara-revisions-select {
+                        display: flex;
+                    }
+
+                    .yara-revisions > div.yara-revisions-select > div {
+                        flex: 1;
+                        padding: 4px 8px;
+                    }
+
+                    .yara-revisions > div.yara-revisions-select > div > label {
+                        display: block;
+                    }
+
+                    .yara-revisions > div.yara-revisions-select > div > select {
+                        display: block;
+                        width: 100%;
+                    }
+
+                    .yara-revisions > div.yara-revisions-view {
+                        overflow-y: auto;
+                    }
+
+                    .yara-revisions > div.yara-revisions-view > pre {
+                        height: 320px;
+                    }
+
+                    .yara-revisions > div.yara-revisions-view > pre del {
+                        background: #fdaeb7;
+                    }
+
+                    .yara-revisions > div.yara-revisions-view > pre ins {
+                        background: #cdffd8;
+                    }
+                </style>
+                <label>Revisions ({{ yara_rule.revisions.length + 1 }})</label>
+                <div class="yara-revisions">
+                    <div class="yara-revisions-select">
+                        <div>
+                            <label>View</label>
+                            <select ng-model="selectedRevisions.main">
+                                <option ng-value="null" selected>
+                                    CURRENT @{{ yara_rule.last_revision_date | date:'yyyy-MM-dd' }}
+                                    by {{ yara_rule.modified_user.email }}
+                                </option>
+                                <option ng-repeat="rev in yara_rule.revisions | orderBy:'-date_created'"
+                                        ng-value="rev">
+                                    Rev. {{ rev.revision }} @{{ rev.date_created | date:'yyyy-MM-dd' }}
+                                    by {{ rev.user.email }}
+                                    {{ rev.releases ? ' - Releases: ' + rev.releases   : '' }}
+                                </option>
+                            </select>
+                        </div>
+                        <div>
+                            <label>Diff</label>
+                            <select ng-model="selectedRevisions.compared">
+                                <option ng-value="null"> None</option>
+                                <option ng-repeat="rev in yara_rule.revisions | orderBy:'-date_created'"
+                                        ng-value="rev"
+                                        ng-show="rev.revision < (selectedRevisions.main ? selectedRevisions.main.revision : yara_rule.revision)">
+                                    Rev. {{ rev.revision }} @{{ rev.date_created | date:'yyyy-MM-dd' }}
+                                    by {{ rev.user.email }}
+                                </option>
+                            </select>
+                        </div>
+                    </div>
+                    <div class="yara-revisions-view" ng-controller="Yara_revisionController">
+                        <pre ng-if="revision_diff"><code ng-bind-html="revision_diff"></code></pre>
+                        <pre ng-if="!revision_diff"><code> {{ selectedRevisions.main ? selectedRevisions.main.yara_rule_string : yara_rule.yara_rule_string }} </code></pre>
+                    </div>
+                </div>
+        </div>
+        <div class="modal-footer">
+            <button type="button" class="btn btn-default" ng-click="cancel()">Close
+            </button>
+        </div>
+    </script>
     <script type="text/ng-template" id="yara_rule-save.html">
         <style type="text/css">
 

migrations/versions/fc0cab9d77dc_fetch_revision_count_limit.py

@@ -0,0 +1,50 @@
+"""Fetch Revision Count Limit
+
+Revision ID: fc0cab9d77dc
+Revises: a688cf44cd8a
+Create Date: 2023-02-19 22:45:12.789465
+
+"""
+import datetime
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+from app.models import cfg_settings
+
+revision = 'fc0cab9d77dc'
+down_revision = 'a688cf44cd8a'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    date_created = datetime.datetime.now().isoformat()
+    date_modified = datetime.datetime.now().isoformat()
+
+    op.bulk_insert(
+        cfg_settings.Cfg_settings.__table__,
+        [
+            {
+                "key": "FETCH_REVISION_COUNT_LIMIT",
+                "value": "25",
+                "public": True,
+                "date_created": date_created,
+                "date_modified": date_modified,
+                "description": "The number of revisions to limit on fetch of Yara Rules."
+            }
+        ]
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    keys = ["FETCH_REVISION_COUNT_LIMIT"]
+    for key in keys:
+        op.execute("""DELETE from cfg_settings where `key`='%s';""" % (key))
+
+    # ### end Alembic commands ###