diff --git a/src/detect-engine-build.c b/src/detect-engine-build.c
index 42b439945b80..33b8ca206b22 100644
--- a/src/detect-engine-build.c
+++ b/src/detect-engine-build.c
@@ -1176,8 +1176,10 @@ static DetectPort *RulesGroupByPorts(DetectEngineCtx *de_ctx, uint8_t ipproto, u
         /* IP Only rules are handled separately */
         if (s->type == SIG_TYPE_IPONLY)
             goto next;
+        /* Protocol does not match the Signature protocol and is neither IP or pkthdr */
         if (!(s->proto.proto[ipproto / 8] & (1<<(ipproto % 8)) || (s->proto.flags & DETECT_PROTO_ANY)))
             goto next;
+        /* Direction does not match Signature direction */
         if (direction == SIG_FLAG_TOSERVER) {
             if (!(s->flags & SIG_FLAG_TOSERVER))
                 goto next;
@@ -1186,14 +1188,6 @@ static DetectPort *RulesGroupByPorts(DetectEngineCtx *de_ctx, uint8_t ipproto, u
                 goto next;
         }
 
-        DetectPort *p = NULL;
-        if (direction == SIG_FLAG_TOSERVER)
-            p = s->dp;
-        else if (direction == SIG_FLAG_TOCLIENT)
-            p = s->sp;
-        else
-            BUG_ON(1);
-
         /* see if we want to exclude directionless sigs that really care only for
          * to_server syn scans/floods */
         if ((direction == SIG_FLAG_TOCLIENT) && DetectFlagsSignatureNeedsSynOnlyPackets(s) &&
@@ -1206,6 +1200,14 @@ static DetectPort *RulesGroupByPorts(DetectEngineCtx *de_ctx, uint8_t ipproto, u
             goto next;
         }
 
+        DetectPort *p = NULL;
+        if (direction == SIG_FLAG_TOSERVER)
+            p = s->dp;
+        else if (direction == SIG_FLAG_TOCLIENT)
+            p = s->sp;
+        else
+            BUG_ON(1);
+
         int wl = s->init_data->score;
         while (p) {
             int pwl = PortIsWhitelisted(de_ctx, p, ipproto) ? DETECT_PGSCORE_RULE_PORT_WHITELISTED