2.2.2 (#132)

* Fix text

* Fix false positives

* Add git URL tests

* Fix path

* Detect certificates

* Rollback cert detection

* Exclude known keys

* Improve AWS account detection

* Add Elixir config detection

* Improve secrets rule

* Add Elixir tests

* Update fixture test

* Fix config exclusions for relative paths

* Improve AWS account rule

* Bump version

* Improve find_line_number test coverage

tests/fixtures/aws.json

@@ -1,13 +1,24 @@
 {
-  "compliant": {
-    "aws_id": "{{ AWS ID }}",
-    "aws_key": "${AWS_KEY}",
-    "commit_id": "912ec803b2ce49e4a541068d495ab57000000000",
-    "role": "arn:aws:iam::123456789000:role/role-name"
-  },
-  "noncompliant": {
-    "aws_id": "AKIAHI38FAKE1IWUQEEN",
-    "aws_key": "PA3XsxZ8d8cPQLmnZzFAKEdzC6ND2a8vhbyXU/Dw",
-    "aws_token": "FakeYXdzELv//////////wEldj3948yOJRO84jgpoip239232hEOHhfkjhefkwue97jorhfiuh+XjFC9Je/YG7JCqKjrspab2lB+7/Fb1NJFjgwur47Dbhs/L7nh+/VGnwLoAo8CIqoPBLRmXItaoiuuofZnr+ktihZk1Yi55sYZ12hfRMPVbDmhf9Ke683+e9bJirhUEghw9424JOhgwrgqq99MvzCEFe4eXPOSgAcQcD2xqnnKO738tjhoh23HFqjflhefibWegfqefgqUF12hvgfwegqf"
-  }   
+  "compliant": [
+    {
+      "aws_id": "{{ AWS ID }}",
+      "aws_key": "${AWS_KEY}",
+      "commit_id": "912ec803b2ce49e4a541068d495ab57000000000",
+      "role": "arn:aws:iam::123456789000:role/role-name"
+    },
+    {
+      "aws_account01": "000000000000",
+      "aws_account02": "111111111111"
+    }
+  ],
+  "noncompliant": [
+    {
+      "aws_id": "AKIAHI38FAKE1IWUQEEN",
+      "aws_key": "PA3XsxZ8d8cPQLmnZzFAKEdzC6ND2a8vhbyXU/Dw",
+      "aws_token": "FakeYXdzELv//////////wEldj3948yOJRO84jgpoip239232hEOHhfkjhefkwue97jorhfiuh+XjFC9Je/YG7JCqKjrspab2lB+7/Fb1NJFjgwur47Dbhs/L7nh+/VGnwLoAo8CIqoPBLRmXItaoiuuofZnr+ktihZk1Yi55sYZ12hfRMPVbDmhf9Ke683+e9bJirhUEghw9424JOhgwrgqq99MvzCEFe4eXPOSgAcQcD2xqnnKO738tjhoh23HFqjflhefibWegfqefgqUF12hvgfwegqf"
+    },
+    {
+      "aws_account01": "123456789123"
+    }
+  ]
 }

tests/fixtures/aws.xml

@@ -5,10 +5,13 @@
     <aws_key>${AWS_KEY}</aws_key>
     <commit_id>912ec803b2ce49e4a541068d495ab57000000000</commit_id>
     <role>arn:aws:iam::123456789000:role/role-name</role>
+    <aws_account>000000000000</aws_account>
+    <aws_account>111111111111</aws_account>
   </compliant>
   <noncompliant>
     <aws_id>AKIAHI38FAKE1IWUQEEN</aws_id>
     <aws_key>PA3XsxZ8d8cPQLmnZzFAKEdzC6ND2a8vhbyXU/Dw</aws_key>
     <aws_token>FakeYXdzELv//////////wEldj3948yOJRO84jgpoip239232hEOHhfkjhefkwue97jorhfiuh+XjFC9Je/YG7JCqKjrspab2lB+7/Fb1NJFjgwur47Dbhs/L7nh+/VGnwLoAo8CIqoPBLRmXItaoiuuofZnr+ktihZk1Yi55sYZ12hfRMPVbDmhf9Ke683+e9bJirhUEghw9424JOhgwrgqq99MvzCEFe4eXPOSgAcQcD2xqnnKO738tjhoh23HFqjflhefibWegfqefgqUF12hvgfwegqf</aws_token>
+    <aws_account>123456789123</aws_account>
   </noncompliant>
 </tests>

tests/fixtures/aws.yml

@@ -1,10 +1,13 @@
-# Compliant
-aws_id: "{{ AWS ID }}"
-aws_key: "${AWS_KEY}"
-commit_id: 912ec803b2ce49e4a541068d495ab57000000000
-role: arn:aws:iam::123456789000:role/role-name
+compliant:
+  aws_id: "{{ AWS ID }}"
+  aws_key: "${AWS_KEY}"
+  commit_id: 912ec803b2ce49e4a541068d495ab57000000000
+  role: arn:aws:iam::123456789000:role/role-name
+  aws_account01: '000000000000'
+  aws_account02: '111111111111'
 
-# Noncompliant
-aws_id: AKIAHI38FAKE1IWUQEEN
-aws_key: PA3XsxZ8d8cPQLmnZzFAKEdzC6ND2a8vhbyXU/Dw
-aws_token: FakeYXdzELv//////////wEldj3948yOJRO84jgpoip239232hEOHhfkjhefkwue97jorhfiuh+XjFC9Je/YG7JCqKjrspab2lB+7/Fb1NJFjgwur47Dbhs/L7nh+/VGnwLoAo8CIqoPBLRmXItaoiuuofZnr+ktihZk1Yi55sYZ12hfRMPVbDmhf9Ke683+e9bJirhUEghw9424JOhgwrgqq99MvzCEFe4eXPOSgAcQcD2xqnnKO738tjhoh23HFqjflhefibWegfqefgqUF12hvgfwegqf
+noncompliant:
+  aws_id: AKIAHI38FAKE1IWUQEEN
+  aws_key: PA3XsxZ8d8cPQLmnZzFAKEdzC6ND2a8vhbyXU/Dw
+  aws_token: FakeYXdzELv//////////wEldj3948yOJRO84jgpoip239232hEOHhfkjhefkwue97jorhfiuh+XjFC9Je/YG7JCqKjrspab2lB+7/Fb1NJFjgwur47Dbhs/L7nh+/VGnwLoAo8CIqoPBLRmXItaoiuuofZnr+ktihZk1Yi55sYZ12hfRMPVbDmhf9Ke683+e9bJirhUEghw9424JOhgwrgqq99MvzCEFe4eXPOSgAcQcD2xqnnKO738tjhoh23HFqjflhefibWegfqefgqUF12hvgfwegqf
+  aws_account: '123456789123'

tests/fixtures/falsepositive/deps.txt

@@ -0,0 +1,3 @@
+------------------------------------------------------------
+this is not a privatekey
+--------------

tests/fixtures/falsepositive/keys.yml

@@ -0,0 +1,9 @@
+compliant:
+  public_key: h4rdc0dedOkay01
+  ProjectKey: h4rdc0dedOkay02
+  translationKey: h4rdc0dedOkay03
+  rowIDkey: h4rdc0dedOkay04
+  colKey: h4rdc0dedOkay05
+  columnkey: h4rdc0dedOkay06
+  uniq_key: h4rdc0dedOkay07
+  uniqueKey: h4rdc0dedOkay08

tests/fixtures/falsepositive/semver.json

@@ -0,0 +1,3 @@
+{
+    "getpass": "^1.2.3"
+}

tests/fixtures/runtime.exs

@@ -0,0 +1,12 @@
+use Mix.Config
+
+config :pxblog, Pxblog.Endpoint,
+  secret_key_base: "HardCodedaGFyZGNvZGVkVG9rZW4xMjM0Cg+/aGFyZGNvZGVkVG9rZW4xMjM0Cg"
+
+# Configure your database
+config :pxblog, Pxblog.Repo,
+  adapter: Ecto.Adapters.Postgres,
+  username: "pxblog",
+  password: "h4rdc0d3dp@$$w0rd",
+  database: "pxblog_prod",
+  pool_size: 20

tests/fixtures/uri.yml

@@ -8,4 +8,5 @@ compliant:
 noncompliant:
   - "jdbc:mysql://localhost/authority?user=admin&userpass=hardcoded0"
   - "https://admin:hardcoded1@localhost:8000/admin"
-  - 'amqp://root:[email protected]:5434/topic'
+  - "amqp://root:[email protected]:5434/topic"
+  - "git+https://token:[email protected]/org/repo.git"

tests/integration/test_whispers.py

@@ -9,7 +9,7 @@
     [
         (f"-c {config_path('integration.yml')} {fixture_path()}", 5),
         (f"-r apikey-known {fixture_path('apikeys-known.yml')}", 56),
-        (f"--rules file-known {fixture_path('files')}", 3),
+        (f"-F None --rules file-known {fixture_path('files')}", 3),
         (f"-s Critical {fixture_path('aws.yml')}", 3),
     ],
 )

tests/unit/core/test_scope.py

@@ -15,7 +15,8 @@
     ],
 )
 def test_load_scope(src, expected):
-    args = parse_args([src])
+    argv = ["-F", "None", src]
+    args = parse_args(argv)
     config = load_config(args)
     scope = load_scope(args, config)
     assert len(list(scope)) == expected

tests/unit/core/test_secrets.py

@@ -61,8 +61,11 @@ def test_detect_secrets_by_key(src, expected):
         ("arn.yml", "Low", 4),
         ("arn.xml", "Low", 3),
         ("aws.yml", "Critical", 3),
+        ("aws.yml", DEFAULT_SEVERITY, 7),
         ("aws.json", "Critical", 3),
+        ("aws.json", DEFAULT_SEVERITY, 7),
         ("aws.xml", "Critical", 3),
+        ("aws.xml", DEFAULT_SEVERITY, 7),
         ("beans.xml", "High", 1),
         ("beans.xml.dist", "High", 1),
         ("beans.xml.template", "High", 1),
@@ -81,7 +84,6 @@ def test_detect_secrets_by_key(src, expected):
         ("excluded.json", "Critical", 0),
         ("excluded.xml", "Critical", 0),
         ("excluded.yml", "Critical", 0),
-        ("falsepositive.yml", DEFAULT_SEVERITY, 4),
         ("Groups.xml", "High", 2),
         ("hardcoded.json", "High", 5),
         ("hardcoded.xml", "High", 5),
@@ -126,8 +128,11 @@ def test_detect_secrets_by_key(src, expected):
         ("settings02.ini", "High", 1),
         ("severity.yml", "Critical", 1),
         ("sops.yml", DEFAULT_SEVERITY, 1),
-        ("uri.yml", "High", 3),
+        ("uri.yml", "High", 4),
         ("webhooks.yml", "Low", 6),
+        ("falsepositive/values.yml", DEFAULT_SEVERITY, 4),
+        ("falsepositive/plain.txt", DEFAULT_SEVERITY, 0),
+        ("falsepositive/semver.json", DEFAULT_SEVERITY, 0),
     ],
 )
 def test_detect_secrets_by_value(src, severity, expected):

tests/unit/core/test_utils.py

@@ -16,6 +16,7 @@
     is_iac,
     is_luhn,
     is_path,
+    is_semver,
     is_static,
     is_uri,
     list_rule_prop,
@@ -104,6 +105,7 @@ def test_similar_strings(str1, str2, expected):
         ("apikeys.yml", "GITHUBKEY", "YXNkZmZmZmZm_HARDcoded", 19),
         ("pip.conf", "username", "hardcoded1", 7),
         ("java.properties", "sonar.jdbc.password", "hardcoded02", 10),
+        ("404", "password", "hardcoded", 0),
     ],
 )
 def test_find_line_number_single(src, key, value, expected):
@@ -304,6 +306,22 @@ def test_is_luhn(data, expected):
     assert is_luhn(data) == expected
 
 
+@pytest.mark.parametrize(
+    ("data", "expected"),
+    [
+        ("a.a.a", False),
+        ("1,2,3", False),
+        ("1.2.3", True),
+        ("^1.2.3", True),
+        ("v1.2.3", True),
+        ("==1.2", True),
+        (">=1.2", True),
+    ],
+)
+def test_is_semver(data, expected):
+    assert is_semver(data) == expected
+
+
 def test_default_rules():
     rules = default_rules()
     rule_files = Path("whispers/rules").glob("*.yml")

tests/unit/models/test_pair.py

@@ -1,15 +1,24 @@
 from whispers.models.pair import KeyValuePair
 
 
-def test_pair():
-    pair = KeyValuePair("key", "value", ["key", "path"], "file", 123, {"rule_id": "test"})
+def test_pair(rule_fixture):
+    pair = KeyValuePair("key", "value", ["key", "path"], "file", 123, rule_fixture)
     assert pair.__dict__ == {
         "key": "key",
         "value": "value",
         "keypath": ["key", "path"],
         "file": "file",
         "line": 123,
-        "rule": {"rule_id": "test"},
+        "rule": rule_fixture,
+    }
+    assert pair.to_json() == {
+        "key": "key",
+        "value": "value",
+        "file": "file",
+        "line": 123,
+        "rule_id": "fixture",
+        "message": "test",
+        "severity": "Info",
     }
 
 

tests/unit/test_main.py

@@ -11,6 +11,7 @@ def test_main():
 
 
 def test_run():
-    args = parse_args([fixture_path()])
+    argv = ["-F", "None", fixture_path()]
+    args = parse_args(argv)
     secrets = list(run(args))
-    assert len(secrets) == 318
+    assert len(secrets) == 325

whispers/__version__.py

@@ -1,4 +1,4 @@
-VERSION = (2, 2, 1)
+VERSION = (2, 2, 2)
 
 __version__ = ".".join(map(str, VERSION))
 

whispers/config.yml

@@ -4,13 +4,13 @@ include:
 
 exclude:
   files:
-    - __pycache__|\.eggs|build|dev|\.vscode|\.git
-    - .*/(locale|spec|test|mock)s?/
-    - integration|node_modules
-    - (package(-lock)?|npm-shrinkwrap)\.json
+    - .*(__pycache__|\.eggs|build|dev|\.vscode|\.git)
+    - .*(locale|spec|test|mock|dummy|fixture)s?
+    - .*(integration|node_modules)
+    - .*(package(-lock)?|npm-shrinkwrap)\.json
 
   keys:
-    - .*(public|project).*
+    - .*(public|project|translation|row|col(umn)?|uniq(ue)?).*
 
   values:
     - ^(true|false|yes|no|1|0)$

whispers/core/constants.py

@@ -1,14 +1,15 @@
-import re
 from pathlib import Path
+from re import IGNORECASE, compile
 
 DEFAULT_PATH = Path(__file__).parents[1]
 
 DEFAULT_SEVERITY = ["Critical", "High", "Medium", "Low", "Info"]
 
 ESCAPED_CHARS = str.maketrans({"'": r"\'", '"': r"\""})
 
-REGEX_URI = re.compile(r"[:\w\d]+://.+", flags=re.IGNORECASE)
-REGEX_PATH = re.compile(r"^((([A-Z]|file|root):)?(\.+)?[/\\]+).*$", flags=re.IGNORECASE)
-REGEX_IAC = re.compile(r"\![A-Za-z]+ .+", flags=re.IGNORECASE)
-REGEX_PRIVKEY_FILE = re.compile(r"(rsa|dsa|ed25519|ecdsa|pem|crt|cer|ca-bundle|p7b|p7c|p7s|ppk|pkcs12|pfx|p12)")
-REGEX_ENVVAR = re.compile(r"^\$\$?\{?[A-Z0-9_]+\}?$")
+REGEX_URI = compile(r"[:\w\d]+://.+", flags=IGNORECASE)
+REGEX_PATH = compile(r"^((([A-Z]|file|root):)?(\.+)?[/\\]+).*$", flags=IGNORECASE)
+REGEX_IAC = compile(r"\![A-Za-z]+ .+", flags=IGNORECASE)
+REGEX_PRIVKEY_FILE = compile(r"(rsa|dsa|ed25519|ecdsa|pem|crt|cer|ca-bundle|p7b|p7c|p7s|ppk|pkcs12|pfx|p12)")
+REGEX_ENVVAR = compile(r"^\$\$?\{?[A-Z0-9_]+\}?$")
+REGEX_SEMVER = compile(r"^[\^~\-=vV<>]{0,3}([0-9]+\.){1,2}[0-9]+(\-.*)?$")

whispers/core/pairs.py

@@ -9,6 +9,7 @@
 from whispers.plugins.config import Config
 from whispers.plugins.dockercfg import Dockercfg
 from whispers.plugins.dockerfile import Dockerfile
+from whispers.plugins.elixir import Elixir
 from whispers.plugins.gradle import Gradle
 from whispers.plugins.html import Html
 from whispers.plugins.htpasswd import Htpasswd
@@ -157,6 +158,9 @@ def load_plugin(file: Path) -> Optional[object]:
     elif filetype in ["py", "py3", "py35", "py36", "py37", "py38", "py39"]:
         return Python
 
+    elif filetype == "exs":
+        return Elixir
+
     elif REGEX_PRIVKEY_FILE.match(filetype):
         return Plaintext
 

whispers/core/utils.py

@@ -9,7 +9,7 @@
 from luhn import verify as luhn_verify
 from yaml import safe_load, safe_load_all
 
-from whispers.core.constants import DEFAULT_PATH, REGEX_ENVVAR, REGEX_IAC, REGEX_PATH, REGEX_URI
+from whispers.core.constants import DEFAULT_PATH, REGEX_ENVVAR, REGEX_IAC, REGEX_PATH, REGEX_SEMVER, REGEX_URI
 from whispers.models.pair import KeyValuePair
 
 
@@ -224,6 +224,14 @@ def is_luhn(data: str) -> bool:
     return luhn_verify(str(data))
 
 
+def is_semver(data: str) -> bool:
+    """Checks if given data resembles a SemVer-like format"""
+    if not is_ascii(data):
+        return False
+
+    return bool(REGEX_SEMVER.match(data))
+
+
 def is_similar(key: str, value: str, similarity: float) -> bool:
     """
     Checks similarity between key and value.
@@ -258,7 +266,7 @@ def find_line_number(pair: KeyValuePair) -> int:
                 if not findpath:
                     return foundline
 
-    except Exception:  # pragma: no cover
+    except Exception:
         global_exception_handler(pair.file, "find_line_number()")
 
     return 0

whispers/models/rule.py

@@ -3,7 +3,7 @@
 from dataclasses import dataclass, field
 from typing import Any, Dict, Optional, Pattern
 
-from whispers.core.utils import is_ascii, is_base64, is_base64_bytes, is_luhn, is_similar, is_uri
+from whispers.core.utils import is_ascii, is_base64, is_base64_bytes, is_luhn, is_semver, is_similar, is_uri
 from whispers.models.pair import KeyValuePair
 
 
@@ -52,6 +52,9 @@ def matches(self, target: str) -> bool:
         if self.isLuhn is not None and self.isLuhn is not is_luhn(target):
             return False
 
+        if is_semver(target):
+            return False
+
         return True
 
 

whispers/plugins/elixir.py

@@ -0,0 +1,15 @@
+from pathlib import Path
+from typing import Iterator
+
+from whispers.models.pair import KeyValuePair
+
+
+class Elixir:
+    def pairs(self, filepath: Path) -> Iterator[KeyValuePair]:
+        for lineno, line in enumerate(filepath.open(), 1):
+            for statement in line.split(","):
+                if ": " not in statement:
+                    continue
+
+                key, value = statement.split(": ")
+                yield KeyValuePair(key, value, line=lineno)

whispers/plugins/json.py

@@ -11,7 +11,7 @@
 class Json(StructuredDocument):
     def pairs(self, filepath: Path) -> Iterator[KeyValuePair]:
         """
-        Try to load as as. Otherwise, try to
+        Try to load JSON as is. Otherwise, try as a custom format.
         """
         try:
             document = json.load(filepath.open())