Merge pull request #18 from Martlark/1.1.5

1.1.5

README.rst

@@ -68,11 +68,23 @@ These environment variables will override options from the initialisation.
 
 
 Methods
--------
+=======
+
+init_app(app)
+-------------
+
+Initialise and start ip_ban with the given Flask application.
+
+block(ip_address, permanent=False)
+----------------------------------
+
+Block the specific address, optionally forever
+
+
+add(ip=None, url=None)
+----------------------
 
--  ``init_app(app)`` - Initialise and start ip_ban with the given Flask application.
--  ``block(ip_address, permanent=False)`` - block the specific address, optionally forever
--  ``add(ip=None, url=None)`` - increase the observations for the current request ip or given ip address
+increase the observations for the current request ip or given ip address
 
 Example for add:
 
@@ -93,8 +105,14 @@ Example for add:
             if request.arg.get('password') != 'secret':
                 ip_ban.add()
 
--  ``remove(ip_address)`` - remove the given ip address from the ban list.  Returns true if ban removed.
--  ``url_pattern_add('reg-ex-pattern', match_type='regex')`` - exclude any url matching the pattern from checking
+remove(ip_address)
+------------------
+Remove the given ip address from the ban list.  Returns true if ban removed.
+
+url_pattern_add('reg-ex-pattern', match_type='regex')
+-----------------------------------------------------
+
+Exclude any url matching the pattern from checking
 
 
 Example of url_pattern_add:
@@ -110,12 +128,31 @@ Example of url_pattern_add:
     ip_ban.url_pattern_add('/flash/dance', match_type='string')
 
 
--  ``url_pattern_remove('reg-ex-pattern')`` - remove pattern from the url whitelist
--  ``url_block_pattern_add('reg-ex-pattern', match_type='regex')`` - add any url matching the pattern to the block list. match_type can be 'string' or 'regex'.  String is direct match.  Regex is a regex pattern.
--  ``url_block_pattern_remove('reg-ex-pattern')`` - remove pattern from the url block list
--  ``ip_whitelist_add('ip-address')`` - exclude the given ip from checking
--  ``ip_whitelist_remove('ip-address')`` - remove the given ip from the ip whitelist
+url_pattern_remove('reg-ex-pattern')
+------------------------------------
+
+Remove pattern from the url whitelist
+
+
+url_block_pattern_add('reg-ex-pattern', match_type='regex')
+-----------------------------------------------------------
+
+Add any url matching the pattern to the block list. match_type can be 'string' or 'regex'.  String is direct match.  Regex is a regex pattern.
+
+url_block_pattern_remove('reg-ex-pattern')
+------------------------------------------
+
+Remove pattern from the url block list
 
+ip_whitelist_add('ip-address')
+------------------------------
+
+Exclude the given ip from checking
+
+ip_whitelist_remove('ip-address')
+---------------------------------
+
+Remove the given ip from the ip whitelist
 
 Example of ip_whitelist_add
 
@@ -129,7 +166,9 @@ Example of ip_whitelist_add
     ip_ban.ip_whitelist_add('127.0.0.1')
 
 
--  ``load_nuisances(file_name=None)`` - add a list of nuisances to url pattern block list from a file.  See below for more information.
+load_nuisances(file_name=None)
+------------------------------
+Add a list of nuisances to url pattern block list from a file.  See below for more information.
 
 Example:
 
@@ -140,9 +179,12 @@ Example:
     ip_ban.init_app(app)
     ip_ban.load_nuisances()
 
--  ``load_allowed(file_name=None)`` - add a list of allowed patterns from a file.  See nuisance for format details.
-    By default `allowed.yaml` in the ip_ban folder is used.  To add to the default patterns supply your own file.
-    Must be a yaml file with the following example format (which are also the default patterns):
+load_allowed(file_name=None)
+----------------------------
+
+Add a list of allowed patterns from a file.  See nuisance for format details.
+By default `allowed.yaml` in the ip_ban folder is used.  To add to the default patterns supply your own file.
+Must be a yaml file with the following example format (which are also the default patterns):
 
 
 .. code:: yaml
@@ -165,13 +207,38 @@ Example:
     ip_ban.load_allowed()
 
 
+get_block_list()
+----------------
+
+return a copy of the internal block list.  Usually will be a dict with the key of `ip` and have
+dict values of `count`, `permanent`, `url` and `timestamp`.
+
+    - timestamp:  datetime object
+    - count: number of offences
+    - url: offending url requested
+    - permanent: bool if ban is permanent
+
+Example:
+
+.. code:: python
+
+    s = ''
+    s += '<table class="table"><thead>\n'
+    s += '<tr><th>ip</th><th>count</th><th>permanent</th><th>url</th><th>timestamp</th></tr>\n'
+    s += '</thead><tbody>\n'
+    for k, r in ip_ban.get_block_list().items():
+        s += '<tr><td>{}</td><td>{}</td><td>{}</td><td>{}</td><td>{}</td></tr>\n'.format(k, r['count'],
+                                                                                         r.get('permanent', ''),
+                                                                                         r.get('url', ''),
+                                                                                         r['timestamp'])
+
 Url patterns
-------------
+============
 
 Url matching match_type can be 'string' or 'regex'.  String is direct match.  Regex is a regex pattern.
 
 Block networks / cidr
----------------------
+=====================
 
 Use the `block_cidr(network)` method to block a range of addresses or whole regions.
 
@@ -187,7 +254,7 @@ Example:
 
 
 Nuisance file
--------------
+=============
 
 ip_ban includes a file of common web nuisances that should not be allowed on a flask site.  It includes:
 
@@ -204,7 +271,7 @@ You can add your own nuisance yaml file by calling with the parameter `file_name
 See the nuisance.yaml file in the source for formatting and details.
 
 IPC and persistence
--------------------
+===================
 
 When you have multiple applications or processes serving a web application it can be handy to share
 any abuse ip between processes.  The ipc option allows this.
@@ -224,12 +291,12 @@ of the Flask app. It only updates every 5 seconds at the most. If the app does n
 request handling between bans then that ban record won't be shared between processes.
 
 IP Header
----------
+=========
+
 When running a flask app in a docker hosted environment (or similar) the ip address will be the virtual
 adapter ip and won't change for differing requests.  Use your proxy server to set the real IP address in a header
 so that ip-ban can find what it really is.  For apache:
 
-
     ``RequestHeader set X_TRUE_IP "%{REMOTE_ADDR}s"``
 
     ``ProxyPass / http://localhost:8080/``
@@ -239,7 +306,7 @@ so that ip-ban can find what it really is.  For apache:
 Then when initializing ip_ban set the header name using the parameter ``ip_header``, in this example: ip_header='X_TRUE_IP'.
 
 Abuse IPDB
-----------
+==========
 
 see: https://docs.abuseipdb.com/#introduction
 
@@ -257,17 +324,18 @@ abuse_IPDB_config = {key=, report=False, load=False, debug=False}
 
 
 Release History
----------------
+===============
 
 * 1.0.13 - Remove reason= which did nothing.  Add url to report table.  Add more nuisances.  Add release history.
 * 1.1.0 - Add more nuisances.  Add ability to block regions by using `block_cidr()`.  Remove support for obsolete Python releases (2.7,3.4,3.5).
 * 1.1.1 - Fix doco typo.
 * 1.1.2 - allow ip as list for ip_whitelist_add()/ip_whitelist_remove().
 * 1.1.3 - Fix documentation errors.  Add wellknown.yaml and default web URLs commonly used by bots.  Remove raise exception for ip abuse db.
 * 1.1.4 - Fix missing allowed.yaml in MANIFEST.in
+* 1.1.5 - Add new nuisances.  Add more allowed.  Do not repeat report ips to abuse ip. Use utcnow for timestamps.
 
 Licensing
----------
+=========
 
 - Apache 2.0
 

flask_ipban/abuse_ipdb.py

@@ -41,7 +41,7 @@ def __init__(self, logger, ip_ban, key, report=False, load=False, debug=False):
         self.categories = [21]  # Web App Attack
         self.lock_name = 'flask-ip-ban-abuse-ipdb-load'
         self.debug = debug
-        self.reported = {}  # url and ip alread reported
+        self.reported = {}  # url and ip already reported
         if load:
             self.import_black_list()
 
@@ -57,7 +57,7 @@ def report_ip(self, ip, reason, categories=None):
 
         key = ip + '-' + reason
 
-        if self.reported.get(key):
+        if self.reported.get(ip):
             self.logger.info('Already reported {}'.format(key))
             # already reported ip and reason combination
             return 'already'
@@ -80,7 +80,7 @@ def report_ip(self, ip, reason, categories=None):
             response = requests.post(url, data=data, headers=headers).content
             json_response = json.loads(response)
             if json_response.get('data'):
-                self.reported[key] = datetime.utcnow()
+                self.reported[ip] = datetime.utcnow()
                 self.logger.warn('reported ip {} for {}.  ok: {}'.format(ip, reason, json_response.get('data')))
             else:
                 self.logger.error('reported ip {} for {}.  Error: {}'.format(ip, reason, response))

flask_ipban/allowed.yaml

@@ -4,3 +4,5 @@ regex:
   - ^/robots\.txt$
   - ^/ads\.txt$
   - ^/favicon\.ico$
+  - ^/sitemap\.xml
+  - ^/sitemap_index.xml

flask_ipban/ip_ban.py

@@ -36,7 +36,7 @@ class IpBan:
 
     """
 
-    VERSION = '1.1.4'
+    VERSION = '1.1.5'
 
     def __init__(self, app=None, ban_count=20, ban_seconds=3600 * 24, persist=False, record_dir=None, ipc=False,
                  secret_key=None, ip_header=None, abuse_IPDB_config=None):
@@ -131,7 +131,7 @@ def block(self, ip_list, permanent=False, no_write=False, timestamp=None, url='b
             ip_list = [ip_list]
 
         if not timestamp:
-            timestamp = datetime.now()
+            timestamp = datetime.utcnow()
 
         for ip in ip_list:
             entry = self._ip_ban_list.get(ip)
@@ -154,6 +154,9 @@ def block(self, ip_list, permanent=False, no_write=False, timestamp=None, url='b
                 self.ip_record.write(ip, record_type='permanent' if permanent else 'block')
         return len(self._ip_ban_list)
 
+    def get_block_list(self):
+        return self._ip_ban_list.copy()
+
     def get_ip(self):
         """
         return the ip for the current request from flask or from
@@ -214,15 +217,15 @@ def _before_request_check(self):
         entry = self._ip_ban_list.get(ip)
 
         if entry and entry.get('count', 0) > self.ban_count:
-            now = datetime.now()
+            now = datetime.utcnow()
             delta = now - entry.get('timestamp', now)
 
             if entry.get('permanent', False):
                 abort(403)
 
             if delta.seconds < self.ban_seconds or self.ban_seconds == 0:
                 self._logger.debug('IP updated in ban list {}.  Url: {}'.format(ip, url))
-                entry['timestamp'] = datetime.now()
+                entry['timestamp'] = datetime.utcnow()
                 entry['count'] += 1
                 self.ip_record.write(ip, count=entry['count'])
                 abort(403)
@@ -379,10 +382,10 @@ def add(self, ip=None, url=None, no_write=False, timestamp=None):
                     self.abuse_reporter.report_ip(ip, reason='Flask-IPban - exploit URL requested:{}'.format(url))
                 return True
 
-        if timestamp and timestamp > datetime.now():
-            timestamp = datetime.now()
+        if timestamp and timestamp > datetime.utcnow():
+            timestamp = datetime.utcnow()
 
-        timestamp = timestamp or datetime.now()
+        timestamp = timestamp or datetime.utcnow()
 
         if entry:
             entry['count'] += 1

flask_ipban/nuisance.yaml

@@ -57,14 +57,15 @@ regex:
   - .*\.war$
   - .*\.wss$
   # hacking or scanning attempts
+  - ^/api/
   - .*\/login
   - .*/a2billing/.*
   - \/.*\/(calendar|date|fortune|redirect|passwd)$
   - .*?PHPSESSID=
   - .*\/usage_201811$
   - .*\/usage_201811.html$
   - .*\wallet.dat$
-  - .*\/wp-admin\/$
+  - .*\/wp-\w+\/
   - .*netease\.com\:25$
   - \/([sf]|web|ows|mp)cgi(-bin|)\/
   - \/bin\/
@@ -76,11 +77,12 @@ regex:
   - \/solr\/
   - \/HNAP1\/
   - \/WEB-INF\/
-  - \/wp-
+  - ^/wp-
   - \/web.config.txt$
   - \+CSCOE\+/
   - ^/joomla/
   - ^/\.git
+  - ^/_ignition
 
 string:
   # all of these are hacking attempts by vulnerability scanners
@@ -92,6 +94,7 @@ string:
   - /bea_wls_deployment_internal
   - /HNAP1/
   - /index.php/admin/
+  - /jenkins/login
   - /joomla/
   - /libs/js/iframe.js
   - /manager/html

flask_ipban/test.py

@@ -146,6 +146,14 @@ def testAdd(self):
         response = self.client.get('/')
         self.assertEqual(response.status_code, 403)
 
+    def testGetBlockList(self):
+        block_list = self.ip_ban.get_block_list()
+        self.assertFalse(block_list)
+        for x in range(self.ip_ban.ban_count + 1):
+            self.ip_ban.add(ip=localhost, url='/')
+        block_list = self.ip_ban.get_block_list()
+        self.assertIn(localhost, block_list)
+
     def testKeepOnBlocking(self):
         # block should not timeout if spamming continues
         test_url = '/doesnotexist'

pypar.commands.sh

@@ -20,10 +20,10 @@ pip install twine
 
 python setup.py sdist bdist_wheel
 #
-twine check dist/flask_ipban-1.1.4*
+twine check dist/flask_ipban-1.1.5*
 # test
 # pip install -e .
 # twine upload --repository-url https://test.pypi.org/legacy/ dist/* -u martlark
 # prod pypi
 # add release in git hub to match the version
-twine upload dist/flask_ipban-1.1.4* -u martlark
+twine upload dist/flask_ipban-1.1.5* -u martlark

setup.py

@@ -16,7 +16,7 @@
 
 from setuptools import setup
 
-VERSION = '1.1.4'
+VERSION = '1.1.5'
 LONG_DESCRIPTION = open('README.rst', 'r', encoding='utf-8').read()
 
 setup(