Merge pull request #9 from Martlark/1.0.14

1.1.0

.travis.yml

@@ -5,12 +5,6 @@ matrix:
   include:
   - python: 3.7
     env: TOXENV=lint
-  - python: 2.7
-    env: TOXENV=py27
-  - python: 3.4
-    env: TOXENV=py34
-  - python: 3.5
-    env: TOXENV=py35
   - python: 3.6
     env: TOXENV=py36
 

README.rst

@@ -146,6 +146,22 @@ Url patterns
 
 Url matching match_type can be 'string' or 'regex'.  String is direct match.  Regex is a regex pattern.
 
+Block networks / cidr
+---------------------
+
+Use the `block_cidr(network)` method to block a range of addresses or whole regions.
+
+Example:
+
+.. code:: python
+
+    ip_ban = IpBan()
+    app = Flask(__name__)
+    ip_ban.init_app(app)
+    # block a network in Aruba
+    ip_ban.block_cidr('190.220.142.104/29')
+
+
 Nuisance file
 -------------
 
@@ -159,7 +175,7 @@ that use nuisance url patterns they won't result in a block.
 
 Load them by calling ip_ban.load_nuisances()
 
-You can add your own nuisance yaml file by calling with the parameter file_name=.
+You can add your own nuisance yaml file by calling with the parameter `file_name`.
 
 See the nuisance.yaml file in the source for formatting and details.
 
@@ -179,6 +195,10 @@ This folder and secret key is also used by the persistence feature.
 Only ip records using the `block`, `add` and `remove` methods or by 404; are persisted or shared.  Any whitelisting or 
 pattern bans are not persisted/shared and must be done for each instance of your application.
 
+The bit that shares ipc records between processes only updates during the `before_request` handler
+of the Flask app. It only updates every 5 seconds at the most. If the app does no
+request handling between bans then that ban record won't be shared between processes.
+
 IP Header
 ---------
 When running a flask app in a docker hosted environment (or similar) the ip address will be the virtual
@@ -216,6 +236,7 @@ Release History
 ---------------
 
 1.0.13 - Remove reason= which did nothing.  Add url to report table.  Add more nuisances.  Add release history.
+1.1.0 - Add more nuisances.  Add ability to block regions by using `block_cidr()`.  Remove support for obsolete Python releases (2.7,3.4,3.5).
 
 Licensing
 ---------

flask_ipban/ip_ban.py

@@ -11,17 +11,16 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from __future__ import absolute_import
-
+import ipaddress
 import os
 import re
 from datetime import datetime
 
 import yaml
 from flask import request, abort
 
-from flask_ipban.ip_record import IpRecord
 from flask_ipban.abuse_ipdb import AbuseIPDB
+from flask_ipban.ip_record import IpRecord
 
 
 class IpBan:
@@ -37,7 +36,7 @@ class IpBan:
 
     """
 
-    VERSION = '1.0.13'
+    VERSION = '1.1.0'
 
     def __init__(self, app=None, ban_count=20, ban_seconds=3600 * 24, persist=False, record_dir=None, ipc=False,
                  secret_key=None, ip_header=None, abuse_IPDB_config=None):
@@ -60,6 +59,7 @@ def __init__(self, app=None, ban_count=20, ban_seconds=3600 * 24, persist=False,
         self._ip_whitelist = {'127.0.0.1': True}
         # self._ip_whitelist = {}
         self._ip_ban_list = {}
+        self._cidr_entries = {}
         # initialise with well known search bot links
         self._url_whitelist_patterns = {
             '^/.well-known/': dict(pattern=re.compile('^/.well-known'), match_type='regex'),
@@ -122,7 +122,7 @@ def _after_request(self, response):
 
     def block(self, ip_list, permanent=False, no_write=False, timestamp=None, url='block'):
         """
-        add a list of ip address to the block list
+        add a list of ip addresses to the block list
 
         :param ip_list: list of ip addresses to block
         :param permanent: (optional) True=do not allow entries to expire
@@ -170,11 +170,11 @@ def get_ip(self):
             ip = request.headers.get(self.ip_header)
         return ip or request.environ.get('REMOTE_ADDR')
 
-    def test_pattern_blocklist(self, url, ip=None):
+    def test_pattern_blocklist(self, url='', ip=None):
         """
         return true if the url or ip pattern matches an existing block
 
-        :param url: the url to check
+        :param url: (optional) the url to check
         :param ip: (optional) an ip to check
         :return:
         """
@@ -191,6 +191,13 @@ def test_pattern_blocklist(self, url, ip=None):
                 self._logger.warning('ip block match {}'.format(ip))
                 return True
 
+        if ip:
+            ip_address = ipaddress.ip_address(ip)
+            for c, c_ip in self._cidr_entries.items():
+                if ip_address in c_ip:
+                    self._logger.warning(f'ip block match {ip}. CIDR: {c}')
+                    return True
+
         return False
 
     def _before_request_check(self):
@@ -227,6 +234,9 @@ def _before_request_check(self):
                 self._logger.debug('IP expired from ban list {}.  Url: {}'.format(ip, url))
                 entry['count'] = 0
 
+    def block_cidr(self, cidr):
+        self._cidr_entries[cidr] = ipaddress.ip_network(cidr)
+
     def ip_whitelist_add(self, ip):
         """
         add the ip to the list of ips to whitelist
@@ -325,9 +335,9 @@ def display(self, option='html'):
             s += '</thead><tbody>\n'
             for k, r in self._ip_ban_list.items():
                 s += '<tr><td>{}</td><td>{}</td><td>{}</td><td>{}</td><td>{}</td></tr>\n'.format(k, r['count'],
-                                                                                      r.get('permanent', ''),
-                                                                                      r.get('url', ''),
-                                                                                      r['timestamp'])
+                                                                                                 r.get('permanent', ''),
+                                                                                                 r.get('url', ''),
+                                                                                                 r['timestamp'])
             s += '</tbody></table>'
         elif option == 'csv':
             for k, r in self._ip_ban_list.items():
@@ -448,6 +458,7 @@ def load_nuisances(self, file_name=None):
     my_ip = '127.0.0.1'
     test_ip_ban = IpBan(ban_count=4, ban_seconds=20, persist=True, record_dir='.flask-ip-ban-test-app',
                         # ip_header='X_IP_HEADER',
+                        ipc=True,
                         abuse_IPDB_config=dict(
                             key=os.environ.get('ABUSE_IPDB_KEY'),
                             report=True, load=False, debug=True))
@@ -482,6 +493,7 @@ def route_block_perm():
     def route_display():
         return test_ip_ban.display()
 
+
     @app.route('/clean')
     def route_clean():
         return str(test_ip_ban.ip_record.clean())

flask_ipban/nuisance.yaml

@@ -13,7 +13,7 @@
 # limitations under the License.
 #
 # some common nuisances that should be blocked in a flask app
-# updated: 28-02-2019
+# updated: 17-Mar-2020
 #
 # format:
 #
@@ -31,6 +31,7 @@ regex:
   - .*\.asp$
   - .*\.aspx$
   - .*\.asx$
+  - .*\.bak$
   - .*\.cfm$
   - .*\.cf$
   - .*\.cgi$
@@ -55,6 +56,7 @@ regex:
   - .*\.war$
   - .*\.wss$
   # hacking or scanning attempts
+  - .*\/login
   - .*/a2billing/.*
   - \/.*\/(calendar|date|fortune|redirect|passwd)$
   - .*?PHPSESSID=
@@ -67,10 +69,16 @@ regex:
   - \/bin\/
   - \/cgi-bin\/
   - \/htbin\/
+  - mysql
+  - phpmyadmin
   - \/scripts\/
+  - \/solr\/
   - \/HNAP1\/
   - \/WEB-INF\/
   - \/wp-
+  - \/web.config.txt$
+  - \+CSCOE\+/
+  - ^/joomla/
 
 string:
   # all of these are hacking attempts by vulnerability scanners

flask_ipban/test.py

@@ -41,6 +41,12 @@ def setUp(self):
 
         self.app.route('/')(hello_world)
 
+    def test_cidr(self):
+        self.assertFalse(self.ip_ban.test_pattern_blocklist(ip='192.0.2.1'))
+        self.ip_ban.block_cidr('192.0.2.0/28')
+        self.assertTrue(self.ip_ban.test_pattern_blocklist(ip='192.0.2.1'))
+        self.assertFalse(self.ip_ban.test_pattern_blocklist(ip='203.0.2.1'))
+
     def testAddRemoveIpWhitelist(self):
         self.assertEqual(self.ip_ban.ip_whitelist_add(localhost), 1)
         for x in range(self.ip_ban.ban_count * 2):

pypar.commands.sh

@@ -20,10 +20,10 @@ pip install twine
 
 python setup.py sdist bdist_wheel
 #
-twine check dist/flask_ipban-1.0.13*
+twine check dist/flask_ipban-1.1.0*
 # test
 # pip install -e .
 # twine upload --repository-url https://test.pypi.org/legacy/ dist/* -u martlark
 # prod pypi
 # add release in git hub to match the version
-twine upload dist/flask_ipban-1.0.13* -u martlark
+twine upload dist/flask_ipban-1.1.0* -u martlark

setup.py

@@ -15,7 +15,7 @@
 from codecs import open
 from setuptools import setup
 
-VERSION = '1.0.13'
+VERSION = '1.1.0'
 LONG_DESCRIPTION = open('README.rst', 'r', encoding='utf-8').read()
 
 setup(
@@ -44,11 +44,7 @@
         'License :: OSI Approved :: Apache Software License',
 
         'Programming Language :: Python',
-        'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 2.7',
-        'Programming Language :: Python :: 3.4',
-        'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',