Rebasing #1700 (Change Black/White List to Block/Allow list)

docs/config-carbon.rst

@@ -269,13 +269,28 @@ aggregation.
 **Note:** if you plan to use the ``=`` sign in your rewrite rules. Use its octal value: ``\075``.
 For example ``foo=bar = foo.bar`` would be ``foo\075bar = foo.bar``
 
+Metric Filters: allowed and blocked Metrics
+-------------------------------------------
+
+The metric filter functionality allows any of the carbon daemons to only accept
+metrics that are explicitly allowed and/or to reject rejected metrics. The
+functionality can be enabled in carbon.conf with the ``USE_METRIC_FILTERS``
+flag. This can be useful when too many metrics are being sent to a Graphite
+instance or when there are metric senders sending useless or invalid metrics.
+
+``GRAPHITE_CONF_DIR`` is searched for ``allowed_metrics.conf`` and
+``blocked_metrics.conf``. Each file contains one regular expression per line to
+match against metric values. If the allowed_metrics configuration is missing or
+empty, all metrics will be passed through by default.
+
 whitelist and blacklist
 -----------------------
-The whitelist functionality allows any of the carbon daemons to only accept metrics that are explicitly
-whitelisted and/or to reject blacklisted metrics. The functionality can be enabled in carbon.conf with
-the ``USE_WHITELIST`` flag. This can be useful when too many metrics are being sent to a Graphite
-instance or when there are metric senders sending useless or invalid metrics.
+The whitelist/blacklist functionality has been renamed to 'allowed' and
+'blocked' so as to use less ambiguous language, and remove possible connotations
+associated with those terms.
+
+The capabilities have been renamed as of this point, but he existing ``whitelist.conf``
+and ``blacklist.conf`` will still be functional for the time being.
 
-``GRAPHITE_CONF_DIR`` is searched for ``whitelist.conf`` and ``blacklist.conf``. Each file contains one regular
-expressions per line to match against metric values. If the whitelist configuration is missing or empty,
-all metrics will be passed through by default.
+Additionally, the ``USE_WHITELIST`` flag in carbon.conf will still be respected,
+but treated as ``USE_METRIC_FILTERS`` until it is deprecated.

docs/config-local-settings.rst

@@ -434,7 +434,7 @@ POOL_MAX_WORKERS
 REMOTE_RETRY_DELAY
  `Default: 60`
 
- Time in seconds to blacklist a webapp after a timed-out request.
+ Time in seconds to block/filter a webapp after a timed-out request.
 
 FIND_CACHE_DURATION
  `Default: 300`

setup.py

@@ -101,7 +101,7 @@ def read(fname):
  'graphite.url_shortener',
  'graphite.url_shortener.migrations',
  'graphite.version',
- 'graphite.whitelist',
+ 'graphite.metric_filters',
  'graphite.worker_pool',
  ],
  package_data={'graphite': ['templates/*', 'local_settings.py.example']},

tox.ini

@@ -11,7 +11,7 @@ envlist =
  lint, docs
 
 [testenv]
-whitelist_externals =
+whitelist_externals = # Unrelated to graphite project deprecation of white/black
  mkdir
 setenv =
  DJANGO_SETTINGS_MODULE=tests.settings

webapp/content/js/composer_widgets.js

@@ -1041,20 +1041,20 @@ var GraphDataWindow = {
 
  addWlSelected: function (item, e) {
  Ext.Ajax.request({
- url: document.body.dataset.baseUrl + 'whitelist/add',
+ url: document.body.dataset.baseUrl + 'metric_filters/add',
  method: 'POST',
- success: function () { Ext.Msg.alert('Result', 'Successfully added metrics to whitelist.'); },
- failure: function () { Ext.Msg.alert('Result', 'Failed to add metrics to whitelist.'); },
+ success: function () { Ext.Msg.alert('Result', 'Successfully added metrics to filter.'); },
+ failure: function () { Ext.Msg.alert('Result', 'Failed to add metrics to filter.'); },
  params: {metrics: this.getSelectedTargets().join('\n') }
  });
  },
 
  removeWlSelected: function (item, e) {
  Ext.Ajax.request({
- url: document.body.dataset.baseUrl + 'whitelist/remove',
+ url: document.body.dataset.baseUrl + 'metric_filters/remove',
  method: 'POST',
- success: function () { Ext.Msg.alert('Result', 'Successfully removed metrics from whitelist.'); },
- failure: function () { Ext.Msg.alert('Result', 'Failed to remove metrics from whitelist.'); },
+ success: function () { Ext.Msg.alert('Result', 'Successfully removed metrics from filter.'); },
+ failure: function () { Ext.Msg.alert('Result', 'Failed to remove metrics from filter.'); },
  params: {metrics: this.getSelectedTargets().join('\n') }
  });
  },

webapp/graphite/app_settings.py

@@ -105,7 +105,7 @@
  'graphite.render',
  'graphite.tags',
  'graphite.url_shortener',
- 'graphite.whitelist',
+ 'graphite.metric_filters',
  'django.contrib.auth',
  'django.contrib.sessions',
  'django.contrib.admin',

webapp/graphite/local_settings.py.example

@@ -337,7 +337,7 @@ DEFAULT_XFILES_FACTOR = 0
 #
 #CARBONLINK_HOSTS = ["127.0.0.1:7002:a", "127.0.0.1:7102:b", "127.0.0.1:7202:c"]
 #CARBONLINK_TIMEOUT = 1.0
-#CARBONLINK_RETRY_DELAY = 15 # Seconds to blacklist a failed remote server
+#CARBONLINK_RETRY_DELAY = 15 # Seconds to block a failed remote server
 # Set pickle protocol to use for Carbonlink requests,
 # (default of -1 is HIGHEST_AVAILABLE for your Python version)
 # see more: https://docs.python.org/3/library/pickle.html#data-stream-format

webapp/graphite/whitelist/urls.py → webapp/graphite/metric_filters/urls.py

@@ -16,7 +16,7 @@
 from . import views
 
 urlpatterns = [
- url(r'^/add$', views.add, name='whitelist_add'),
- url(r'^/remove$', views.remove, name='whitelist_remove'),
- url(r'^/?$', views.show, name='whitelist_show'),
+ url(r'^/add$', views.add, name='metric_filters_add'),
+ url(r'^/remove$', views.remove, name='metric_filters_remove'),
+ url(r'^/?$', views.show, name='metric_filters_show'),
 ]

webapp/graphite/whitelist/views.py → webapp/graphite/metric_filters/views.py

@@ -23,43 +23,43 @@
 
 def add(request):
  metrics = set( request.POST['metrics'].split() )
- whitelist = load_whitelist()
- new_whitelist = whitelist | metrics
- save_whitelist(new_whitelist)
+ allowed_metrics = load_allowed_metrics()
+ new_allowed_metrics = allowed_metrics | metrics
+ save_allowed_metrics(new_allowed_metrics)
  return HttpResponse(content_type="text/plain", content="OK")
 
 
 def remove(request):
  metrics = set( request.POST['metrics'].split() )
- whitelist = load_whitelist()
- new_whitelist = whitelist - metrics
- save_whitelist(new_whitelist)
+ allowed_metrics = load_allowed_metrics()
+ new_allowed_metrics = allowed_metrics - metrics
+ save_allowed_metrics(new_allowed_metrics)
  return HttpResponse(content_type="text/plain", content="OK")
 
 
 def show(request):
- whitelist = load_whitelist()
- members = '\n'.join( sorted(whitelist) )
+ allowed_metrics = load_allowed_metrics()
+ members = '\n'.join( sorted(allowed_metrics) )
  return HttpResponse(content_type="text/plain", content=members)
 
 
-def load_whitelist():
- buffer = open(settings.WHITELIST_FILE, 'rb').read()
- whitelist = unpickle.loads(buffer)
- return whitelist
+def load_allowed_metrics():
+ buffer = open(settings.METRIC_FILTERS_FILE, 'rb').read()
+ allowed_metrics = unpickle.loads(buffer)
+ return allowed_metrics
 
 
-def save_whitelist(whitelist):
+def save_allowed_metrics(allowed_metrics):
  # do this instead of dump() to raise potential exceptions before open()
- serialized = pickle.dumps(whitelist, protocol=-1)
- tmpfile = '%s-%d' % (settings.WHITELIST_FILE, randint(0, 100000))
+ serialized = pickle.dumps(allowed_metrics, protocol=-1)
+ tmpfile = '%s-%d' % (settings.METRIC_FILTERS_FILE, randint(0, 100000))
  try:
  fh = open(tmpfile, 'wb')
  fh.write(serialized)
  fh.close()
- if os.path.exists(settings.WHITELIST_FILE):
- os.unlink(settings.WHITELIST_FILE)
- os.rename(tmpfile, settings.WHITELIST_FILE)
+ if os.path.exists(settings.METRIC_FILTERS_FILE):
+ os.unlink(settings.METRIC_FILTERS_FILE)
+ os.rename(tmpfile, settings.METRIC_FILTERS_FILE)
  finally:
  if os.path.exists(tmpfile):
  os.unlink(tmpfile)

webapp/graphite/settings.py

@@ -50,7 +50,7 @@
 DASHBOARD_CONF = ''
 GRAPHTEMPLATES_CONF = ''
 STORAGE_DIR = ''
-WHITELIST_FILE = ''
+METRIC_FILTERS_FILE = ''
 INDEX_FILE = ''
 LOG_DIR = ''
 CERES_DIR = ''
@@ -266,8 +266,8 @@
 
 if not STORAGE_DIR:
  STORAGE_DIR = os.environ.get('GRAPHITE_STORAGE_DIR', join(GRAPHITE_ROOT, 'storage'))
-if not WHITELIST_FILE:
- WHITELIST_FILE = join(STORAGE_DIR, 'lists', 'whitelist')
+if not METRIC_FILTERS_FILE:
+ METRIC_FILTERS_FILE = join(STORAGE_DIR, 'lists', 'allowed_metrics')
 if not INDEX_FILE:
  INDEX_FILE = join(STORAGE_DIR, 'index')
 if not LOG_DIR:

webapp/graphite/urls.py

@@ -25,7 +25,7 @@
  url('^browser', include('graphite.browser.urls')),
  url('^account', include('graphite.account.urls')),
  url('^dashboard', include('graphite.dashboard.urls')),
- url('^whitelist', include('graphite.whitelist.urls')),
+ url('^metric_filters/?', include('graphite.metric_filters.urls')),
  url('^version', include('graphite.version.urls')),
  url('^events', include('graphite.events.urls')),
  url('^tags', include('graphite.tags.urls')),

webapp/tests/test_whitelist.py

@@ -12,108 +12,108 @@
  from django.core.urlresolvers import reverse
 from .base import TestCase
 
-from graphite.whitelist.views import load_whitelist, save_whitelist
+from graphite.metric_filters.views import load_allowed_metrics, save_allowed_metrics
 
 
 class WhitelistTester(TestCase):
- settings.WHITELIST_FILE = os.path.join(DATA_DIR, 'lists/whitelist')
+ settings.METRIC_FILTERS_FILE = os.path.join(DATA_DIR, 'lists/allowlist')
 
- def wipe_whitelist(self):
+ def wipe_allowlist(self):
  try:
- os.remove(settings.WHITELIST_FILE)
+ os.remove(settings.METRIC_FILTERS_FILE)
  except OSError:
  pass
 
- def create_whitelist(self):
+ def create_allowlist(self):
  try:
- os.makedirs(settings.WHITELIST_FILE.replace('whitelist', ''))
+ os.makedirs(settings.METRIC_FILTERS_FILE.replace('allowlist', ''))
  except OSError:
  pass
- fh = open(settings.WHITELIST_FILE, 'wb')
+ fh = open(settings.METRIC_FILTERS_FILE, 'wb')
  pickle.dump({'a.b.c.d', 'e.f.g.h'}, fh)
  fh.close()
 
- def test_whitelist_show_no_whitelist(self):
- url = reverse('whitelist_show')
+ def test_allowed_metrics_show_no_allowlist(self):
+ url = reverse('metric_filters_show')
  with self.assertRaises(IOError):
  _ = self.client.get(url)
 
- def test_whitelist_show(self):
- url = reverse('whitelist_show')
- self.create_whitelist()
- self.addCleanup(self.wipe_whitelist)
+ def test_allowed_metrics_show(self):
+ url = reverse('metric_filters_show')
+ self.create_allowlist()
+ self.addCleanup(self.wipe_allowlist)
  response = self.client.get(url)
  self.assertEqual(response.status_code, 200)
  self.assertEqual(response.content, b"a.b.c.d\ne.f.g.h")
 
- def test_whitelist_add(self):
- self.create_whitelist()
- self.addCleanup(self.wipe_whitelist)
+ def test_allowed_metrics_add(self):
+ self.create_allowlist()
+ self.addCleanup(self.wipe_allowlist)
 
- url = reverse('whitelist_add')
+ url = reverse('metric_filters_add')
  response = self.client.post(url, {'metrics': ['i.j.k.l']})
  self.assertEqual(response.status_code, 200)
  self.assertEqual(response.content, b"OK")
 
- url = reverse('whitelist_show')
+ url = reverse('metric_filters_show')
  response = self.client.get(url)
  self.assertEqual(response.status_code, 200)
  self.assertEqual(response.content, b"a.b.c.d\ne.f.g.h\ni.j.k.l")
 
- def test_whitelist_add_existing(self):
- self.create_whitelist()
- self.addCleanup(self.wipe_whitelist)
+ def test_allowed_metrics_add_existing(self):
+ self.create_allowlist()
+ self.addCleanup(self.wipe_allowlist)
 
- url = reverse('whitelist_add')
+ url = reverse('metric_filters_add')
  response = self.client.post(url, {'metrics': ['a.b.c.d']})
  self.assertEqual(response.status_code, 200)
  self.assertEqual(response.content, b"OK")
 
- url = reverse('whitelist_show')
+ url = reverse('metric_filters_show')
  response = self.client.get(url)
  self.assertEqual(response.status_code, 200)
  self.assertEqual(response.content, b"a.b.c.d\ne.f.g.h")
 
- def test_whitelist_remove(self):
- self.create_whitelist()
- self.addCleanup(self.wipe_whitelist)
+ def test_allowed_metrics_remove(self):
+ self.create_allowlist()
+ self.addCleanup(self.wipe_allowlist)
 
- url = reverse('whitelist_remove')
+ url = reverse('metric_filters_remove')
  response = self.client.post(url, {'metrics': ['a.b.c.d']})
  self.assertEqual(response.status_code, 200)
  self.assertEqual(response.content, b"OK")
 
- url = reverse('whitelist_show')
+ url = reverse('metric_filters_show')
  response = self.client.get(url)
  self.assertEqual(response.status_code, 200)
  self.assertEqual(response.content, b"e.f.g.h")
 
- def test_whitelist_remove_missing(self):
- self.create_whitelist()
- self.addCleanup(self.wipe_whitelist)
+ def test_allowed_metrics_remove_missing(self):
+ self.create_allowlist()
+ self.addCleanup(self.wipe_allowlist)
 
- url = reverse('whitelist_remove')
+ url = reverse('metric_filters_remove')
  response = self.client.post(url, {'metrics': ['i.j.k.l']})
  self.assertEqual(response.status_code, 200)
  self.assertEqual(response.content, b"OK")
 
- url = reverse('whitelist_show')
+ url = reverse('metric_filters_show')
  response = self.client.get(url)
  self.assertEqual(response.status_code, 200)
  self.assertEqual(response.content, b"a.b.c.d\ne.f.g.h")
 
- def test_save_whitelist(self):
+ def test_save_allowed_metrics(self):
  try:
- os.makedirs(settings.WHITELIST_FILE.replace('whitelist', ''))
+ os.makedirs(settings.METRIC_FILTERS_FILE.replace('allowlist', ''))
  except OSError:
  pass
- self.addCleanup(self.wipe_whitelist)
- self.assertEqual(save_whitelist({'a.b.c.d','e.f.g.h'}), None)
- self.assertEqual(load_whitelist(), {'a.b.c.d','e.f.g.h'})
+ self.addCleanup(self.wipe_allowlist)
+ self.assertEqual(save_allowed_metrics({'a.b.c.d','e.f.g.h'}), None)
+ self.assertEqual(load_allowed_metrics(), {'a.b.c.d','e.f.g.h'})
 
  @mock.patch('os.rename')
- def test_save_whitelist_rename_failure(self, rename):
- self.addCleanup(self.wipe_whitelist)
+ def test_save_allowed_metrics_rename_failure(self, rename):
+ self.addCleanup(self.wipe_allowlist)
  rename.side_effect = OSError(errno.EPERM, 'Operation not permitted')
  with self.assertRaises(OSError):
- save_whitelist({'a.b.c.d','e.f.g.h'})
+ save_allowed_metrics({'a.b.c.d','e.f.g.h'})