BloomFilter去重默认使用同一个key，其它类型去重默认使用redis_key

feapder/VERSION

@@ -1 +1 @@
-1.9.0-beta1
+1.9.0-beta2

feapder/buffer/item_buffer.py

@@ -58,9 +58,18 @@ def __init__(self, redis_key, task_table=None):
  self._mysql_pipeline = None
 
  if setting.ITEM_FILTER_ENABLE and not self.__class__.dedup:
- self.__class__.dedup = Dedup(
- to_md5=False, **setting.ITEM_FILTER_SETTING
- )
+ if setting.ITEM_FILTER_SETTING.get(
+ "filter_type"
+ ) == Dedup.BloomFilter or setting.ITEM_FILTER_SETTING.get("name"):
+ self.__class__.dedup = Dedup(
+ to_md5=False, **setting.ITEM_FILTER_SETTING
+ )
+ else:
+ self.__class__.dedup = Dedup(
+ to_md5=False,
+ name=self._redis_key,
+ **setting.ITEM_FILTER_SETTING,
+ )
 
  # 导出重试的次数
  self.export_retry_times = 0

feapder/buffer/request_buffer.py

@@ -28,14 +28,16 @@ def __init__(self, db=None, dedup_name: str = None):
  self._db = db or MemoryDB()
 
  if not self.__class__.dedup and setting.REQUEST_FILTER_ENABLE:
- if dedup_name:
+ if setting.REQUEST_FILTER_SETTING.get(
+ "filter_type"
+ ) == Dedup.BloomFilter or setting.REQUEST_FILTER_SETTING.get("name"):
  self.__class__.dedup = Dedup(
- name=dedup_name, to_md5=False, **setting.REQUEST_FILTER_SETTING
- ) # 默认使用内存去重
+ to_md5=False, **setting.REQUEST_FILTER_SETTING
+ )
  else:
  self.__class__.dedup = Dedup(
- to_md5=False, **setting.REQUEST_FILTER_SETTING
- ) # 默认使用内存去重
+ to_md5=False, name=dedup_name, **setting.REQUEST_FILTER_SETTING
+ )
 
  def is_exist_request(self, request):
  if (