From 3a1f3f6014fdfd8e180e7d48b40ef6b9f9a97b2c Mon Sep 17 00:00:00 2001 From: B1ACK917 <39761503@qq.com> Date: Thu, 21 Dec 2023 22:25:41 +0800 Subject: [PATCH 1/2] fix: docip bug --- proxypool/crawlers/public/docip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxypool/crawlers/public/docip.py b/proxypool/crawlers/public/docip.py index 070c598..154871f 100644 --- a/proxypool/crawlers/public/docip.py +++ b/proxypool/crawlers/public/docip.py @@ -25,7 +25,7 @@ def parse(self, html): proxy_list = result['data'] for proxy_item in proxy_list: host = proxy_item['ip'] - port = proxy_item['port'] + port = host.split(':')[-1] yield Proxy(host=host, port=port) except json.JSONDecodeError: print("json.JSONDecodeError") From d0daeb0d95b8df0bd185432a8f1c568f60f21afa Mon Sep 17 00:00:00 2001 From: B1ACK917 <39761503@qq.com> Date: Thu, 21 Dec 2023 23:11:40 +0800 Subject: [PATCH 2/2] update: support for python3.11 --- proxypool/processors/tester.py | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/proxypool/processors/tester.py b/proxypool/processors/tester.py index 353332a..5879528 100644 --- a/proxypool/processors/tester.py +++ b/proxypool/processors/tester.py @@ -82,7 +82,7 @@ def run(self): logger.debug(f'testing proxies use cursor {cursor}, count {TEST_BATCH}') cursor, proxies = self.redis.batch(cursor, count=TEST_BATCH) if proxies: - tasks = [self.test(proxy) for proxy in proxies] + tasks = [self.loop.create_task(self.test(proxy)) for proxy in proxies] self.loop.run_until_complete(asyncio.wait(tasks)) if not cursor: break diff --git a/requirements.txt b/requirements.txt index c9407c7..33f35c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ redis>=3.5.3,<4.0.0 lxml>=4.6.5,<5.0.0 fake_headers>=1.0.2,<2.0.0 maxminddb_geolite2==2018.703 -gevent>=21.8.0,<22.0.0 +gevent>=21.8.0,<24.0.0 tornado>=6.0,<7.0 itsdangerous==0.24 MarkupSafe<2.1.0