更新拨号策略实现拨号的稳定和监控

README.md

@@ -1,3 +1,20 @@
+## 更新和优化(相对于崔大神原脚本)
+1. 增加邮件提醒
+2. 修正拨号间隔错误,原脚本的小bug,拨号间隔会是settings中的两倍
+3. 增加拨号统计:每次拨出的IP放入redis,每拨一次value +1,如果是2会重新拨号,防止重复IP出现.如果需要可以重置下这个频次的值,参考proxy_reset.py.这个考虑到平台对IP的封禁并非长期,通常24小时后能解封
+4. 增加拨号日志可视化监控,在本地运行proxy_stats.py读取远程拨号服务器日志并可视化展示拨号状态,比如这里的adsl1_proxy_quality_monitor.jpg
+![image](https://raw.githubusercontent.com/chenxuzhen/AdslProxy/master/adsl1_proxy_quality_monitor.jpg)
+5. 连续三次拨号无效IP系统会重启,因为这时候服务器已经不能继续拨号了
+6. 从redis删除IP失败系统会重启,这个时候一般都是无法拨号了
+7. 更新proxy检测方式为ping,拨号一次只需要6-7秒(当然和代理商有关系).这个针对单地区adsl vps特别有效,因为单地区拨号服务器带宽都没问题,拨出的IP都很稳定,只要能ping通都是高速可用的.个人建议
+抛弃混拨服务器,带宽低而且拨号慢,不如多个地区的组合.本人测试过三家的拨号服务器,如有需要可提供免费建议.
+8. service.sh放到/etc/init.d目录下, /bin/bash /etc/init.d/service.sh放在/etc/rc.local最后,系统重启后会自动运行拨号脚本.
+9. 增加了拨号服务器一键部署,用squid作代理服务器,中间只需要输入一次密码,参考vpsadsl.sh,用户名默认czhen可以自己修改.另外,如果没有需要,hash掉那段关于修改sshd端口的脚本
+10. 基于以上更新,脚本可以长期运行
+
+Field                                     Value
+czhen:[email protected]:3389  1
+
 ## 拨号主机设置
 
 首先配置好代理，如使用 Squid，运行在 3128 端口，并设置好用户名和密码。

adslproxy/db.py

@@ -1,8 +1,9 @@
 # coding=utf-8
+# 放入Redis之前加了一个时间戳，方便判断IP存活时间
 import redis
 import random
 from adslproxy.settings import *
-
+import time
 
 class RedisClient(object):
     def __init__(self, host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD, redis_key=REDIS_KEY):
@@ -23,7 +24,7 @@ def set(self, name, proxy):
         :param proxy: 代理
         :return: 设置结果
         """
-        return self.db.hset(self.redis_key, name, proxy)
+        return self.db.hset(self.redis_key, name, proxy + '_' + str(int(time.time())))
 
     def get(self, name):
         """

adslproxy/sender/sender.py

@@ -1,14 +1,26 @@
 # coding=utf-8
+# 采用了新的proxy验证方式,ping一次速度更快
+# 修正了原来版本的几个小bug，比如拨号间隔原来实际上是设定值的两倍，如果proxy无效等待时间太长（改成6秒或者其他最低拨号间隔即可)
+# IP与上次的相同会自动会重新拨号
+# 连续三次拨号失败自动重启(ADSL VPS这种情况下基本上等于无法继续拨号了)
+# 增加邮件提醒
+# 每次拨出的IP存入redis,方便统计和去重.IP出现2次以上会重新拨号
+# 从redis移除移除IP失败立即重启,这个情况下VPS通常已经无法拨号了
+
 import re
 import time
 import requests
 from requests.exceptions import ConnectionError, ReadTimeout
 from adslproxy.db import RedisClient
 from adslproxy.settings import *
+from adslproxy.sendemail import EmailClient
 import platform
 from loguru import logger
 from retrying import retry, RetryError
 import redis
+import datetime
+import os
+import random
 
 if platform.python_version().startswith('2.'):
     import commands as subprocess
@@ -22,7 +34,8 @@ class Sender(object):
     """
     拨号并发送到 Redis
     """
-
+    ip_pre = ''
+    invalid_ip_list = []
     def extract_ip(self):
         """
         获取本机IP
@@ -42,22 +55,31 @@ def test_proxy(self, proxy):
         测试代理，返回测试结果
         :param proxy: 代理
         :return: 测试结果
+        :ping一次测试速度更快，只需要几十毫秒
         """
-        try:
-            response = requests.get(TEST_URL, proxies={
-                'http': 'http://' + proxy,
-                'https': 'https://' + proxy
-            }, timeout=TEST_TIMEOUT)
-            if response.status_code == 200:
-                return True
-        except (ConnectionError, ReadTimeout):
+        # try:
+        #    response = requests.get(TEST_URL, proxies={
+        #        'http': 'http://' + proxy,
+        #        'https': 'https://' + proxy
+        #    }, timeout=TEST_TIMEOUT)
+        #    if response.status_code == 200:
+        #        logger.info(f'proxy: {proxy}')
+        #        return True
+        #except (ConnectionError, ReadTimeout):
+        #    return False
+        con = os.system('ping -c 1 www.baidu.com')
+        print(con)
+        if con==0:
+            return True
+        else:
             return False
 
     @retry(retry_on_result=lambda x: x is not True, stop_max_attempt_number=10)
     def remove_proxy(self):
         """
         移除代理
         :return: None
+        通常情况下，连续拨号失败几次就需要重启机器了，这时候VPS已经无法成功拨号连接互联网了
         """
         logger.info(f'Removing {CLIENT_NAME}...')
         try:
@@ -70,16 +92,35 @@ def remove_proxy(self):
             return True
         except redis.ConnectionError:
             logger.info(f'Remove {CLIENT_NAME} failed')
-
+            logger.error('删除IP失败!从代理池删除IP并重启系统.......')
+            os.system('/usr/sbin/shutdown -r now')      
     def set_proxy(self, proxy):
         """
         设置代理
         :param proxy: 代理
         :return: None
         """
         self.redis = RedisClient()
-        if self.redis.set(CLIENT_NAME, proxy):
-            logger.info(f'Successfully set proxy {proxy}')
+        self.db = RedisClient().db
+        # 哈希表来统计拨号VPS的IP
+        if not self.db.hexists('dialed_IPs', proxy):            
+            self.db.hset('dialed_IPs', proxy, 1)
+            # 往IP池里插入数据
+            if self.redis.set(CLIENT_NAME, proxy):
+                logger.info(f'Successfully set proxy {proxy}')             
+            return True
+        else:
+            num = int(self.db.hget('dialed_IPs', proxy))
+            logger.info(f'{proxy} in proxy pools {num} times already')
+            if num <2:
+                self.db.hset('dialed_IPs', proxy, num+1)
+                # 往IP池里插入数据
+                if self.redis.set(CLIENT_NAME, proxy):
+                    logger.info(f'Successfully set proxy {proxy}')   
+                return True
+            else:
+
+                return False
 
     def loop(self):
         """
@@ -88,23 +129,44 @@ def loop(self):
         """
         while True:
             logger.info('Starting dial...')
-            self.run()
-            time.sleep(DIAL_CYCLE)
+            now = datetime.datetime.now()
+            if now.minute%5==0 and now.second==0:
+                logger.info('dial time: %s', now.strftime('%Y-%m-%d %H:%M:%S'))
+
+            new_ip = self.run()
+            if new_ip != self.ip_pre:
+
+                self.ip_pre = new_ip
+            else:
+                logger.info('IP和上次相同，等待重播......')
+                self.run()
 
     def run(self):
         """
         拨号主进程
         :return: None
         """
+        #time.sleep(10) #给正在运行的作业留出时间结束
         logger.info('Dial started, remove proxy')
         try:
             self.remove_proxy()
         except RetryError:
             logger.error('Retried for max times, continue')
-        # 拨号
-        (status, output) = subprocess.getstatusoutput(DIAL_BASH)
+            self.emailclient = EmailClient()
+            self.emailclient.notification(f'failed too many times {datetime.datetime.now().strftime("%m-%d-%H-%M")}', f'Warning{random.randint(1000,299999)}: 22457 retry error {datetime.datetime.now().strftime("%m-%d-%H-%M")}')
+
+        for i in range(3):
+            # 拨号
+            (status, output) = subprocess.getstatusoutput('adsl-stop;adsl-start')
+            if not status == 0:
+                logger.error('Dial failed')
+                time.sleep(20)
+            else:
+                break 
         if not status == 0:
-            logger.error('Dial failed')
+            print('连续三次拨号失败,系统重启......')
+            os.system('sudo reboot')
+
         # 获取拨号 IP
         ip = self.extract_ip()
         if ip:
@@ -115,24 +177,56 @@ def run(self):
                                                                    ip=ip, port=PROXY_PORT)
             else:
                 proxy = '{ip}:{port}'.format(ip=ip, port=PROXY_PORT)
-            time.sleep(10)
+            # time.sleep(1)
             if self.test_proxy(proxy):
                 logger.info(f'Valid proxy {proxy}')
+                self.ip_validity_statistics('valid')
                 # 将代理放入数据库
-                self.set_proxy(proxy)
-                time.sleep(DIAL_CYCLE)
+                if self.set_proxy(proxy):
+                    time.sleep(DIAL_CYCLE)                                                      
             else:
                 logger.error(f'Proxy invalid {proxy}')
+                # 连续三次拨号无效
+                self.ip_validity_statistics('invalid')                                   
+                if len(self.invalid_ip_list) > 0:
+                    if self.invalid_ip_list.count('invalid') == 3:
+                        logger.error('连续三次拨号失败!从代理池删除IP并重启系统.......')
+                        self.remove_proxy()
+                        os.system('/usr/sbin/shutdown -r now')                        
+                time.sleep(DIAL_ERROR_CYCLE)
         else:
             # 获取 IP 失败，重新拨号
             logger.error('Get IP failed, re-dialing')
+            ip = ''
+            time.sleep(DIAL_ERROR_CYCLE)
             self.run()
-
-
+        return ip
+    def ip_validity_statistics(self, ele):
+        if len(self.invalid_ip_list) < 3:
+            self.invalid_ip_list.append(ele)
+        else:
+            self.invalid_ip_list.pop(0)
+            self.invalid_ip_list.append(ele)
+
 def send(loop=True):
     sender = Sender()
     sender.loop() if loop else sender.run()
 
 
 if __name__ == '__main__':
-    send()
+    try:
+        emailclient = EmailClient()
+        emailclient.notification(f'{datetime.datetime.now().strftime("%m-%d-%H:%M")} {random.randint(300, 9999)} proxy restarted', f'{datetime.datetime.now().strftime("%m-%d-%H:%M")} 22457 proxyserver is back {random.randint(300, 9999)}')
+        print('email test success')
+    except Exception as e:
+        print(e)
+    while True:
+        con = os.system('ping -c 1 www.baidu.com')
+        print(con)
+        if con==0:
+            time.sleep(6)
+            send()
+            break
+        else:
+            time.sleep(1)
+

adslproxy/sendmail.py

@@ -0,0 +1,43 @@
+
+# -*- coding: utf-8 -*-
+import time
+# from playsound import playsound
+from datetime import datetime
+import smtplib
+from email.mime.text import MIMEText
+from email.header import Header
+
+class EmailClient(object):
+    def __init__(self):
+        """
+        初始化邮件列表     
+
+        """
+        self.to_list = [SENDER_EMAIL, RECEIVER_EMAIL]
+    def notification(self, body, subj):
+        sender = SENDER_EMAIL  # 邮件发送人
+        receiver = RECEIVER_EMAIL  # 邮件收件人
+        subject = 'adslproxy notification: ' + subj + ' ' + str(datetime.today())[:16]  # 主题
+        smtpserver = 'smtp.163.com'  # 网易的STMP地址 默认端口号为25
+        username = EMAIL  # 发送邮件的人
+        password = PASS  # 你所设置的密码.网易在开通SMTP服务后会有个密码设置
+
+        # 中文需参数‘utf-8'，单字节字符不需要
+        msg = MIMEText(body, 'plain', 'utf-8')
+        msg['Subject'] = Header(subject, 'utf-8')  # 头部信息:标题
+        msg['From'] = 'user<SENDER_EMAIL>'  # 头部信息:名称<发件人的地址>
+        msg['To'] = ",".join(self.to_list)  # 头部信息:收件人地址
+        m = 0
+        while m < 3:
+            try:        
+                smtp = smtplib.SMTP_SSL('smtp.163.com', 465)
+                smtp.login(username, password)
+                smtp.sendmail(sender, receiver, msg.as_string())
+                smtp.quit()
+                print('success')
+                m += 1
+                break
+            except smtplib.SMTPException as e:
+                print('Error: ', e)
+                m += 1
+                time.sleep(25)

hosts

@@ -0,0 +1,24 @@
+127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
+::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
+
+# GitHub Start
+140.82.113.4      github.com
+140.82.113.4      gist.github.com
+140.82.114.5      api.github.com
+185.199.111.153   assets-cdn.github.com
+199.232.96.133    raw.githubusercontent.com
+199.232.96.133    raw.github.com
+199.232.96.133    gist.githubusercontent.com
+199.232.96.133    cloud.githubusercontent.com
+199.232.96.133    camo.githubusercontent.com
+199.232.96.133    avatars0.githubusercontent.com
+199.232.96.133    avatars1.githubusercontent.com
+199.232.96.133    avatars2.githubusercontent.com
+199.232.96.133    avatars3.githubusercontent.com
+199.232.96.133    avatars4.githubusercontent.com
+199.232.96.133    avatars5.githubusercontent.com
+199.232.96.133    avatars6.githubusercontent.com
+199.232.96.133    avatars7.githubusercontent.com
+199.232.96.133    avatars8.githubusercontent.com
+# GitHub End
+

proxy_reset.py

@@ -0,0 +1,15 @@
+# coding=utf-8
+# 因为网站封禁IP并非长期,24小时后可以运行该脚本减小拨号出现次数的值,提高IP利用率
+import redis
+import random
+import time
+import redis
+import re
+
+client = redis.Redis(host=REDIS_HOST, port=7379, db=0, password=REDIS_PASSORD)
+client.hvals('dialed_IPs')
+client.hkeys('dialed_IPs')
+for i in client.hkeys('dialed_IPs'):
+    num = int(client.hget('dialed_IPs', i))
+    if num >=1:
+        client.hset('dialed_IPs', i, num-1)