diff --git a/README.md b/README.md index a25d705..4bbf553 100644 --- a/README.md +++ b/README.md @@ -256,6 +256,8 @@ get random proxy 116.196.115.209:8080 - TEST_TIMEOUT:测试超时时间,默认 10 秒 - TEST_BATCH:批量测试数量,默认 20 个代理 - TEST_VALID_STATUS:测试有效的状态码 +- TEST_ANONYMOUS:是否只保留匿名代理,默认 true +- TEST_ANONYMOUS_URL:匿名 / 出口 IP 检测地址,默认 `https://httpbin.org/ip`,需返回 httpbin 格式的 JSON(`{"origin": "1.2.3.4"}`)。可指向自建 httpbin 服务以避免公共服务限流 - API_HOST:代理 Server 运行 Host,默认 0.0.0.0 - API_PORT:代理 Server 运行端口,默认 5555 - API_THREADED:代理 Server 是否使用多线程,默认 true diff --git a/proxypool/processors/tester.py b/proxypool/processors/tester.py index bdc26b0..5d70d23 100644 --- a/proxypool/processors/tester.py +++ b/proxypool/processors/tester.py @@ -4,7 +4,7 @@ from proxypool.schemas import Proxy from proxypool.storages.redis import RedisClient from proxypool.setting import TEST_TIMEOUT, TEST_BATCH, TEST_URL, TEST_VALID_STATUS, TEST_ANONYMOUS, \ - TEST_DONT_SET_MAX_SCORE + TEST_DONT_SET_MAX_SCORE, TEST_ANONYMOUS_URL from aiohttp import ClientProxyConnectionError, ServerDisconnectedError, ClientOSError, ClientHttpProxyError, \ ClientResponseError, ContentTypeError from asyncio import TimeoutError @@ -49,7 +49,7 @@ async def test(self, proxy: Proxy, session: aiohttp.ClientSession): # the proxy has the effect of hiding the real IP # logger.debug(f'TEST_ANONYMOUS {TEST_ANONYMOUS}') if TEST_ANONYMOUS: - url = 'https://httpbin.org/ip' + url = TEST_ANONYMOUS_URL async with session.get(url, timeout=TEST_TIMEOUT) as response: resp_json = await response.json() origin_ip = resp_json['origin'] diff --git a/proxypool/setting.py b/proxypool/setting.py index a445667..4d1cec5 100644 --- a/proxypool/setting.py +++ b/proxypool/setting.py @@ -75,6 +75,10 @@ TEST_BATCH = env.int('TEST_BATCH', 20) # only save anonymous proxy TEST_ANONYMOUS = env.bool('TEST_ANONYMOUS', True) +# the url used to check the proxy anonymity and its exit ip; +# must return json like httpbin.org/ip ({"origin": "1.2.3.4"}); +# point this to a self-hosted httpbin to avoid public rate limits +TEST_ANONYMOUS_URL = env.str('TEST_ANONYMOUS_URL', 'https://httpbin.org/ip') # TEST_HEADERS = env.json('TEST_HEADERS', { # 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36', # })