From a932eaafb44ba82ff3c82ac0f6d76a886f931ef2 Mon Sep 17 00:00:00 2001 From: qinjin Date: Fri, 3 Nov 2023 14:52:51 +0800 Subject: [PATCH 1/6] =?UTF-8?q?1=E3=80=81=E8=AE=BE=E7=BD=AEscheduler.py?= =?UTF-8?q?=E4=B8=ADdev=E6=A8=A1=E5=BC=8F=E7=9A=84flask=20run=EF=BC=8C?= =?UTF-8?q?=E4=B8=8D=E8=BF=9B=E8=A1=8C=E5=90=AF=E5=8A=A8=E6=97=B6=E7=9A=84?= =?UTF-8?q?=E8=87=AA=E5=8A=A8=E9=87=8D=E6=96=B0=E5=8A=A0=E8=BD=BDreload?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- proxypool/scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxypool/scheduler.py b/proxypool/scheduler.py index f29b5655..a2d18abe 100644 --- a/proxypool/scheduler.py +++ b/proxypool/scheduler.py @@ -92,7 +92,7 @@ def run_server(self): logger.error("unsupported APP_PROD_METHOD") return else: - app.run(host=API_HOST, port=API_PORT, threaded=API_THREADED) + app.run(host=API_HOST, port=API_PORT, threaded=API_THREADED, use_reloader=False) def run(self): global tester_process, getter_process, server_process From a959507a80deb86b19b691c2fb2514da461b7d8d Mon Sep 17 00:00:00 2001 From: qinjin Date: Fri, 3 Nov 2023 14:55:40 +0800 Subject: [PATCH 2/6] =?UTF-8?q?1=E3=80=81=E4=BF=AE=E6=94=B9PROXY=5FSCORE?= =?UTF-8?q?=5FMAX=EF=BC=8CPROXY=5FSCORE=5FMIN=EF=BC=8CPROXY=5FSCORE=5FINIT?= =?UTF-8?q?=E4=B8=89=E9=A1=B9=E9=85=8D=E7=BD=AE=EF=BC=8C=E4=B8=BA=E5=8F=AF?= =?UTF-8?q?=E7=8E=AF=E5=A2=83=E5=8F=98=E9=87=8F=E9=85=8D=E7=BD=AE=202?= =?UTF-8?q?=E3=80=81=E6=B7=BB=E5=8A=A0=E5=8F=AF=E7=8E=AF=E5=A2=83=E5=8F=98?= =?UTF-8?q?=E9=87=8F=E9=85=8D=E7=BD=AE=E9=A1=B9TEST=5FDONT=5FSET=5FMAX=5FS?= =?UTF-8?q?CORE=EF=BC=8C=E5=85=81=E8=AE=B8=E8=AE=BE=E7=BD=AE=E5=BD=93teste?= =?UTF-8?q?r=E6=A3=80=E6=B5=8B=E5=88=B0=E6=9F=90=E4=B8=AAproxy=E5=8F=AF?= =?UTF-8?q?=E7=94=A8=E6=97=B6=EF=BC=8C=E5=8F=AA=E6=98=AF=E4=BF=9D=E6=8C=81?= =?UTF-8?q?=E5=8E=9Fscore=EF=BC=8C=E8=80=8C=E4=B8=8D=E5=B0=86=E5=85=B6scor?= =?UTF-8?q?e=E8=AE=BE=E7=BD=AE=E6=88=90max=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- proxypool/processors/tester.py | 20 ++++++++++++-------- proxypool/setting.py | 8 +++++--- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/proxypool/processors/tester.py b/proxypool/processors/tester.py index f002056a..353332ac 100644 --- a/proxypool/processors/tester.py +++ b/proxypool/processors/tester.py @@ -3,11 +3,11 @@ from loguru import logger from proxypool.schemas import Proxy from proxypool.storages.redis import RedisClient -from proxypool.setting import TEST_TIMEOUT, TEST_BATCH, TEST_URL, TEST_VALID_STATUS, TEST_ANONYMOUS +from proxypool.setting import TEST_TIMEOUT, TEST_BATCH, TEST_URL, TEST_VALID_STATUS, TEST_ANONYMOUS, \ + TEST_DONT_SET_MAX_SCORE from aiohttp import ClientProxyConnectionError, ServerDisconnectedError, ClientOSError, ClientHttpProxyError from asyncio import TimeoutError - EXCEPTIONS = ( ClientProxyConnectionError, ConnectionRefusedError, @@ -23,14 +23,14 @@ class Tester(object): """ tester for testing proxies in queue """ - + def __init__(self): """ init redis """ self.redis = RedisClient() self.loop = asyncio.get_event_loop() - + async def test(self, proxy: Proxy): """ test single proxy @@ -55,15 +55,18 @@ async def test(self, proxy: Proxy): async with session.get(TEST_URL, proxy=f'http://{proxy.string()}', timeout=TEST_TIMEOUT, allow_redirects=False) as response: if response.status in TEST_VALID_STATUS: - self.redis.max(proxy) - logger.debug(f'proxy {proxy.string()} is valid, set max score') + if TEST_DONT_SET_MAX_SCORE: + logger.debug(f'proxy {proxy.string()} is valid, remain current score') + else: + self.redis.max(proxy) + logger.debug(f'proxy {proxy.string()} is valid, set max score') else: self.redis.decrease(proxy) logger.debug(f'proxy {proxy.string()} is invalid, decrease score') except EXCEPTIONS: self.redis.decrease(proxy) logger.debug(f'proxy {proxy.string()} is invalid, decrease score') - + @logger.catch def run(self): """ @@ -84,14 +87,15 @@ def run(self): if not cursor: break + def run_tester(): host = '96.113.165.182' port = '3128' tasks = [tester.test(Proxy(host=host, port=port))] tester.loop.run_until_complete(asyncio.wait(tasks)) + if __name__ == '__main__': tester = Tester() tester.run() # run_tester() - diff --git a/proxypool/setting.py b/proxypool/setting.py index e68f45ca..31c9ab3b 100644 --- a/proxypool/setting.py +++ b/proxypool/setting.py @@ -53,9 +53,9 @@ 'REDIS_KEY', 'proxies:universal')) # definition of proxy scores -PROXY_SCORE_MAX = 100 -PROXY_SCORE_MIN = 0 -PROXY_SCORE_INIT = 10 +PROXY_SCORE_MAX = env.int('PROXY_SCORE_MAX', 100) +PROXY_SCORE_MIN = env.int('PROXY_SCORE_MIN', 0) +PROXY_SCORE_INIT = env.int('PROXY_SCORE_INIT', 10) # definition of proxy number PROXY_NUMBER_MAX = 50000 @@ -77,6 +77,8 @@ # 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36', # }) TEST_VALID_STATUS = env.list('TEST_VALID_STATUS', [200, 206, 302]) +# whether to set max score when one proxy is tested valid +TEST_DONT_SET_MAX_SCORE = env.bool('TEST_DONT_SET_MAX_SCORE', False) # definition of api API_HOST = env.str('API_HOST', '0.0.0.0') From d12f897139fbf9c58e35e8b734d3c652c12a0c17 Mon Sep 17 00:00:00 2001 From: qinjin Date: Sat, 4 Nov 2023 00:54:47 +0800 Subject: [PATCH 3/6] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E8=8E=B7=E5=8F=96proxy?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E7=9A=84=E8=AE=A4=E8=AF=81header,=20API-KEY?= =?UTF-8?q?=E3=80=82=E5=8F=AF=E9=85=8D=E7=BD=AE=EF=BC=8C=E9=BB=98=E8=AE=A4?= =?UTF-8?q?=E4=B8=8D=E9=9C=80=E8=A6=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- proxypool/processors/server.py | 29 ++++++++++++++++++++++++++--- proxypool/setting.py | 4 ++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/proxypool/processors/server.py b/proxypool/processors/server.py index f7138c64..aa0500ac 100644 --- a/proxypool/processors/server.py +++ b/proxypool/processors/server.py @@ -1,7 +1,7 @@ -from flask import Flask, g +from flask import Flask, g, request from proxypool.storages.redis import RedisClient -from proxypool.setting import API_HOST, API_PORT, API_THREADED, IS_DEV - +from proxypool.setting import API_HOST, API_PORT, API_THREADED, API_KEY, IS_DEV +import functools __all__ = ['app'] @@ -10,6 +10,25 @@ app.debug = True +def auth_required(func): + @functools.wraps(func) + def decorator(*args, **kwargs): + # conditional decorator, when setting API_KEY is set, otherwise just ignore this decorator + if API_KEY == "": + return func(*args, **kwargs) + if request.headers.get('API-KEY', None) is not None: + api_key = request.headers.get('API-KEY') + else: + return {"message": "Please provide an API key in header"}, 400 + # Check if API key is correct and valid + if request.method == "GET" and api_key == API_KEY: + return func(*args, **kwargs) + else: + return {"message": "The provided API key is not valid"}, 403 + + return decorator + + def get_conn(): """ get redis client object @@ -21,6 +40,7 @@ def get_conn(): @app.route('/') +@auth_required def index(): """ get home page, you can define your own templates @@ -30,6 +50,7 @@ def index(): @app.route('/random') +@auth_required def get_proxy(): """ get a random proxy @@ -40,6 +61,7 @@ def get_proxy(): @app.route('/all') +@auth_required def get_proxy_all(): """ get a random proxy @@ -56,6 +78,7 @@ def get_proxy_all(): @app.route('/count') +@auth_required def get_count(): """ get the count of proxies diff --git a/proxypool/setting.py b/proxypool/setting.py index 31c9ab3b..495ad55f 100644 --- a/proxypool/setting.py +++ b/proxypool/setting.py @@ -84,6 +84,10 @@ API_HOST = env.str('API_HOST', '0.0.0.0') API_PORT = env.int('API_PORT', 5555) API_THREADED = env.bool('API_THREADED', True) +# add an api key to get proxy +# need a header of `API-KEY` in get request to pass the authenticate +# API_KEY='', do not need `API-KEY` header +API_KEY = env.str('API_KEY', '') # flags of enable ENABLE_TESTER = env.bool('ENABLE_TESTER', True) From 55b41aff3e0805e15946b7a1805469090cf147bc Mon Sep 17 00:00:00 2001 From: qinjin Date: Fri, 15 Mar 2024 22:58:33 +0800 Subject: [PATCH 4/6] add 3 parameter in settings, to get proxy IPs from /random api more flexible (#210) --- proxypool/processors/tester.py | 5 ++++- proxypool/setting.py | 7 +++++++ proxypool/storages/redis.py | 10 +++++----- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/proxypool/processors/tester.py b/proxypool/processors/tester.py index 58795285..15423672 100644 --- a/proxypool/processors/tester.py +++ b/proxypool/processors/tester.py @@ -4,7 +4,7 @@ from proxypool.schemas import Proxy from proxypool.storages.redis import RedisClient from proxypool.setting import TEST_TIMEOUT, TEST_BATCH, TEST_URL, TEST_VALID_STATUS, TEST_ANONYMOUS, \ - TEST_DONT_SET_MAX_SCORE + TEST_DONT_SET_MAX_SCORE, TEST_ALLWAYS_DECREASE_SCORE from aiohttp import ClientProxyConnectionError, ServerDisconnectedError, ClientOSError, ClientHttpProxyError from asyncio import TimeoutError @@ -57,6 +57,9 @@ async def test(self, proxy: Proxy): if response.status in TEST_VALID_STATUS: if TEST_DONT_SET_MAX_SCORE: logger.debug(f'proxy {proxy.string()} is valid, remain current score') + elif TEST_ALLWAYS_DECREASE_SCORE: + self.redis.decrease(proxy) + logger.debug(f'proxy {proxy.string()} is valid, but still decrease score') else: self.redis.max(proxy) logger.debug(f'proxy {proxy.string()} is valid, set max score') diff --git a/proxypool/setting.py b/proxypool/setting.py index 495ad55f..1ca20863 100644 --- a/proxypool/setting.py +++ b/proxypool/setting.py @@ -56,6 +56,10 @@ PROXY_SCORE_MAX = env.int('PROXY_SCORE_MAX', 100) PROXY_SCORE_MIN = env.int('PROXY_SCORE_MIN', 0) PROXY_SCORE_INIT = env.int('PROXY_SCORE_INIT', 10) +# highest/lowest proxy scores that /random api should get. +# default /random api get the max score proxy(if exists) or all proxies in pool +PROXY_SCORE_RAND_MAX = env.int('PROXY_SCORE_RAND_MAX', PROXY_SCORE_MAX) +PROXY_SCORE_RAND_MIN = env.int('PROXY_SCORE_RAND_MIN', PROXY_SCORE_MIN) # definition of proxy number PROXY_NUMBER_MAX = 50000 @@ -79,6 +83,9 @@ TEST_VALID_STATUS = env.list('TEST_VALID_STATUS', [200, 206, 302]) # whether to set max score when one proxy is tested valid TEST_DONT_SET_MAX_SCORE = env.bool('TEST_DONT_SET_MAX_SCORE', False) +# whether to decrease the score even when one proxy is tested valid +# use this param combined with proper PROXY_SCORE_XX params, can let you allways get the newest proxy with /random api +TEST_ALLWAYS_DECREASE_SCORE = env.bool('TEST_ALLWAYS_DECREASE_SCORE', False) # definition of api API_HOST = env.str('API_HOST', '0.0.0.0') diff --git a/proxypool/storages/redis.py b/proxypool/storages/redis.py index 8ab0e41d..04845bad 100644 --- a/proxypool/storages/redis.py +++ b/proxypool/storages/redis.py @@ -2,7 +2,7 @@ from proxypool.exceptions import PoolEmptyException from proxypool.schemas.proxy import Proxy from proxypool.setting import REDIS_CONNECTION_STRING, REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_DB, REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MIN, \ - PROXY_SCORE_INIT + PROXY_SCORE_INIT, PROXY_SCORE_RAND_MAX, PROXY_SCORE_RAND_MIN from random import choice from typing import List from loguru import logger @@ -52,19 +52,19 @@ def add(self, proxy: Proxy, score=PROXY_SCORE_INIT) -> int: def random(self) -> Proxy: """ get random proxy - firstly try to get proxy with max score - if not exists, try to get proxy by rank + firstly try to get proxy with score of PROXY_SCORE_RAND_MAX ~ PROXY_SCORE_MAX + if not exists, try to get proxy with score of PROXY_SCORE_RAND_MIN ~ PROXY_SCORE_MAX if not exists, raise error :return: proxy, like 8.8.8.8:8 """ # try to get proxy with max score proxies = self.db.zrangebyscore( - REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MAX) + REDIS_KEY, PROXY_SCORE_RAND_MAX, PROXY_SCORE_MAX) if len(proxies): return convert_proxy_or_proxies(choice(proxies)) # else get proxy by rank proxies = self.db.zrevrange( - REDIS_KEY, PROXY_SCORE_MIN, PROXY_SCORE_MAX) + REDIS_KEY, PROXY_SCORE_RAND_MIN, PROXY_SCORE_MAX) if len(proxies): return convert_proxy_or_proxies(choice(proxies)) # else raise error From a476e9dbb6aca0f74e29f2bf5275def325fe0a8f Mon Sep 17 00:00:00 2001 From: 15002136809 <15002136809@163.com> Date: Mon, 25 Mar 2024 20:43:06 +0800 Subject: [PATCH 5/6] Revert "add 3 parameter in settings, to get proxy IPs from /random api more flexible (#210)" This reverts commit 55b41aff3e0805e15946b7a1805469090cf147bc. --- proxypool/processors/tester.py | 5 +---- proxypool/setting.py | 7 ------- proxypool/storages/redis.py | 10 +++++----- 3 files changed, 6 insertions(+), 16 deletions(-) diff --git a/proxypool/processors/tester.py b/proxypool/processors/tester.py index 15423672..58795285 100644 --- a/proxypool/processors/tester.py +++ b/proxypool/processors/tester.py @@ -4,7 +4,7 @@ from proxypool.schemas import Proxy from proxypool.storages.redis import RedisClient from proxypool.setting import TEST_TIMEOUT, TEST_BATCH, TEST_URL, TEST_VALID_STATUS, TEST_ANONYMOUS, \ - TEST_DONT_SET_MAX_SCORE, TEST_ALLWAYS_DECREASE_SCORE + TEST_DONT_SET_MAX_SCORE from aiohttp import ClientProxyConnectionError, ServerDisconnectedError, ClientOSError, ClientHttpProxyError from asyncio import TimeoutError @@ -57,9 +57,6 @@ async def test(self, proxy: Proxy): if response.status in TEST_VALID_STATUS: if TEST_DONT_SET_MAX_SCORE: logger.debug(f'proxy {proxy.string()} is valid, remain current score') - elif TEST_ALLWAYS_DECREASE_SCORE: - self.redis.decrease(proxy) - logger.debug(f'proxy {proxy.string()} is valid, but still decrease score') else: self.redis.max(proxy) logger.debug(f'proxy {proxy.string()} is valid, set max score') diff --git a/proxypool/setting.py b/proxypool/setting.py index 1ca20863..495ad55f 100644 --- a/proxypool/setting.py +++ b/proxypool/setting.py @@ -56,10 +56,6 @@ PROXY_SCORE_MAX = env.int('PROXY_SCORE_MAX', 100) PROXY_SCORE_MIN = env.int('PROXY_SCORE_MIN', 0) PROXY_SCORE_INIT = env.int('PROXY_SCORE_INIT', 10) -# highest/lowest proxy scores that /random api should get. -# default /random api get the max score proxy(if exists) or all proxies in pool -PROXY_SCORE_RAND_MAX = env.int('PROXY_SCORE_RAND_MAX', PROXY_SCORE_MAX) -PROXY_SCORE_RAND_MIN = env.int('PROXY_SCORE_RAND_MIN', PROXY_SCORE_MIN) # definition of proxy number PROXY_NUMBER_MAX = 50000 @@ -83,9 +79,6 @@ TEST_VALID_STATUS = env.list('TEST_VALID_STATUS', [200, 206, 302]) # whether to set max score when one proxy is tested valid TEST_DONT_SET_MAX_SCORE = env.bool('TEST_DONT_SET_MAX_SCORE', False) -# whether to decrease the score even when one proxy is tested valid -# use this param combined with proper PROXY_SCORE_XX params, can let you allways get the newest proxy with /random api -TEST_ALLWAYS_DECREASE_SCORE = env.bool('TEST_ALLWAYS_DECREASE_SCORE', False) # definition of api API_HOST = env.str('API_HOST', '0.0.0.0') diff --git a/proxypool/storages/redis.py b/proxypool/storages/redis.py index 04845bad..8ab0e41d 100644 --- a/proxypool/storages/redis.py +++ b/proxypool/storages/redis.py @@ -2,7 +2,7 @@ from proxypool.exceptions import PoolEmptyException from proxypool.schemas.proxy import Proxy from proxypool.setting import REDIS_CONNECTION_STRING, REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_DB, REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MIN, \ - PROXY_SCORE_INIT, PROXY_SCORE_RAND_MAX, PROXY_SCORE_RAND_MIN + PROXY_SCORE_INIT from random import choice from typing import List from loguru import logger @@ -52,19 +52,19 @@ def add(self, proxy: Proxy, score=PROXY_SCORE_INIT) -> int: def random(self) -> Proxy: """ get random proxy - firstly try to get proxy with score of PROXY_SCORE_RAND_MAX ~ PROXY_SCORE_MAX - if not exists, try to get proxy with score of PROXY_SCORE_RAND_MIN ~ PROXY_SCORE_MAX + firstly try to get proxy with max score + if not exists, try to get proxy by rank if not exists, raise error :return: proxy, like 8.8.8.8:8 """ # try to get proxy with max score proxies = self.db.zrangebyscore( - REDIS_KEY, PROXY_SCORE_RAND_MAX, PROXY_SCORE_MAX) + REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MAX) if len(proxies): return convert_proxy_or_proxies(choice(proxies)) # else get proxy by rank proxies = self.db.zrevrange( - REDIS_KEY, PROXY_SCORE_RAND_MIN, PROXY_SCORE_MAX) + REDIS_KEY, PROXY_SCORE_MIN, PROXY_SCORE_MAX) if len(proxies): return convert_proxy_or_proxies(choice(proxies)) # else raise error From 66c77d21c4baa88af2a56a4981e86c331812f158 Mon Sep 17 00:00:00 2001 From: qinjin Date: Mon, 25 Mar 2024 21:10:00 +0800 Subject: [PATCH 6/6] add sub proxy pool mechanics. with an individual tester class provided, it can test and store the available proxy ip for a specific URL, and provide an api. --- proxypool/processors/getter.py | 5 +++- proxypool/processors/server.py | 21 +++++++++++--- proxypool/processors/tester.py | 28 +++++++++++++++++++ proxypool/setting.py | 2 ++ proxypool/storages/redis.py | 50 +++++++++++++++++----------------- proxypool/testers/__init__.py | 16 +++++++++++ proxypool/testers/base.py | 19 +++++++++++++ 7 files changed, 111 insertions(+), 30 deletions(-) create mode 100644 proxypool/testers/__init__.py create mode 100644 proxypool/testers/base.py diff --git a/proxypool/processors/getter.py b/proxypool/processors/getter.py index 877e198a..c5c16296 100644 --- a/proxypool/processors/getter.py +++ b/proxypool/processors/getter.py @@ -2,7 +2,7 @@ from proxypool.storages.redis import RedisClient from proxypool.setting import PROXY_NUMBER_MAX from proxypool.crawlers import __all__ as crawlers_cls - +from proxypool.testers import __all__ as testers_cls class Getter(object): """ @@ -16,6 +16,8 @@ def __init__(self): self.redis = RedisClient() self.crawlers_cls = crawlers_cls self.crawlers = [crawler_cls() for crawler_cls in self.crawlers_cls] + self.testers_cls = testers_cls + self.testers = [tester_cls() for tester_cls in self.testers_cls] def is_full(self): """ @@ -36,6 +38,7 @@ def run(self): logger.info(f'crawler {crawler} to get proxy') for proxy in crawler.crawl(): self.redis.add(proxy) + [self.redis.add(proxy, redis_key=tester.key) for tester in self.testers] if __name__ == '__main__': diff --git a/proxypool/processors/server.py b/proxypool/processors/server.py index aa0500ac..50144590 100644 --- a/proxypool/processors/server.py +++ b/proxypool/processors/server.py @@ -1,6 +1,7 @@ from flask import Flask, g, request +from proxypool.exceptions import PoolEmptyException from proxypool.storages.redis import RedisClient -from proxypool.setting import API_HOST, API_PORT, API_THREADED, API_KEY, IS_DEV +from proxypool.setting import API_HOST, API_PORT, API_THREADED, API_KEY, IS_DEV, PROXY_RAND_KEY_DEGRADED import functools __all__ = ['app'] @@ -53,10 +54,19 @@ def index(): @auth_required def get_proxy(): """ - get a random proxy + get a random proxy, can query the specific sub-pool according the (redis) key + if PROXY_RAND_KEY_DEGRADED is set to True, will get a universal random proxy if no proxy found in the sub-pool :return: get a random proxy """ + key = request.args.get('key') conn = get_conn() + # return conn.random(key).string() if key else conn.random().string() + if key: + try: + return conn.random(key).string() + except PoolEmptyException: + if not PROXY_RAND_KEY_DEGRADED: + raise return conn.random().string() @@ -67,8 +77,10 @@ def get_proxy_all(): get a random proxy :return: get a random proxy """ + key = request.args.get('key') + conn = get_conn() - proxies = conn.all() + proxies = conn.all(key) if key else conn.all() proxies_string = '' if proxies: for proxy in proxies: @@ -85,7 +97,8 @@ def get_count(): :return: count, int """ conn = get_conn() - return str(conn.count()) + key = request.args.get('key') + return str(conn.count(key)) if key else conn.count() if __name__ == '__main__': diff --git a/proxypool/processors/tester.py b/proxypool/processors/tester.py index 58795285..470259a9 100644 --- a/proxypool/processors/tester.py +++ b/proxypool/processors/tester.py @@ -7,6 +7,7 @@ TEST_DONT_SET_MAX_SCORE from aiohttp import ClientProxyConnectionError, ServerDisconnectedError, ClientOSError, ClientHttpProxyError from asyncio import TimeoutError +from proxypool.testers import __all__ as testers_cls EXCEPTIONS = ( ClientProxyConnectionError, @@ -30,6 +31,8 @@ def __init__(self): """ self.redis = RedisClient() self.loop = asyncio.get_event_loop() + self.testers_cls = testers_cls + self.testers = [tester_cls() for tester_cls in self.testers_cls] async def test(self, proxy: Proxy): """ @@ -63,8 +66,33 @@ async def test(self, proxy: Proxy): else: self.redis.decrease(proxy) logger.debug(f'proxy {proxy.string()} is invalid, decrease score') + # if independent tester class found, create new set of storage and do the extra test + for tester in self.testers: + key = tester.key + if self.redis.exists(proxy, key): + test_url = tester.test_url + headers = tester.headers() + cookies = tester.cookies() + async with session.get(test_url, proxy=f'http://{proxy.string()}', + timeout=TEST_TIMEOUT, + headers=headers, + cookies=cookies, + allow_redirects=False) as response: + resp_text = await response.text() + is_valid = await tester.parse(resp_text, test_url, proxy.string()) + if is_valid: + if tester.test_dont_set_max_score: + logger.info(f'key[{key}] proxy {proxy.string()} is valid, remain current score') + else: + self.redis.max(proxy, key, tester.proxy_score_max) + logger.info(f'key[{key}] proxy {proxy.string()} is valid, set max score') + else: + self.redis.decrease(proxy, tester.key, tester.proxy_score_min) + logger.info(f'key[{key}] proxy {proxy.string()} is invalid, decrease score') + except EXCEPTIONS: self.redis.decrease(proxy) + [self.redis.decrease(proxy, tester.key, tester.proxy_score_min) for tester in self.testers] logger.debug(f'proxy {proxy.string()} is invalid, decrease score') @logger.catch diff --git a/proxypool/setting.py b/proxypool/setting.py index 495ad55f..a445667e 100644 --- a/proxypool/setting.py +++ b/proxypool/setting.py @@ -56,6 +56,8 @@ PROXY_SCORE_MAX = env.int('PROXY_SCORE_MAX', 100) PROXY_SCORE_MIN = env.int('PROXY_SCORE_MIN', 0) PROXY_SCORE_INIT = env.int('PROXY_SCORE_INIT', 10) +# whether to get a universal random proxy if no proxy exists in the sub-pool identified by a specific key +PROXY_RAND_KEY_DEGRADED = env.bool('TEST_ANONYMOUS', True) # definition of proxy number PROXY_NUMBER_MAX = 50000 diff --git a/proxypool/storages/redis.py b/proxypool/storages/redis.py index 8ab0e41d..2d052323 100644 --- a/proxypool/storages/redis.py +++ b/proxypool/storages/redis.py @@ -34,7 +34,7 @@ def __init__(self, host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD, db self.db = redis.StrictRedis( host=host, port=port, password=password, db=db, decode_responses=True, **kwargs) - def add(self, proxy: Proxy, score=PROXY_SCORE_INIT) -> int: + def add(self, proxy: Proxy, score=PROXY_SCORE_INIT, redis_key=REDIS_KEY) -> int: """ add proxy and set it to init score :param proxy: proxy, ip:port, like 8.8.8.8:88 @@ -44,12 +44,12 @@ def add(self, proxy: Proxy, score=PROXY_SCORE_INIT) -> int: if not is_valid_proxy(f'{proxy.host}:{proxy.port}'): logger.info(f'invalid proxy {proxy}, throw it') return - if not self.exists(proxy): + if not self.exists(proxy, redis_key): if IS_REDIS_VERSION_2: - return self.db.zadd(REDIS_KEY, score, proxy.string()) - return self.db.zadd(REDIS_KEY, {proxy.string(): score}) + return self.db.zadd(redis_key, score, proxy.string()) + return self.db.zadd(redis_key, {proxy.string(): score}) - def random(self) -> Proxy: + def random(self, redis_key=REDIS_KEY, proxy_score_min=PROXY_SCORE_MIN, proxy_score_max=PROXY_SCORE_MAX) -> Proxy: """ get random proxy firstly try to get proxy with max score @@ -59,74 +59,74 @@ def random(self) -> Proxy: """ # try to get proxy with max score proxies = self.db.zrangebyscore( - REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MAX) + redis_key, proxy_score_max, proxy_score_max) if len(proxies): return convert_proxy_or_proxies(choice(proxies)) # else get proxy by rank proxies = self.db.zrevrange( - REDIS_KEY, PROXY_SCORE_MIN, PROXY_SCORE_MAX) + redis_key, proxy_score_min, proxy_score_max) if len(proxies): return convert_proxy_or_proxies(choice(proxies)) # else raise error raise PoolEmptyException - def decrease(self, proxy: Proxy) -> int: + def decrease(self, proxy: Proxy, redis_key=REDIS_KEY, proxy_score_min=PROXY_SCORE_MIN) -> int: """ decrease score of proxy, if small than PROXY_SCORE_MIN, delete it :param proxy: proxy :return: new score """ if IS_REDIS_VERSION_2: - self.db.zincrby(REDIS_KEY, proxy.string(), -1) + self.db.zincrby(redis_key, proxy.string(), -1) else: - self.db.zincrby(REDIS_KEY, -1, proxy.string()) - score = self.db.zscore(REDIS_KEY, proxy.string()) + self.db.zincrby(redis_key, -1, proxy.string()) + score = self.db.zscore(redis_key, proxy.string()) logger.info(f'{proxy.string()} score decrease 1, current {score}') - if score <= PROXY_SCORE_MIN: + if score <= proxy_score_min: logger.info(f'{proxy.string()} current score {score}, remove') - self.db.zrem(REDIS_KEY, proxy.string()) + self.db.zrem(redis_key, proxy.string()) - def exists(self, proxy: Proxy) -> bool: + def exists(self, proxy: Proxy, redis_key=REDIS_KEY) -> bool: """ if proxy exists :param proxy: proxy :return: if exists, bool """ - return not self.db.zscore(REDIS_KEY, proxy.string()) is None + return not self.db.zscore(redis_key, proxy.string()) is None - def max(self, proxy: Proxy) -> int: + def max(self, proxy: Proxy, redis_key=REDIS_KEY, proxy_score_max=PROXY_SCORE_MAX) -> int: """ set proxy to max score :param proxy: proxy :return: new score """ - logger.info(f'{proxy.string()} is valid, set to {PROXY_SCORE_MAX}') + logger.info(f'{proxy.string()} is valid, set to {proxy_score_max}') if IS_REDIS_VERSION_2: - return self.db.zadd(REDIS_KEY, PROXY_SCORE_MAX, proxy.string()) - return self.db.zadd(REDIS_KEY, {proxy.string(): PROXY_SCORE_MAX}) + return self.db.zadd(redis_key, proxy_score_max, proxy.string()) + return self.db.zadd(redis_key, {proxy.string(): proxy_score_max}) - def count(self) -> int: + def count(self, redis_key=REDIS_KEY) -> int: """ get count of proxies :return: count, int """ - return self.db.zcard(REDIS_KEY) + return self.db.zcard(redis_key) - def all(self) -> List[Proxy]: + def all(self, redis_key=REDIS_KEY, proxy_score_min=PROXY_SCORE_MIN, proxy_score_max=PROXY_SCORE_MAX) -> List[Proxy]: """ get all proxies :return: list of proxies """ - return convert_proxy_or_proxies(self.db.zrangebyscore(REDIS_KEY, PROXY_SCORE_MIN, PROXY_SCORE_MAX)) + return convert_proxy_or_proxies(self.db.zrangebyscore(redis_key, proxy_score_min, proxy_score_max)) - def batch(self, cursor, count) -> List[Proxy]: + def batch(self, cursor, count, redis_key=REDIS_KEY) -> List[Proxy]: """ get batch of proxies :param cursor: scan cursor :param count: scan count :return: list of proxies """ - cursor, proxies = self.db.zscan(REDIS_KEY, cursor, count=count) + cursor, proxies = self.db.zscan(redis_key, cursor, count=count) return cursor, convert_proxy_or_proxies([i[0] for i in proxies]) diff --git a/proxypool/testers/__init__.py b/proxypool/testers/__init__.py new file mode 100644 index 00000000..4e4df95e --- /dev/null +++ b/proxypool/testers/__init__.py @@ -0,0 +1,16 @@ +import pkgutil +from .base import BaseTester +import inspect + + +# load classes subclass of BaseCrawler +classes = [] +for loader, name, is_pkg in pkgutil.walk_packages(__path__): + module = loader.find_module(name).load_module(name) + for name, value in inspect.getmembers(module): + globals()[name] = value + if inspect.isclass(value) and issubclass(value, BaseTester) and value is not BaseTester \ + and not getattr(value, 'ignore', False): + classes.append(value) +__all__ = __ALL__ = classes + diff --git a/proxypool/testers/base.py b/proxypool/testers/base.py new file mode 100644 index 00000000..796b7cfc --- /dev/null +++ b/proxypool/testers/base.py @@ -0,0 +1,19 @@ +from proxypool.setting import TEST_DONT_SET_MAX_SCORE, PROXY_SCORE_INIT, PROXY_SCORE_MAX, PROXY_SCORE_MIN + + +class BaseTester(object): + test_url = "" + key = "" + test_dont_set_max_score = TEST_DONT_SET_MAX_SCORE + proxy_score_init = PROXY_SCORE_INIT + proxy_score_max = PROXY_SCORE_MAX + proxy_score_min = PROXY_SCORE_MIN + + def headers(self): + return None + + def cookies(self): + return None + + async def parse(self, html, url, proxy, expr='{"code":0'): + return True if expr in html else False