diff --git a/Dockerfile b/Dockerfile index c5ca5440..0de17573 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,16 @@ -FROM python:3.7-alpine AS build +FROM python:3.11-slim AS build COPY requirements.txt . -RUN apk update &&\ - apk add --no-cache gcc g++ libffi-dev openssl-dev libxml2-dev libxslt-dev build-base musl-dev &&\ +RUN apt-get update &&\ + apt-get install -y --no-install-recommends gcc g++ libxml2-dev libxslt1-dev &&\ pip install -U pip &&\ - pip install --timeout 30 --user --no-cache-dir --no-warn-script-location -r requirements.txt + pip install --timeout 60 --user --no-cache-dir --no-warn-script-location -r requirements.txt &&\ + rm -rf /var/lib/apt/lists/* -FROM python:3.7-alpine +FROM python:3.11-slim ENV APP_ENV=prod ENV LOCAL_PKG="/root/.local" COPY --from=build ${LOCAL_PKG} ${LOCAL_PKG} -RUN apk update && apk add --no-cache libffi-dev openssl-dev libxslt-dev &&\ - ln -sf ${LOCAL_PKG}/bin/* /usr/local/bin/ +RUN ln -sf ${LOCAL_PKG}/bin/* /usr/local/bin/ WORKDIR /app COPY . . EXPOSE 5555 diff --git a/README.md b/README.md index d01d7a08..4b6d4694 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,18 @@ 代理池原理解析可见「[如何搭建一个高效的代理池](https://cuiqingcai.com/7048.html)」,建议使用之前阅读。 +## 使用前注意 + +本代理池是基于市面上各种公开代理源搭建的,所以可用性并不高,很可能上百上千个代理中才能找到一两个可用代理,不适合直接用于爬虫爬取任务。 + +如果您的目的是为了尽快使用代理完成爬取任务,建议您对接一些付费代理或者直接使用已有代理资源;如果您的目的是为了学习如何搭建一个代理池,您可以参考本项目继续完成后续步骤。 + +付费代理推荐: + +- [ADSL 拨号代理](https://platform.acedata.cloud/documents/a82a528a-8e32-4c4c-a9d0-a21be7c9ef8c):海量拨号(中国境内)高质量代理 +- [海外/全球代理](https://platform.acedata.cloud/documents/50f1437a-1857-43c5-85cf-5800ae1b31e4):中国境外高质量代理 +- [蜂窝 4G/5G 代理](https://platform.acedata.cloud/documents/1cc59b19-1550-4169-a59d-ad6faf7f7517):极高质量(中国境内)防风控代理 + ## 使用准备 首先当然是克隆代码并进入 ProxyPool 文件夹: @@ -74,12 +86,6 @@ proxypool | 2020-02-19 17:09:46,596 INFO success: tester entered RUNNING stat 这时候访问 [http://localhost:5555/random](http://localhost:5555/random) 即可获取一个随机可用代理。 -当然你也可以选择自己 Build,直接运行如下命令即可: - -``` -docker-compose -f build.yaml up -``` - 如果下载速度特别慢,可以自行修改 Dockerfile,修改: ```diff @@ -216,6 +222,27 @@ get random proxy 116.196.115.209:8080 可以看到成功获取了代理,并请求 httpbin.org 验证了代理的可用性。 +### 获取多个代理 + +如果一次需要多个代理,可以给 `/random` 接口传入 `count` 参数,一次返回多个随机代理(每行一个): + +``` +GET http://localhost:5555/random?count=5 +``` + +`count` 不传或为 1 时行为不变,仍返回单个代理;`count` 大于可用数量时返回全部可用代理。也可与 `key` 参数组合使用。 + +### 按地区(国家)筛选代理 + +可以给 `/random` 和 `/all` 接口传入 `area` 参数,按代理 IP 所属国家筛选(ISO 国家码,大小写不敏感),例如只获取国内(中国)代理: + +``` +GET http://localhost:5555/random?area=CN +GET http://localhost:5555/all?area=CN +``` + +国家信息由内置的 GeoLite2 离线库解析,无法解析归属地的代理会被排除。`area` 可与 `count`、`key` 参数组合使用。 + ## 可配置项 代理池可以通过设置环境变量来配置一些参数。 @@ -250,9 +277,14 @@ get random proxy 116.196.115.209:8080 - TEST_TIMEOUT:测试超时时间,默认 10 秒 - TEST_BATCH:批量测试数量,默认 20 个代理 - TEST_VALID_STATUS:测试有效的状态码 +- TEST_ANONYMOUS:是否只保留匿名代理,默认 true +- TEST_ANONYMOUS_URL:匿名 / 出口 IP 检测地址,默认 `https://httpbin.org/ip`,需返回 httpbin 格式的 JSON(`{"origin": "1.2.3.4"}`)。可指向自建 httpbin 服务以避免公共服务限流 - API_HOST:代理 Server 运行 Host,默认 0.0.0.0 - API_PORT:代理 Server 运行端口,默认 5555 - API_THREADED:代理 Server 是否使用多线程,默认 true +- API_KEY:API 访问鉴权密钥,默认空(即不鉴权)。设置后,调用 `/random`、`/all`、`/count` 需在请求头携带 `API-KEY`,详见下方「安全性」说明 + +> ⚠️ 安全提示:代理 Server 默认监听 `0.0.0.0` 且 `API_KEY` 默认为空,任何能访问该端口的人都可以调用 `/random`、`/all`、`/count`。如果将代理池**暴露到公网**,请务必设置 `API_KEY`,并配合防火墙/安全组限制来源。`key` 查询参数已做格式校验,仅允许字母、数字及 `_ : -`,最长 64 位。 ### 日志 @@ -347,11 +379,6 @@ class Daili66Crawler(BaseCrawler): 本项目提供了 Kubernetes 部署脚本,如需部署到 Kubernetes,请参考 [kubernetes](./kubernetes)。 -## 待开发 - -- [ ] 前端页面管理 -- [ ] 使用情况统计分析 - 如有一起开发的兴趣可以在 Issue 留言,非常感谢! ## LICENSE diff --git a/build.yaml b/build.yaml deleted file mode 100644 index 74b2fd0b..00000000 --- a/build.yaml +++ /dev/null @@ -1,18 +0,0 @@ -version: "3" -services: - redis4proxypool: - image: redis:alpine - container_name: redis4proxypool - ports: - - "6374:6379" - proxypool: - build: . - image: "germey/proxypool:master" - container_name: proxypool - ports: - - "5555:5555" - restart: always - # volumes: - # - proxypool/crawlers/private:/app/proxypool/crawlers/private - environment: - PROXYPOOL_REDIS_CONNECTION_STRING: redis://@redis4proxypool:6379/0 diff --git a/docker-compose.yml b/docker-compose.yml index cf367f42..4e4d5936 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,16 +3,15 @@ services: redis4proxypool: image: redis:alpine container_name: redis4proxypool - # ports: - # - "6374:6379" proxypool: + build: . image: "germey/proxypool:master" container_name: proxypool ports: - "5555:5555" restart: always # volumes: - # - proxypool/crawlers/private:/app/proxypool/crawlers/private + # - proxypool/crawlers/private:~/proxypool/crawlers/private environment: PROXYPOOL_REDIS_HOST: redis4proxypool diff --git a/proxypool/crawlers/public/docip.py b/proxypool/crawlers/public/docip.py new file mode 100644 index 00000000..154871fb --- /dev/null +++ b/proxypool/crawlers/public/docip.py @@ -0,0 +1,38 @@ +import time +from retrying import RetryError +from loguru import logger +from proxypool.schemas.proxy import Proxy +from proxypool.crawlers.base import BaseCrawler +import json + +BASE_URL = 'https://www.docip.net/data/free.json?t={date}' + + + +class DocipCrawler(BaseCrawler): + """ + Docip crawler, https://www.docip.net/data/free.json + """ + urls = [BASE_URL.format(date=time.strftime("%Y%m%d", time.localtime()))] + + def parse(self, html): + """ + parse html file to get proxies + :return: + """ + try: + result = json.loads(html) + proxy_list = result['data'] + for proxy_item in proxy_list: + host = proxy_item['ip'] + port = host.split(':')[-1] + yield Proxy(host=host, port=port) + except json.JSONDecodeError: + print("json.JSONDecodeError") + return + + +if __name__ == '__main__': + crawler = DocipCrawler() + for proxy in crawler.crawl(): + print(proxy) diff --git a/proxypool/crawlers/public/uqidata.py b/proxypool/crawlers/public/uqidata.py new file mode 100644 index 00000000..3e54b2dc --- /dev/null +++ b/proxypool/crawlers/public/uqidata.py @@ -0,0 +1,49 @@ +from pyquery import PyQuery as pq +from proxypool.schemas.proxy import Proxy +from proxypool.crawlers.base import BaseCrawler +from loguru import logger + +BASE_URL = 'https://ip.uqidata.com/free/index.html' + + +class UqidataCrawler(BaseCrawler): + """ + Uqidata crawler, https://ip.uqidata.com/free/index.html + """ + urls = [BASE_URL] + ignore = True + + def encode(input_str): + tmp = [] + for i in range(len(input_str)): + tmp.append("ABCDEFGHIZ".find(input_str[i])) + result = "".join(str(i) for i in tmp) + result = int(result) >> 0x03 + return result + + def parse(self, html): + """ + parse html file to get proxies + :return: + """ + doc = pq(html) + trs = doc('#main_container .inner table tbody tr:nth-child(n+3)').items() + for tr in trs: + ip_html = tr('td.ip').find("*").items() + host = '' + for i in ip_html: + if i.attr('style') is not None and 'none' in i.attr('style'): + continue + if i.text() == '': + continue + host += i.text() + + port_code = tr('td.port').attr('class').split(' ')[1] + port = UqidataCrawler.encode(port_code) + yield Proxy(host=host, port=port) + + +if __name__ == '__main__': + crawler = UqidataCrawler() + for proxy in crawler.crawl(): + print(proxy) diff --git a/proxypool/processors/getter.py b/proxypool/processors/getter.py index 877e198a..c5c16296 100644 --- a/proxypool/processors/getter.py +++ b/proxypool/processors/getter.py @@ -2,7 +2,7 @@ from proxypool.storages.redis import RedisClient from proxypool.setting import PROXY_NUMBER_MAX from proxypool.crawlers import __all__ as crawlers_cls - +from proxypool.testers import __all__ as testers_cls class Getter(object): """ @@ -16,6 +16,8 @@ def __init__(self): self.redis = RedisClient() self.crawlers_cls = crawlers_cls self.crawlers = [crawler_cls() for crawler_cls in self.crawlers_cls] + self.testers_cls = testers_cls + self.testers = [tester_cls() for tester_cls in self.testers_cls] def is_full(self): """ @@ -36,6 +38,7 @@ def run(self): logger.info(f'crawler {crawler} to get proxy') for proxy in crawler.crawl(): self.redis.add(proxy) + [self.redis.add(proxy, redis_key=tester.key) for tester in self.testers] if __name__ == '__main__': diff --git a/proxypool/processors/server.py b/proxypool/processors/server.py index f7138c64..f95f21dc 100644 --- a/proxypool/processors/server.py +++ b/proxypool/processors/server.py @@ -1,7 +1,12 @@ -from flask import Flask, g +import hmac +import re +from flask import Flask, g, request, abort +from proxypool.exceptions import PoolEmptyException from proxypool.storages.redis import RedisClient -from proxypool.setting import API_HOST, API_PORT, API_THREADED, IS_DEV - +from proxypool.setting import API_HOST, API_PORT, API_THREADED, API_KEY, IS_DEV, PROXY_RAND_KEY_DEGRADED +import functools +from random import choice, sample +from proxypool.utils.geo import get_country_iso __all__ = ['app'] @@ -9,6 +14,29 @@ if IS_DEV: app.debug = True +# allowed characters for the `key` query parameter that selects a redis sub-pool; +# restricts to a safe charset to avoid probing arbitrary redis keys via the API +VALID_KEY_PATTERN = re.compile(r'^[a-zA-Z0-9_:\-]{1,64}$') + + +def auth_required(func): + @functools.wraps(func) + def decorator(*args, **kwargs): + # conditional decorator, when setting API_KEY is set, otherwise just ignore this decorator + if API_KEY == "": + return func(*args, **kwargs) + if request.headers.get('API-KEY', None) is not None: + api_key = request.headers.get('API-KEY') + else: + return {"message": "Please provide an API key in header"}, 400 + # Check if API key is correct and valid + if request.method == "GET" and hmac.compare_digest(api_key, API_KEY): + return func(*args, **kwargs) + else: + return {"message": "The provided API key is not valid"}, 403 + + return decorator + def get_conn(): """ @@ -20,7 +48,34 @@ def get_conn(): return g.redis +def get_request_key(): + """ + read the `key` query parameter and validate its format; + reject unexpected characters to avoid redis key probing/injection + :return: validated key or None + """ + key = request.args.get('key') + if key and not VALID_KEY_PATTERN.match(key): + abort(400, description='invalid key parameter') + return key + + +def filter_proxies_by_area(proxies, area): + """ + filter proxies by country iso code (e.g. 'CN', 'US'), case-insensitive; + proxies whose country cannot be resolved are excluded + :param proxies: list of Proxy + :param area: country iso code, or falsy to skip filtering + :return: filtered list of Proxy + """ + if not area: + return proxies + area = area.upper() + return [proxy for proxy in proxies if get_country_iso(proxy.host) == area] + + @app.route('/') +@auth_required def index(): """ get home page, you can define your own templates @@ -30,23 +85,64 @@ def index(): @app.route('/random') +@auth_required def get_proxy(): """ - get a random proxy + get a random proxy, can query the specific sub-pool according the (redis) key + if PROXY_RAND_KEY_DEGRADED is set to True, will get a universal random proxy if no proxy found in the sub-pool + can pass a `count` parameter to get multiple random proxies at once + can pass an `area` parameter to only get proxies from a country (iso code, e.g. CN) :return: get a random proxy """ + key = get_request_key() + count = request.args.get('count', type=int) + area = request.args.get('area') conn = get_conn() + # return conn.random(key).string() if key else conn.random().string() + if area: + # area filtering needs the candidate set first, then filter by country + candidates = conn.all(key) if key else conn.all() + candidates = filter_proxies_by_area(candidates, area) + if not candidates and key and PROXY_RAND_KEY_DEGRADED: + candidates = filter_proxies_by_area(conn.all(), area) + if not candidates: + raise PoolEmptyException + if count and count > 1: + count = min(count, len(candidates)) + return '\n'.join(proxy.string() for proxy in sample(candidates, count)) + return choice(candidates).string() + if count and count > 1: + # return multiple random proxies, one per line + try: + proxies = conn.randoms(count, key) if key else conn.randoms(count) + except PoolEmptyException: + if key and PROXY_RAND_KEY_DEGRADED: + proxies = conn.randoms(count) + else: + raise + return '\n'.join(proxy.string() for proxy in proxies) + if key: + try: + return conn.random(key).string() + except PoolEmptyException: + if not PROXY_RAND_KEY_DEGRADED: + raise return conn.random().string() @app.route('/all') +@auth_required def get_proxy_all(): """ - get a random proxy - :return: get a random proxy + get all proxies, optionally filtered by `area` (country iso code, e.g. CN) + :return: all proxies """ + key = get_request_key() + area = request.args.get('area') + conn = get_conn() - proxies = conn.all() + proxies = conn.all(key) if key else conn.all() + proxies = filter_proxies_by_area(proxies, area) proxies_string = '' if proxies: for proxy in proxies: @@ -56,13 +152,15 @@ def get_proxy_all(): @app.route('/count') +@auth_required def get_count(): """ get the count of proxies :return: count, int """ conn = get_conn() - return str(conn.count()) + key = get_request_key() + return str(conn.count(key)) if key else str(conn.count()) if __name__ == '__main__': diff --git a/proxypool/processors/tester.py b/proxypool/processors/tester.py index f002056a..5d70d238 100644 --- a/proxypool/processors/tester.py +++ b/proxypool/processors/tester.py @@ -3,10 +3,12 @@ from loguru import logger from proxypool.schemas import Proxy from proxypool.storages.redis import RedisClient -from proxypool.setting import TEST_TIMEOUT, TEST_BATCH, TEST_URL, TEST_VALID_STATUS, TEST_ANONYMOUS -from aiohttp import ClientProxyConnectionError, ServerDisconnectedError, ClientOSError, ClientHttpProxyError +from proxypool.setting import TEST_TIMEOUT, TEST_BATCH, TEST_URL, TEST_VALID_STATUS, TEST_ANONYMOUS, \ + TEST_DONT_SET_MAX_SCORE, TEST_ANONYMOUS_URL +from aiohttp import ClientProxyConnectionError, ServerDisconnectedError, ClientOSError, ClientHttpProxyError, \ + ClientResponseError, ContentTypeError from asyncio import TimeoutError - +from proxypool.testers import __all__ as testers_cls EXCEPTIONS = ( ClientProxyConnectionError, @@ -15,6 +17,8 @@ ServerDisconnectedError, ClientOSError, ClientHttpProxyError, + ClientResponseError, + ContentTypeError, AssertionError ) @@ -23,47 +27,109 @@ class Tester(object): """ tester for testing proxies in queue """ - + def __init__(self): """ init redis """ self.redis = RedisClient() - self.loop = asyncio.get_event_loop() - - async def test(self, proxy: Proxy): + self.testers_cls = testers_cls + self.testers = [tester_cls() for tester_cls in self.testers_cls] + + async def test(self, proxy: Proxy, session: aiohttp.ClientSession): """ test single proxy :param proxy: Proxy object + :param session: shared aiohttp session :return: """ - async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=False)) as session: - try: - logger.debug(f'testing {proxy.string()}') - # if TEST_ANONYMOUS is True, make sure that - # the proxy has the effect of hiding the real IP - if TEST_ANONYMOUS: - url = 'https://httpbin.org/ip' - async with session.get(url, timeout=TEST_TIMEOUT) as response: - resp_json = await response.json() - origin_ip = resp_json['origin'] - async with session.get(url, proxy=f'http://{proxy.string()}', timeout=TEST_TIMEOUT) as response: - resp_json = await response.json() - anonymous_ip = resp_json['origin'] - assert origin_ip != anonymous_ip - assert proxy.host == anonymous_ip - async with session.get(TEST_URL, proxy=f'http://{proxy.string()}', timeout=TEST_TIMEOUT, - allow_redirects=False) as response: - if response.status in TEST_VALID_STATUS: - self.redis.max(proxy) - logger.debug(f'proxy {proxy.string()} is valid, set max score') + try: + logger.debug(f'testing {proxy.string()}') + # if TEST_ANONYMOUS is True, make sure that + # the proxy has the effect of hiding the real IP + # logger.debug(f'TEST_ANONYMOUS {TEST_ANONYMOUS}') + if TEST_ANONYMOUS: + url = TEST_ANONYMOUS_URL + async with session.get(url, timeout=TEST_TIMEOUT) as response: + resp_json = await response.json() + origin_ip = resp_json['origin'] + # logger.debug(f'origin ip is {origin_ip}') + async with session.get(url, proxy=f'http://{proxy.string()}', timeout=TEST_TIMEOUT) as response: + resp_json = await response.json() + anonymous_ip = resp_json['origin'] + logger.debug(f'anonymous ip is {anonymous_ip}') + assert origin_ip != anonymous_ip + assert proxy.host == anonymous_ip + async with session.get(TEST_URL, proxy=f'http://{proxy.string()}', timeout=TEST_TIMEOUT, + allow_redirects=False) as response: + if response.status in TEST_VALID_STATUS: + if TEST_DONT_SET_MAX_SCORE: + logger.debug( + f'proxy {proxy.string()} is valid, remain current score') else: - self.redis.decrease(proxy) - logger.debug(f'proxy {proxy.string()} is invalid, decrease score') - except EXCEPTIONS: - self.redis.decrease(proxy) - logger.debug(f'proxy {proxy.string()} is invalid, decrease score') - + self.redis.max(proxy) + logger.debug( + f'proxy {proxy.string()} is valid, set max score') + else: + self.redis.decrease(proxy) + logger.debug( + f'proxy {proxy.string()} is invalid, decrease score') + # if independent tester class found, create new set of storage and do the extra test + for tester in self.testers: + key = tester.key + if self.redis.exists(proxy, key): + test_url = tester.test_url + headers = tester.headers() + cookies = tester.cookies() + async with session.get(test_url, proxy=f'http://{proxy.string()}', + timeout=TEST_TIMEOUT, + headers=headers, + cookies=cookies, + allow_redirects=False) as response: + resp_text = await response.text() + is_valid = await tester.parse(resp_text, test_url, proxy.string()) + if is_valid: + if tester.test_dont_set_max_score: + logger.info( + f'key[{key}] proxy {proxy.string()} is valid, remain current score') + else: + self.redis.max( + proxy, key, tester.proxy_score_max) + logger.info( + f'key[{key}] proxy {proxy.string()} is valid, set max score') + else: + self.redis.decrease( + proxy, tester.key, tester.proxy_score_min) + logger.info( + f'key[{key}] proxy {proxy.string()} is invalid, decrease score') + + except EXCEPTIONS: + self.redis.decrease(proxy) + [self.redis.decrease(proxy, tester.key, tester.proxy_score_min) + for tester in self.testers] + logger.debug( + f'proxy {proxy.string()} is invalid, decrease score') + + async def run_tests(self): + """ + test all proxies in batches, reusing a single aiohttp session + :return: + """ + count = self.redis.count() + logger.debug(f'{count} proxies to test') + cursor = 0 + connector = aiohttp.TCPConnector(ssl=False, limit=TEST_BATCH) + async with aiohttp.ClientSession(connector=connector) as session: + while True: + logger.debug( + f'testing proxies use cursor {cursor}, count {TEST_BATCH}') + cursor, proxies = self.redis.batch(cursor, count=TEST_BATCH) + if proxies: + tasks = [self.test(proxy, session) for proxy in proxies] + await asyncio.gather(*tasks, return_exceptions=True) + if not cursor: + break + @logger.catch def run(self): """ @@ -72,26 +138,32 @@ def run(self): """ # event loop of aiohttp logger.info('stating tester...') - count = self.redis.count() - logger.debug(f'{count} proxies to test') - cursor = 0 - while True: - logger.debug(f'testing proxies use cursor {cursor}, count {TEST_BATCH}') - cursor, proxies = self.redis.batch(cursor, count=TEST_BATCH) - if proxies: - tasks = [self.test(proxy) for proxy in proxies] - self.loop.run_until_complete(asyncio.wait(tasks)) - if not cursor: - break + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + loop.run_until_complete(self.run_tests()) + finally: + loop.close() + def run_tester(): host = '96.113.165.182' port = '3128' - tasks = [tester.test(Proxy(host=host, port=port))] - tester.loop.run_until_complete(asyncio.wait(tasks)) + tester = Tester() + + async def _test(): + async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=False)) as session: + await tester.test(Proxy(host=host, port=port), session) + + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + loop.run_until_complete(_test()) + finally: + loop.close() + if __name__ == '__main__': tester = Tester() tester.run() # run_tester() - diff --git a/proxypool/scheduler.py b/proxypool/scheduler.py index f29b5655..d0582268 100644 --- a/proxypool/scheduler.py +++ b/proxypool/scheduler.py @@ -92,7 +92,7 @@ def run_server(self): logger.error("unsupported APP_PROD_METHOD") return else: - app.run(host=API_HOST, port=API_PORT, threaded=API_THREADED) + app.run(host=API_HOST, port=API_PORT, threaded=API_THREADED, use_reloader=False) def run(self): global tester_process, getter_process, server_process @@ -129,12 +129,15 @@ def run(self): tester_process and tester_process.join() getter_process and getter_process.join() server_process and server_process.join() - logger.info( - f'tester is {"alive" if tester_process.is_alive() else "dead"}') - logger.info( - f'getter is {"alive" if getter_process.is_alive() else "dead"}') - logger.info( - f'server is {"alive" if server_process.is_alive() else "dead"}') + if tester_process: + logger.info( + f'tester is {"alive" if tester_process.is_alive() else "dead"}') + if getter_process: + logger.info( + f'getter is {"alive" if getter_process.is_alive() else "dead"}') + if server_process: + logger.info( + f'server is {"alive" if server_process.is_alive() else "dead"}') logger.info('proxy terminated') diff --git a/proxypool/setting.py b/proxypool/setting.py index e68f45ca..4d1cec56 100644 --- a/proxypool/setting.py +++ b/proxypool/setting.py @@ -53,9 +53,11 @@ 'REDIS_KEY', 'proxies:universal')) # definition of proxy scores -PROXY_SCORE_MAX = 100 -PROXY_SCORE_MIN = 0 -PROXY_SCORE_INIT = 10 +PROXY_SCORE_MAX = env.int('PROXY_SCORE_MAX', 100) +PROXY_SCORE_MIN = env.int('PROXY_SCORE_MIN', 0) +PROXY_SCORE_INIT = env.int('PROXY_SCORE_INIT', 10) +# whether to get a universal random proxy if no proxy exists in the sub-pool identified by a specific key +PROXY_RAND_KEY_DEGRADED = env.bool('TEST_ANONYMOUS', True) # definition of proxy number PROXY_NUMBER_MAX = 50000 @@ -73,15 +75,25 @@ TEST_BATCH = env.int('TEST_BATCH', 20) # only save anonymous proxy TEST_ANONYMOUS = env.bool('TEST_ANONYMOUS', True) +# the url used to check the proxy anonymity and its exit ip; +# must return json like httpbin.org/ip ({"origin": "1.2.3.4"}); +# point this to a self-hosted httpbin to avoid public rate limits +TEST_ANONYMOUS_URL = env.str('TEST_ANONYMOUS_URL', 'https://httpbin.org/ip') # TEST_HEADERS = env.json('TEST_HEADERS', { # 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36', # }) TEST_VALID_STATUS = env.list('TEST_VALID_STATUS', [200, 206, 302]) +# whether to set max score when one proxy is tested valid +TEST_DONT_SET_MAX_SCORE = env.bool('TEST_DONT_SET_MAX_SCORE', False) # definition of api API_HOST = env.str('API_HOST', '0.0.0.0') API_PORT = env.int('API_PORT', 5555) API_THREADED = env.bool('API_THREADED', True) +# add an api key to get proxy +# need a header of `API-KEY` in get request to pass the authenticate +# API_KEY='', do not need `API-KEY` header +API_KEY = env.str('API_KEY', '') # flags of enable ENABLE_TESTER = env.bool('ENABLE_TESTER', True) diff --git a/proxypool/storages/redis.py b/proxypool/storages/redis.py index 8ab0e41d..aed45029 100644 --- a/proxypool/storages/redis.py +++ b/proxypool/storages/redis.py @@ -3,7 +3,7 @@ from proxypool.schemas.proxy import Proxy from proxypool.setting import REDIS_CONNECTION_STRING, REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_DB, REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MIN, \ PROXY_SCORE_INIT -from random import choice +from random import choice, sample from typing import List from loguru import logger from proxypool.utils.proxy import is_valid_proxy, convert_proxy_or_proxies @@ -34,7 +34,7 @@ def __init__(self, host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD, db self.db = redis.StrictRedis( host=host, port=port, password=password, db=db, decode_responses=True, **kwargs) - def add(self, proxy: Proxy, score=PROXY_SCORE_INIT) -> int: + def add(self, proxy: Proxy, score=PROXY_SCORE_INIT, redis_key=REDIS_KEY) -> int: """ add proxy and set it to init score :param proxy: proxy, ip:port, like 8.8.8.8:88 @@ -44,12 +44,12 @@ def add(self, proxy: Proxy, score=PROXY_SCORE_INIT) -> int: if not is_valid_proxy(f'{proxy.host}:{proxy.port}'): logger.info(f'invalid proxy {proxy}, throw it') return - if not self.exists(proxy): + if not self.exists(proxy, redis_key): if IS_REDIS_VERSION_2: - return self.db.zadd(REDIS_KEY, score, proxy.string()) - return self.db.zadd(REDIS_KEY, {proxy.string(): score}) + return self.db.zadd(redis_key, score, proxy.string()) + return self.db.zadd(redis_key, {proxy.string(): score}) - def random(self) -> Proxy: + def random(self, redis_key=REDIS_KEY, proxy_score_min=PROXY_SCORE_MIN, proxy_score_max=PROXY_SCORE_MAX) -> Proxy: """ get random proxy firstly try to get proxy with max score @@ -59,74 +59,95 @@ def random(self) -> Proxy: """ # try to get proxy with max score proxies = self.db.zrangebyscore( - REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MAX) + redis_key, proxy_score_max, proxy_score_max) if len(proxies): return convert_proxy_or_proxies(choice(proxies)) # else get proxy by rank proxies = self.db.zrevrange( - REDIS_KEY, PROXY_SCORE_MIN, PROXY_SCORE_MAX) + redis_key, proxy_score_min, proxy_score_max) if len(proxies): return convert_proxy_or_proxies(choice(proxies)) # else raise error raise PoolEmptyException - def decrease(self, proxy: Proxy) -> int: + def randoms(self, count, redis_key=REDIS_KEY, proxy_score_min=PROXY_SCORE_MIN, proxy_score_max=PROXY_SCORE_MAX) -> List[Proxy]: + """ + get a batch of random proxies + firstly try to get proxies with max score, + if not enough, get proxies by rank (score from high to low) + if none exists, raise error + :param count: number of proxies to return + :return: list of proxies + """ + # try to get proxies with max score first + proxies = self.db.zrangebyscore( + redis_key, proxy_score_max, proxy_score_max) + if len(proxies) < count: + # not enough max-score proxies, fall back to all proxies by rank + proxies = self.db.zrevrangebyscore( + redis_key, proxy_score_max, proxy_score_min) + if not proxies: + raise PoolEmptyException + count = min(count, len(proxies)) + return convert_proxy_or_proxies(sample(proxies, count)) + + def decrease(self, proxy: Proxy, redis_key=REDIS_KEY, proxy_score_min=PROXY_SCORE_MIN) -> int: """ decrease score of proxy, if small than PROXY_SCORE_MIN, delete it :param proxy: proxy :return: new score """ if IS_REDIS_VERSION_2: - self.db.zincrby(REDIS_KEY, proxy.string(), -1) + self.db.zincrby(redis_key, proxy.string(), -1) else: - self.db.zincrby(REDIS_KEY, -1, proxy.string()) - score = self.db.zscore(REDIS_KEY, proxy.string()) + self.db.zincrby(redis_key, -1, proxy.string()) + score = self.db.zscore(redis_key, proxy.string()) logger.info(f'{proxy.string()} score decrease 1, current {score}') - if score <= PROXY_SCORE_MIN: + if score <= proxy_score_min: logger.info(f'{proxy.string()} current score {score}, remove') - self.db.zrem(REDIS_KEY, proxy.string()) + self.db.zrem(redis_key, proxy.string()) - def exists(self, proxy: Proxy) -> bool: + def exists(self, proxy: Proxy, redis_key=REDIS_KEY) -> bool: """ if proxy exists :param proxy: proxy :return: if exists, bool """ - return not self.db.zscore(REDIS_KEY, proxy.string()) is None + return not self.db.zscore(redis_key, proxy.string()) is None - def max(self, proxy: Proxy) -> int: + def max(self, proxy: Proxy, redis_key=REDIS_KEY, proxy_score_max=PROXY_SCORE_MAX) -> int: """ set proxy to max score :param proxy: proxy :return: new score """ - logger.info(f'{proxy.string()} is valid, set to {PROXY_SCORE_MAX}') + logger.info(f'{proxy.string()} is valid, set to {proxy_score_max}') if IS_REDIS_VERSION_2: - return self.db.zadd(REDIS_KEY, PROXY_SCORE_MAX, proxy.string()) - return self.db.zadd(REDIS_KEY, {proxy.string(): PROXY_SCORE_MAX}) + return self.db.zadd(redis_key, proxy_score_max, proxy.string()) + return self.db.zadd(redis_key, {proxy.string(): proxy_score_max}) - def count(self) -> int: + def count(self, redis_key=REDIS_KEY) -> int: """ get count of proxies :return: count, int """ - return self.db.zcard(REDIS_KEY) + return self.db.zcard(redis_key) - def all(self) -> List[Proxy]: + def all(self, redis_key=REDIS_KEY, proxy_score_min=PROXY_SCORE_MIN, proxy_score_max=PROXY_SCORE_MAX) -> List[Proxy]: """ get all proxies :return: list of proxies """ - return convert_proxy_or_proxies(self.db.zrangebyscore(REDIS_KEY, PROXY_SCORE_MIN, PROXY_SCORE_MAX)) + return convert_proxy_or_proxies(self.db.zrangebyscore(redis_key, proxy_score_min, proxy_score_max)) - def batch(self, cursor, count) -> List[Proxy]: + def batch(self, cursor, count, redis_key=REDIS_KEY) -> List[Proxy]: """ get batch of proxies :param cursor: scan cursor :param count: scan count :return: list of proxies """ - cursor, proxies = self.db.zscan(REDIS_KEY, cursor, count=count) + cursor, proxies = self.db.zscan(redis_key, cursor, count=count) return cursor, convert_proxy_or_proxies([i[0] for i in proxies]) diff --git a/proxypool/testers/__init__.py b/proxypool/testers/__init__.py new file mode 100644 index 00000000..4e4df95e --- /dev/null +++ b/proxypool/testers/__init__.py @@ -0,0 +1,16 @@ +import pkgutil +from .base import BaseTester +import inspect + + +# load classes subclass of BaseCrawler +classes = [] +for loader, name, is_pkg in pkgutil.walk_packages(__path__): + module = loader.find_module(name).load_module(name) + for name, value in inspect.getmembers(module): + globals()[name] = value + if inspect.isclass(value) and issubclass(value, BaseTester) and value is not BaseTester \ + and not getattr(value, 'ignore', False): + classes.append(value) +__all__ = __ALL__ = classes + diff --git a/proxypool/testers/base.py b/proxypool/testers/base.py new file mode 100644 index 00000000..796b7cfc --- /dev/null +++ b/proxypool/testers/base.py @@ -0,0 +1,19 @@ +from proxypool.setting import TEST_DONT_SET_MAX_SCORE, PROXY_SCORE_INIT, PROXY_SCORE_MAX, PROXY_SCORE_MIN + + +class BaseTester(object): + test_url = "" + key = "" + test_dont_set_max_score = TEST_DONT_SET_MAX_SCORE + proxy_score_init = PROXY_SCORE_INIT + proxy_score_max = PROXY_SCORE_MAX + proxy_score_min = PROXY_SCORE_MIN + + def headers(self): + return None + + def cookies(self): + return None + + async def parse(self, html, url, proxy, expr='{"code":0'): + return True if expr in html else False diff --git a/proxypool/utils/geo.py b/proxypool/utils/geo.py new file mode 100644 index 00000000..82a105a0 --- /dev/null +++ b/proxypool/utils/geo.py @@ -0,0 +1,35 @@ +from loguru import logger + +# geolite2 provides an offline IP -> country database (bundled with the +# maxminddb_geolite2 dependency). loading it can fail if the optional +# dependency is missing, so degrade gracefully and disable area filtering. +try: + from geolite2 import geolite2 + + _reader = geolite2.reader() +except Exception as e: # pragma: no cover + _reader = None + logger.warning(f'geolite2 is unavailable, area filtering disabled: {e}') + + +def get_country_iso(ip): + """ + look up the ISO country code (e.g. 'CN', 'US') for an ip address + :param ip: ip address string + :return: uppercase iso code, or None if unknown/unavailable + """ + if _reader is None: + return None + try: + record = _reader.get(ip) + except Exception: + return None + if not record: + return None + country = record.get('country') or record.get('registered_country') or {} + return country.get('iso_code') + + +if __name__ == '__main__': + print('8.8.8.8', get_country_iso('8.8.8.8')) + print('114.114.114.114', get_country_iso('114.114.114.114')) diff --git a/proxypool/utils/proxy.py b/proxypool/utils/proxy.py index 79cc27fb..ed81ef8b 100644 --- a/proxypool/utils/proxy.py +++ b/proxypool/utils/proxy.py @@ -56,14 +56,14 @@ def convert_proxy_or_proxies(data): if is_auth_proxy(item): host, port = extract_auth_proxy(item) else: - host, port = item.split(':') + host, port, *_ = item.split(':') result.append(Proxy(host=host, port=int(port))) return result if isinstance(data, str) and is_valid_proxy(data): if is_auth_proxy(data): host, port = extract_auth_proxy(data) else: - host, port = data.split(':') + host, port, *_ = data.split(':') return Proxy(host=host, port=int(port)) diff --git a/requirements.txt b/requirements.txt index c9407c7d..49828461 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,17 +1,16 @@ -environs>=9.3.0,<10.0.0 -Flask>=1.1.2,<2.0.0 -attrs>=20.3.0,<21.0.0 +environs>=9.3.0,<12.0.0 +Flask>=2.0.0,<3.0.0 +Werkzeug>=2.0.0,<3.0.0 +attrs>=20.3.0,<24.0.0 retrying>=1.3.3,<2.0.0 -aiohttp>=3.8.1,<4.0.0 +aiohttp>=3.9.0,<4.0.0 requests>=2.25.1,<3.0.0 loguru>=0.5.3,<1.0.0 pyquery>=1.4.3,<2.0.0 supervisor>=4.2.1,<5.0.0 -redis>=3.5.3,<4.0.0 -lxml>=4.6.5,<5.0.0 +redis>=4.3.0,<6.0.0 +lxml>=4.6.5,<6.0.0 fake_headers>=1.0.2,<2.0.0 maxminddb_geolite2==2018.703 -gevent>=21.8.0,<22.0.0 +gevent>=22.10.2,<25.0.0 tornado>=6.0,<7.0 -itsdangerous==0.24 -MarkupSafe<2.1.0