forked from Boris-code/feapder
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathhandle_failed_requests.py
More file actions
56 lines (45 loc) · 1.72 KB
/
handle_failed_requests.py
File metadata and controls
56 lines (45 loc) · 1.72 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
# -*- coding: utf-8 -*-
"""
Created on 2018-08-13 11:43:01
---------
@summary:
---------
@author: Boris
@email: boris_liu@foxmail.com
"""
import feapder.setting as setting
from feapder.buffer.request_buffer import RequestBuffer
from feapder.db.redisdb import RedisDB
from feapder.network.request import Request
from feapder.utils.log import log
class HandleFailedRequests(object):
"""docstring for HandleFailedRequests"""
def __init__(self, redis_key):
super(HandleFailedRequests, self).__init__()
self._redis_key = redis_key
self._redisdb = RedisDB()
self._request_buffer = RequestBuffer(self._redis_key)
self._table_failed_request = setting.TAB_FAILED_REQUSETS.format(
redis_key=redis_key
)
def get_failed_requests(self, count=10000):
failed_requests = self._redisdb.zget(self._table_failed_request, count=count)
failed_requests = [eval(failed_request) for failed_request in failed_requests]
return failed_requests
def reput_failed_requests_to_requests(self):
log.debug("正在重置失败的requests...")
total_count = 0
while True:
try:
failed_requests = self.get_failed_requests()
if not failed_requests:
break
for request in failed_requests:
request["retry_times"] = 0
request_obj = Request.from_dict(request)
self._request_buffer.put_request(request_obj)
total_count += 1
except Exception as e:
log.exception(e)
self._request_buffer.flush()
log.debug("重置%s条失败requests为待抓取requests" % total_count)