Skip to content

Commit 3314981

Browse files
jjmarjjmar
authored
Cp 7117/add retries to api calls (quandl#124)
* Automatically retry failed 5xx api calls * Add additional tests for retries * Fix test failing on CI by using different way to compare json response * Increase test case speed by modifying backoff_factor to 0 for tests which deal with error responses * Allow test_retries tests to pass even when user has toggled use_retries to False * Dont wait on retry-after headers to keep current functionality intact * Add documentation for retry configuration * Update failing datatable export test following merge * Reduce default retry settings so retries will be less aggressive * Uncaptialize retry status codes config * Allow retries on 429's * Update readme retry examples to have default values * Bump version * Update changelog * Remove debug statements
1 parent ae08eda commit 3314981

9 files changed

Lines changed: 190 additions & 24 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,14 @@
11
### unreleased
22
* Remove dependency on unittest2, use unittest instead (#113)
33

4+
### 3.4.4 - 2018-10-24
5+
6+
* Add functionality to automatically retry failed API calls https://github.com/quandl/quandl-python/pull/124
7+
8+
### 3.4.3 - 2018-10-19
9+
10+
* Allow for exporting of datatables https://github.com/quandl/quandl-python/pull/120
11+
412
### 3.4.2 - 2018-08-21
513

614
* Fix typos in our warning messages https://github.com/quandl/quandl-python/pull/114

‎README.md‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,12 @@ pip3 install quandl
2828
|---|---|---|
2929
| api_key | Your access key | `tEsTkEy123456789` | Used to identify who you are and provide full access. |
3030
| api_version | The API version you wish to use | 2015-04-09 | Can be used to test your code against the latest version without committing to it. |
31-
31+
| use_retries | Whether API calls which return statuses in `retry_status_codes` should be automatically retried | True
32+
| number_of_retries | Maximum number of retries that should be attempted. Only used if `use_retries` is True | 5
33+
| max_wait_between_retries | Maximum amount of time in seconds that should be waited before attempting a retry. Only used if `use_retries` is True | 8
34+
| retry_backoff_factor | Determines the amount of time in seconds that should be waited before attempting another retry. Note that this factor is exponential so a `retry_backoff_factor` of 0.5 will cause waits of [0.5, 1, 2, 4, etc]. Only used if `use_retries` is True | 0.5
35+
| retry_status_codes | A list of HTTP status codes which will trigger a retry to occur. Only used if `use_retries` is True| [429, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511]
36+
3237
```python
3338
import quandl
3439
quandl.ApiConfig.api_key = 'tEsTkEy123456789'

‎quandl/api_config.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,16 @@
11
class ApiConfig:
22
api_key = None
3-
api_base = 'https://www.quandl.com/api/v3'
3+
api_protocol = 'https://'
4+
api_base = '{}www.quandl.com/api/v3'.format(api_protocol)
45
api_version = None
56
page_limit = 100
67

8+
use_retries = True
9+
number_of_retries = 5
10+
retry_backoff_factor = 0.5
11+
max_wait_between_retries = 8
12+
retry_status_codes = [429] + list(range(500, 512))
13+
714

815
def save_key(apikey, filename=None):
916
if filename is None:

‎quandl/connection.py‎

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
import re
22

33
import requests
4+
from urllib3.util.retry import Retry
5+
from requests.adapters import HTTPAdapter
46

57
from .util import Util
68
from .version import VERSION
@@ -37,9 +39,10 @@ def request(cls, http_verb, url, **options):
3739

3840
@classmethod
3941
def execute_request(cls, http_verb, url, **options):
42+
session = cls.get_session()
43+
4044
try:
41-
func = getattr(requests, http_verb)
42-
response = func(url, **options)
45+
response = session.request(method=http_verb, url=url, **options)
4346
if response.status_code < 200 or response.status_code >= 300:
4447
cls.handle_api_error(response)
4548
else:
@@ -49,6 +52,29 @@ def execute_request(cls, http_verb, url, **options):
4952
cls.handle_api_error(e.response)
5053
raise e
5154

55+
@classmethod
56+
def get_session(cls):
57+
session = requests.Session()
58+
adapter = HTTPAdapter(max_retries=cls.get_retries())
59+
session.mount(ApiConfig.api_protocol, adapter)
60+
61+
return session
62+
63+
@classmethod
64+
def get_retries(cls):
65+
if not ApiConfig.use_retries:
66+
return Retry(total=0)
67+
68+
Retry.BACKOFF_MAX = ApiConfig.max_wait_between_retries
69+
retries = Retry(total=ApiConfig.number_of_retries,
70+
connect=ApiConfig.number_of_retries,
71+
read=ApiConfig.number_of_retries,
72+
status_forcelist=ApiConfig.retry_status_codes,
73+
backoff_factor=ApiConfig.retry_backoff_factor,
74+
raise_on_status=False)
75+
76+
return retries
77+
5278
@classmethod
5379
def parse(cls, response):
5480
try:

‎quandl/version.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
VERSION = '3.4.3'
1+
VERSION = '3.4.4'

‎test/test_connection.py‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,18 @@
44
QuandlError, LimitExceededError, InternalServerError,
55
AuthenticationError, ForbiddenError, InvalidRequestError,
66
NotFoundError, ServiceUnavailableError)
7-
import unittest
7+
from test.test_retries import ModifyRetrySettingsTestCase
88
from test.helpers.httpretty_extension import httpretty
99
import json
1010
from mock import patch, call
1111
from quandl.version import VERSION
1212

1313

14-
class ConnectionTest(unittest.TestCase):
14+
class ConnectionTest(ModifyRetrySettingsTestCase):
1515

1616
@httpretty.activate
17-
def test_quandl_exceptions(self):
17+
def test_quandl_exceptions_no_retries(self):
18+
ApiConfig.use_retries = False
1819
quandl_errors = [('QELx04', 429, LimitExceededError),
1920
('QEMx01', 500, InternalServerError),
2021
('QEAx01', 400, AuthenticationError),
@@ -38,6 +39,7 @@ def test_quandl_exceptions(self):
3839

3940
@httpretty.activate
4041
def test_parse_error(self):
42+
ApiConfig.retry_backoff_factor = 0
4143
httpretty.register_uri(httpretty.GET,
4244
"https://www.quandl.com/api/v3/databases",
4345
body="not json", status=500)
@@ -46,6 +48,7 @@ def test_parse_error(self):
4648

4749
@httpretty.activate
4850
def test_non_quandl_error(self):
51+
ApiConfig.retry_backoff_factor = 0
4952
httpretty.register_uri(httpretty.GET,
5053
"https://www.quandl.com/api/v3/databases",
5154
body=json.dumps(
@@ -70,6 +73,4 @@ def test_build_request(self, mock):
7073
'request-source': 'python',
7174
'request-source-version': VERSION},
7275
params={'per_page': 10, 'page': 2})
73-
print(mock.call_args)
74-
print(expected)
7576
self.assertEqual(mock.call_args, expected)

‎test/test_database.py‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
from quandl.api_config import ApiConfig
1515
from quandl.model.database import Database
1616
from quandl.connection import Connection
17+
from test.test_retries import ModifyRetrySettingsTestCase
1718
from mock import patch, call, mock_open
1819
from test.factories.database import DatabaseFactory
1920
from test.factories.meta import MetaFactory
@@ -110,10 +111,9 @@ def test_databases_has_more(self):
110111
self.assertTrue(results.has_more_results())
111112

112113

113-
class BulkDownloadDatabaseTest(unittest.TestCase):
114+
class BulkDownloadDatabaseTest(ModifyRetrySettingsTestCase):
114115

115-
@classmethod
116-
def setUpClass(cls):
116+
def setUp(self):
117117
httpretty.enable()
118118
httpretty.register_uri(httpretty.GET,
119119
re.compile(
@@ -125,17 +125,15 @@ def setUpClass(cls):
125125
httpretty.register_uri(httpretty.GET,
126126
re.compile('https://www.blah.com/'), body='{}')
127127

128-
@classmethod
129-
def tearDownClass(cls):
130-
httpretty.disable()
131-
httpretty.reset()
132-
133-
def setUp(self):
134128
database = {'database': DatabaseFactory.build(database_code='NSE')}
135129
self.database = Database(database['database']['database_code'], database['database'])
136130
ApiConfig.api_key = 'api_token'
137131
ApiConfig.api_version = '2015-04-09'
138132

133+
def tearDown(self):
134+
httpretty.disable()
135+
httpretty.reset()
136+
139137
def test_get_bulk_downnload_url_with_download_type(self):
140138
url = self.database.bulk_download_url(params={'download_type': 'partial'})
141139
parsed_url = urlparse(url)
@@ -179,12 +177,15 @@ def test_bulk_download_raises_exception_when_no_path(self):
179177
QuandlError, lambda: self.database.bulk_download_to_file(None))
180178

181179
def test_bulk_download_raises_exception_when_error_response(self):
180+
ApiConfig.retry_backoff_factor = 0
181+
httpretty.reset()
182182
httpretty.register_uri(httpretty.GET,
183183
re.compile(
184184
'https://www.quandl.com/api/v3/databases/*'),
185185
body=json.dumps(
186186
{'quandl_error':
187187
{'code': 'QEMx01', 'message': 'something went wrong'}}),
188188
status=500)
189+
189190
self.assertRaises(
190191
InternalServerError, lambda: self.database.bulk_download_to_file('.'))

‎test/test_datatable.py‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,12 @@
1111
from quandl.model.datatable import Datatable
1212
from mock import patch, call, mock_open
1313
from test.factories.datatable import DatatableFactory
14+
from test.test_retries import ModifyRetrySettingsTestCase
1415
from quandl.api_config import ApiConfig
1516
from quandl.errors.quandl_error import (InternalServerError, QuandlError)
1617

1718

18-
class GetDatatableDatasetTest(unittest.TestCase):
19+
class GetDatatableDatasetTest(ModifyRetrySettingsTestCase):
1920

2021
@classmethod
2122
def setUpClass(cls):
@@ -114,12 +115,17 @@ def test_bulk_download_raises_exception_when_no_path(self):
114115
QuandlError, lambda: self.datatable.download_file(None))
115116

116117
def test_bulk_download_table_raises_exception_when_error_response(self):
118+
httpretty.reset()
119+
ApiConfig.number_of_retries = 2
120+
error_responses = [httpretty.Response(
121+
body=json.dumps({'quandl_error': {'code': 'QEMx01',
122+
'message': 'something went wrong'}}),
123+
status=500)]
124+
117125
httpretty.register_uri(httpretty.GET,
118126
re.compile(
119127
'https://www.quandl.com/api/v3/datatables/*'),
120-
body=json.dumps(
121-
{'quandl_error':
122-
{'code': 'QEMx01', 'message': 'something went wrong'}}),
123-
status=500)
128+
responses=error_responses)
129+
124130
self.assertRaises(
125131
InternalServerError, lambda: self.datatable.download_file('.'))

‎test/test_retries.py‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
import unittest
2+
import json
3+
4+
from quandl.connection import Connection
5+
from quandl.api_config import ApiConfig
6+
from test.factories.datatable import DatatableFactory
7+
from test.helpers.httpretty_extension import httpretty
8+
from quandl.errors.quandl_error import InternalServerError
9+
10+
11+
class ModifyRetrySettingsTestCase(unittest.TestCase):
12+
13+
def setUp(self):
14+
self.default_use_retries = ApiConfig.use_retries
15+
self.default_number_of_retries = ApiConfig.number_of_retries
16+
self.default_retry_backoff_factor = ApiConfig.retry_backoff_factor
17+
self.default_max_wait_between_retries = ApiConfig.max_wait_between_retries
18+
self.default_retry_status_codes = ApiConfig.retry_status_codes
19+
20+
def tearDown(self):
21+
ApiConfig.use_retries = self.default_use_retries
22+
ApiConfig.number_of_retries = self.default_number_of_retries
23+
ApiConfig.retry_backoff_factor = self.default_retry_backoff_factor
24+
ApiConfig.max_wait_between_retries = self.default_max_wait_between_retries
25+
ApiConfig.retry_status_codes = self.default_retry_status_codes
26+
27+
28+
class TestRetries(ModifyRetrySettingsTestCase):
29+
30+
def setUp(self):
31+
ApiConfig.use_retries = True
32+
super(TestRetries, self).setUp()
33+
34+
@classmethod
35+
def setUpClass(cls):
36+
cls.datatable = {'datatable': DatatableFactory.build(
37+
vendor_code='ZACKS',
38+
datatable_code='FC')}
39+
40+
cls.error_response = httpretty.Response(
41+
body=json.dumps({'quandl_error': {'code': 'QEMx01',
42+
'message': 'something went wrong'}}),
43+
status=500)
44+
cls.success_response = httpretty.Response(body=json.dumps(cls.datatable), status=200)
45+
46+
def test_modifying_use_retries(self):
47+
ApiConfig.use_retries = False
48+
49+
retries = Connection.get_session().get_adapter(ApiConfig.api_protocol).max_retries
50+
self.assertEqual(retries.total, 0)
51+
52+
def test_modifying_number_of_retries(self):
53+
ApiConfig.number_of_retries = 3000
54+
55+
retries = Connection.get_session().get_adapter(ApiConfig.api_protocol).max_retries
56+
57+
self.assertEqual(retries.total, ApiConfig.number_of_retries)
58+
self.assertEqual(retries.connect, ApiConfig.number_of_retries)
59+
self.assertEqual(retries.read, ApiConfig.number_of_retries)
60+
61+
def test_modifying_retry_backoff_factor(self):
62+
ApiConfig.retry_backoff_factor = 3000
63+
64+
retries = Connection.get_session().get_adapter(ApiConfig.api_protocol).max_retries
65+
self.assertEqual(retries.backoff_factor, ApiConfig.retry_backoff_factor)
66+
67+
def test_modifying_retry_status_codes(self):
68+
ApiConfig.retry_status_codes = [1, 2, 3]
69+
70+
retries = Connection.get_session().get_adapter(ApiConfig.api_protocol).max_retries
71+
self.assertEqual(retries.status_forcelist, ApiConfig.retry_status_codes)
72+
73+
def test_modifying_max_wait_between_retries(self):
74+
ApiConfig.max_wait_between_retries = 3000
75+
76+
retries = Connection.get_session().get_adapter(ApiConfig.api_protocol).max_retries
77+
self.assertEqual(retries.BACKOFF_MAX, ApiConfig.max_wait_between_retries)
78+
79+
@httpretty.activate
80+
def test_correct_response_returned_if_retries_succeed(self):
81+
ApiConfig.number_of_retries = 3
82+
ApiConfig.retry_status_codes = [self.error_response.status]
83+
84+
mock_responses = [self.error_response] + [self.error_response] + [self.success_response]
85+
httpretty.register_uri(httpretty.GET,
86+
"https://www.quandl.com/api/v3/databases",
87+
responses=mock_responses)
88+
89+
response = Connection.request('get', 'databases')
90+
self.assertEqual(response.json(), self.datatable)
91+
self.assertEqual(response.status_code, self.success_response.status)
92+
93+
@httpretty.activate
94+
def test_correct_response_exception_raised_if_retries_fail(self):
95+
ApiConfig.number_of_retries = 2
96+
ApiConfig.retry_status_codes = [self.error_response.status]
97+
mock_responses = [self.error_response] * 3
98+
httpretty.register_uri(httpretty.GET,
99+
"https://www.quandl.com/api/v3/databases",
100+
responses=mock_responses)
101+
102+
self.assertRaises(InternalServerError, Connection.request, 'get', 'databases')
103+
104+
@httpretty.activate
105+
def test_correct_response_exception_raised_for_errors_not_in_retry_status_codes(self):
106+
ApiConfig.retry_status_codes = []
107+
mock_responses = [self.error_response]
108+
httpretty.register_uri(httpretty.GET,
109+
"https://www.quandl.com/api/v3/databases",
110+
responses=mock_responses)
111+
112+
self.assertRaises(InternalServerError, Connection.request, 'get', 'databases')

0 commit comments

Comments
 (0)