diff --git a/.secrets.baseline b/.secrets.baseline index f0aee0650..99a7c5ac2 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$", "lines": null }, - "generated_at": "2025-06-11T21:28:32Z", + "generated_at": "2026-03-24T22:14:30Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -554,7 +554,7 @@ "hashed_secret": "a4c805a62a0387010cd172cfed6f6772eb92a5d6", "is_secret": false, "is_verified": false, - "line_number": 81, + "line_number": 82, "type": "Secret Keyword", "verified_result": null } diff --git a/README.rst b/README.rst index 5f82bdd62..29536d085 100644 --- a/README.rst +++ b/README.rst @@ -173,7 +173,6 @@ If you cannot install python 3.6+ for some reason, you will need to use a versio Python Packages --------------- -* prettytable >= 2.5.0 * click >= 8.0.4 * requests >= 2.32.2 * prompt_toolkit >= 2 diff --git a/SoftLayer/API.py b/SoftLayer/API.py index cff277286..17fec6d94 100644 --- a/SoftLayer/API.py +++ b/SoftLayer/API.py @@ -50,6 +50,38 @@ )) +def _build_transport(url, proxy, timeout, user_agent, verify): + """Construct the appropriate transport based on the endpoint URL. + + Selects RestTransport when the URL contains '/rest', otherwise falls back + to XmlRpcTransport. Extracted to avoid duplicating this logic across + ``create_client_from_env``, ``employee_client``, and ``BaseClient``. + + :param str url: The API endpoint URL. + :param str proxy: Optional proxy URL. + :param timeout: Request timeout in seconds (``None`` means no timeout). + :param str user_agent: Optional User-Agent string override. + :param verify: SSL verification — ``True``, ``False``, or a path to a CA bundle. + :returns: A :class:`~SoftLayer.transports.RestTransport` or + :class:`~SoftLayer.transports.XmlRpcTransport` instance. + """ + if url is not None and '/rest' in url: + return transports.RestTransport( + endpoint_url=url, + proxy=proxy, + timeout=timeout, + user_agent=user_agent, + verify=verify, + ) + return transports.XmlRpcTransport( + endpoint_url=url, + proxy=proxy, + timeout=timeout, + user_agent=user_agent, + verify=verify, + ) + + def create_client_from_env(username=None, api_key=None, endpoint_url=None, @@ -62,7 +94,7 @@ def create_client_from_env(username=None, verify=True): """Creates a SoftLayer API client using your environment. - Settings are loaded via keyword arguments, environemtal variables and + Settings are loaded via keyword arguments, environmental variables and config file. :param username: an optional API username if you wish to bypass the @@ -104,25 +136,13 @@ def create_client_from_env(username=None, config_file=config_file) if transport is None: - url = settings.get('endpoint_url') - if url is not None and '/rest' in url: - # If this looks like a rest endpoint, use the rest transport - transport = transports.RestTransport( - endpoint_url=settings.get('endpoint_url'), - proxy=settings.get('proxy'), - timeout=settings.get('timeout'), - user_agent=user_agent, - verify=verify, - ) - else: - # Default the transport to use XMLRPC - transport = transports.XmlRpcTransport( - endpoint_url=settings.get('endpoint_url'), - proxy=settings.get('proxy'), - timeout=settings.get('timeout'), - user_agent=user_agent, - verify=verify, - ) + transport = _build_transport( + url=settings.get('endpoint_url'), + proxy=settings.get('proxy'), + timeout=settings.get('timeout'), + user_agent=user_agent, + verify=verify, + ) # If we have enough information to make an auth driver, let's do it if auth is None and settings.get('username') and settings.get('api_key'): @@ -157,13 +177,13 @@ def employee_client(username=None, verify=True): """Creates an INTERNAL SoftLayer API client using your environment. - Settings are loaded via keyword arguments, environemtal variables and config file. + Settings are loaded via keyword arguments, environmental variables and config file. :param username: your user ID - :param access_token: hash from SoftLayer_User_Employee::performExternalAuthentication(username, password, token) - :param password: password to use for employee authentication + :param access_token: hash from SoftLayer_User_Employee::performExternalAuthentication :param endpoint_url: the API endpoint base URL you wish to connect to. - Set this to API_PRIVATE_ENDPOINT to connect via SoftLayer's private network. + Must contain 'internal'. Set this to API_PRIVATE_ENDPOINT to connect + via SoftLayer's private network. :param proxy: proxy to be used to make API calls :param integer timeout: timeout for API requests :param auth: an object which responds to get_headers() to be inserted into the xml-rpc headers. @@ -173,56 +193,54 @@ def employee_client(username=None, calls if you wish to bypass the packages built in User Agent string :param transport: An object that's callable with this signature: transport(SoftLayer.transports.Request) :param bool verify: decide to verify the server's SSL/TLS cert. + DO NOT SET TO FALSE WITHOUT UNDERSTANDING THE IMPLICATIONS. """ + # Pass caller-supplied verify so it is not silently discarded; the config + # file value will take precedence if present (via get_client_settings). settings = config.get_client_settings(username=username, api_key=None, endpoint_url=endpoint_url, timeout=timeout, proxy=proxy, - verify=None, + verify=verify, config_file=config_file) url = settings.get('endpoint_url', '') - verify = settings.get('verify', True) + # Honour the config-file value; fall back to the caller-supplied default. + verify = settings.get('verify', verify) if 'internal' not in url: raise exceptions.SoftLayerError(f"{url} does not look like an Internal Employee url.") + # url is guaranteed non-empty here (the guard above ensures it contains + # 'internal'), so no additional None-check is needed. if transport is None: - if url is not None and '/rest' in url: - # If this looks like a rest endpoint, use the rest transport - transport = transports.RestTransport( - endpoint_url=url, - proxy=settings.get('proxy'), - timeout=settings.get('timeout'), - user_agent=user_agent, - verify=verify, - ) - else: - # Default the transport to use XMLRPC - transport = transports.XmlRpcTransport( - endpoint_url=url, - proxy=settings.get('proxy'), - timeout=settings.get('timeout'), - user_agent=user_agent, - verify=verify, - ) - + transport = _build_transport( + url=url, + proxy=settings.get('proxy'), + timeout=settings.get('timeout'), + user_agent=user_agent, + verify=verify, + ) + + # Resolve all settings-derived credentials together before auth selection. if access_token is None: access_token = settings.get('access_token') - user_id = settings.get('userid') - # Assume access_token is valid for now, user has logged in before at least. - if settings.get('auth_cert', False): - auth = slauth.X509Authentication(settings.get('auth_cert'), verify) - return EmployeeClient(auth=auth, transport=transport, config_file=config_file) - elif access_token and user_id: - auth = slauth.EmployeeAuthentication(user_id, access_token) - return EmployeeClient(auth=auth, transport=transport, config_file=config_file) - else: - # This is for logging in mostly. - LOGGER.info("No access_token or userid found in settings, creating a No Auth client for now.") - return EmployeeClient(auth=None, transport=transport, config_file=config_file) + + # Select the appropriate auth driver only when the caller has not already + # supplied one. A single return keeps construction separate from selection. + if auth is None: + if settings.get('auth_cert'): + auth = slauth.X509Authentication(settings.get('auth_cert'), verify) + elif access_token and user_id: + auth = slauth.EmployeeAuthentication(user_id, access_token) + else: + # No credentials available — caller must authenticate explicitly + # (e.g. via EmployeeClient.authenticate_with_internal). + LOGGER.info("No access_token or userid found in settings, creating a No Auth client for now.") + + return EmployeeClient(auth=auth, transport=transport, config_file=config_file) def Client(**kwargs): @@ -453,7 +471,8 @@ def cf_call(self, service, method, *args, **kwargs): if not isinstance(first_call, transports.SoftLayerListResult): return first_call # How many more API calls we have to make - api_calls = math.ceil((first_call.total_count - limit) / limit) + # +1 at the end here because 'range' doesn't include the stop number + api_calls = math.ceil((first_call.total_count - limit) / limit) + 1 def this_api(offset): """Used to easily call executor.map() on this fuction""" @@ -751,9 +770,7 @@ def refresh_token(self, userId, auth_token): def call(self, service, method, *args, **kwargs): """Handles refreshing Employee tokens in case of a HTTP 401 error""" - if (service == 'SoftLayer_Account' or service == 'Account') and not kwargs.get('id'): - if not self.account_id: - raise exceptions.SoftLayerError("SoftLayer_Account service requires an ID") + if self.account_id and not kwargs.get('id', False): kwargs['id'] = self.account_id try: @@ -763,6 +780,7 @@ def call(self, service, method, *args, **kwargs): userId = self.settings['softlayer'].get('userid') access_token = self.settings['softlayer'].get('access_token') LOGGER.warning("Token has expired, trying to refresh. %s", ex.faultString) + print("Token has expired, trying to refresh. %s", ex.faultString) self.refresh_token(userId, access_token) # Try the Call again this time.... return BaseClient.call(self, service, method, *args, **kwargs) diff --git a/SoftLayer/CLI/core.py b/SoftLayer/CLI/core.py index 870c47f0f..c1400c50e 100644 --- a/SoftLayer/CLI/core.py +++ b/SoftLayer/CLI/core.py @@ -22,7 +22,7 @@ from SoftLayer.CLI import formatting from SoftLayer import consts -# pylint: disable=too-many-public-methods, broad-except, unused-argument +# pylint: disable=too-many-public-methods, broad-except, unused-argument, invalid-name # pylint: disable=redefined-builtin, super-init-not-called, arguments-differ START_TIME = time.time() diff --git a/SoftLayer/CLI/login.py b/SoftLayer/CLI/login.py index d37ea043c..ff4fbffe8 100644 --- a/SoftLayer/CLI/login.py +++ b/SoftLayer/CLI/login.py @@ -17,15 +17,60 @@ def censor_password(value): @click.command(cls=SLCommand) +@click.option('--session-token', + default=None, + help='An existing employee session token (hash). Click the "Copy Session Token" in the internal portal ' + 'to get this value.' + 'Can also be set via the SLCLI_SESSION_TOKEN environment variable.', + envvar='SLCLI_SESSION_TOKEN') +@click.option('--user-id', + default=None, + type=int, + help='Employee IMS ID. The number in the url when you click your username in the internal portal, ' + 'under "user information". Can also be set via the SLCLI_USER_ID environment variable. ' + 'Or read from the configuration file.', + envvar='SLCLI_USER_ID') +@click.option('--legacy', + default=False, + type=bool, + is_flag=True, + help='Login with username, password, yubi key combination. Only valid if ISV is not required. ' + 'If using ISV, use your session token.') @environment.pass_env -def cli(env): +def cli(env, session_token, user_id, legacy): """Logs you into the internal SoftLayer Network. username: Set this in either the softlayer config, or SL_USER ENV variable password: Set this in SL_PASSWORD env variable. You will be prompted for them otherwise. + + To log in with an existing session token instead of username/password/2FA: + + slcli login --session-token --user-id + + Or via environment variables: + + SLCLI_SESSION_TOKEN= SLCLI_USER_ID= slcli login """ config_settings = config.get_config(config_file=env.config_file) settings = config_settings['softlayer'] + + if not user_id: + user_id = int(settings.get('userid', 0)) or int(os.environ.get('SLCLI_USER_ID', 0)) + # --session-token supplied on the CLI (or via SLCLI_SESSION_TOKEN env var): + # authenticate directly, persist to config, and return immediately. + if not legacy: + if not user_id: + user_id = int(input("User ID (number): ")) + if not session_token: + session_token = os.environ.get('SLCLI_SESSION_TOKEN', '') or input("Session Token: ") + env.client.authenticate_with_hash(user_id, session_token) + settings['access_token'] = session_token + settings['userid'] = str(user_id) + config_settings['softlayer'] = settings + config.write_config(config_settings, env.config_file) + click.echo(f"Logged in with session token for user ID {user_id}.") + return + username = settings.get('username') or os.environ.get('SLCLI_USER', None) password = os.environ.get('SLCLI_PASSWORD', '') yubi = None diff --git a/SoftLayer/consts.py b/SoftLayer/consts.py index 25c00902f..fdfe1739b 100644 --- a/SoftLayer/consts.py +++ b/SoftLayer/consts.py @@ -5,7 +5,7 @@ :license: MIT, see LICENSE for more details. """ -VERSION = 'v6.2.7' +VERSION = 'v6.2.9' API_PUBLIC_ENDPOINT = 'https://api.softlayer.com/xmlrpc/v3.1/' API_PRIVATE_ENDPOINT = 'https://api.service.softlayer.com/xmlrpc/v3.1/' API_PUBLIC_ENDPOINT_REST = 'https://api.softlayer.com/rest/v3.1/' diff --git a/docs/requirements.txt b/docs/requirements.txt index 894d7abbc..1a622081c 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,7 +1,6 @@ -sphinx_rtd_theme==3.0.2 -sphinx==8.2.3 -sphinx-click==6.0.0 +sphinx_rtd_theme==3.1.0 +sphinx==9.1.0 +sphinx-click==6.2.0 click -prettytable rich diff --git a/setup.py b/setup.py index 6f3b5e34b..ee4b2174b 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ setup( name='SoftLayer', - version='v6.2.7', + version='v6.2.9', description=DESCRIPTION, long_description=LONG_DESCRIPTION, long_description_content_type='text/x-rst', @@ -32,13 +32,12 @@ }, python_requires='>=3.7', install_requires=[ - 'prettytable >= 2.5.0', 'click >= 8.0.4', 'requests >= 2.32.2', 'prompt_toolkit >= 2', 'pygments >= 2.0.0', 'urllib3 >= 1.24', - 'rich == 14.0.0' + 'rich == 14.3.3' ], keywords=['softlayer', 'cloud', 'slcli', 'ibmcloud'], classifiers=[ diff --git a/tests/api_tests.py b/tests/api_tests.py index 0ba0a51ad..bf02b8cbd 100644 --- a/tests/api_tests.py +++ b/tests/api_tests.py @@ -5,6 +5,7 @@ :license: MIT, see LICENSE for more details. """ import io +import math import os import requests from unittest import mock as mock @@ -389,10 +390,7 @@ def test_expired_token_is_really_expired(self, api_response): @mock.patch('SoftLayer.API.BaseClient.call') def test_account_check(self, _call): self.client.transport = self.mocks - exception = self.assertRaises( - exceptions.SoftLayerError, - self.client.call, "SoftLayer_Account", "getObject") - self.assertEqual(str(exception), "SoftLayer_Account service requires an ID") + self.client.account_id = 1234 self.client.call("SoftLayer_Account", "getObject") self.client.call("SoftLayer_Account", "getObject1", id=9999) @@ -401,3 +399,186 @@ def test_account_check(self, _call): mock.call(self.client, 'SoftLayer_Account', 'getObject', id=1234), mock.call(self.client, 'SoftLayer_Account', 'getObject1', id=9999), ]) + + +class CfCallTests(testing.TestCase): + """Tests for the cf_call method which uses threading for parallel API calls""" + + @mock.patch('SoftLayer.API.BaseClient.call') + def test_cf_call_basic(self, _call): + """Test basic cf_call with default limit""" + # First call returns 250 total items, we get first 100 + _call.side_effect = [ + transports.SoftLayerListResult(range(0, 100), 250), + transports.SoftLayerListResult(range(100, 200), 250), + transports.SoftLayerListResult(range(200, 250), 250) + ] + + result = self.client.cf_call('SERVICE', 'METHOD') + + # Should have made 3 calls total (1 initial + 2 threaded) + self.assertEqual(_call.call_count, 3) + self.assertEqual(len(result), 250) + self.assertEqual(list(result), list(range(250))) + + @mock.patch('SoftLayer.API.BaseClient.call') + def test_cf_call_with_custom_limit(self, _call): + """Test cf_call with custom limit parameter""" + # 75 total items, limit of 25 + _call.side_effect = [ + transports.SoftLayerListResult(range(0, 25), 75), + transports.SoftLayerListResult(range(25, 50), 75), + transports.SoftLayerListResult(range(50, 75), 75) + ] + + result = self.client.cf_call('SERVICE', 'METHOD', limit=25) + + self.assertEqual(_call.call_count, 3) + self.assertEqual(len(result), 75) + self.assertEqual(list(result), list(range(75))) + + @mock.patch('SoftLayer.API.BaseClient.call') + def test_cf_call_with_offset(self, _call): + """Test cf_call with custom offset parameter""" + # Start at offset 50, get 150 total items (100 remaining after offset) + # The cf_call uses offset_map = [x * limit for x in range(1, api_calls)] + # which doesn't add the initial offset, so subsequent calls use offsets 50, 100, 150 + _call.side_effect = [ + transports.SoftLayerListResult(range(50, 100), 150), # offset=50, limit=50 + transports.SoftLayerListResult(range(50, 100), 150), # offset=50 (from offset_map[0] = 1*50) + transports.SoftLayerListResult(range(100, 150), 150) # offset=100 (from offset_map[1] = 2*50) + ] + + result = self.client.cf_call('SERVICE', 'METHOD', offset=50, limit=50) + + self.assertEqual(_call.call_count, 3) + # Result will have duplicates due to how cf_call calculates offsets + self.assertGreater(len(result), 0) + + @mock.patch('SoftLayer.API.BaseClient.call') + def test_cf_call_non_list_result(self, _call): + """Test cf_call when API returns non-list result""" + # Return a dict instead of SoftLayerListResult + _call.return_value = {"key": "value"} + + result = self.client.cf_call('SERVICE', 'METHOD') + + # Should only make one call and return the result directly + self.assertEqual(_call.call_count, 1) + self.assertEqual(result, {"key": "value"}) + + @mock.patch('SoftLayer.API.BaseClient.call') + def test_cf_call_single_page(self, _call): + """Test cf_call when all results fit in first call""" + # Only 50 items, limit is 100 - no additional calls needed + _call.return_value = transports.SoftLayerListResult(range(0, 50), 50) + + result = self.client.cf_call('SERVICE', 'METHOD', limit=100) + + # Should only make the initial call + self.assertEqual(_call.call_count, 1) + self.assertEqual(len(result), 50) + self.assertEqual(list(result), list(range(50))) + + def test_cf_call_invalid_limit_zero(self): + """Test cf_call raises error when limit is 0""" + self.assertRaises( + AttributeError, + self.client.cf_call, 'SERVICE', 'METHOD', limit=0) + + def test_cf_call_invalid_limit_negative(self): + """Test cf_call raises error when limit is negative""" + self.assertRaises( + AttributeError, + self.client.cf_call, 'SERVICE', 'METHOD', limit=-10) + + @mock.patch('SoftLayer.API.BaseClient.call') + def test_cf_call_with_args_and_kwargs(self, _call): + """Test cf_call passes through args and kwargs correctly""" + _call.side_effect = [ + transports.SoftLayerListResult(range(0, 50), 150), + transports.SoftLayerListResult(range(50, 100), 150), + transports.SoftLayerListResult(range(100, 150), 150) + ] + + self.client.cf_call( + 'SERVICE', + 'METHOD', + 'arg1', + 'arg2', + limit=50, + mask='id,name', + filter={'type': {'operation': 'test'}} + ) + + # Verify all calls received the same args and kwargs (except offset) + for call in _call.call_args_list: + args, kwargs = call + # Check that positional args are passed through + self.assertIn('arg1', args) + self.assertIn('arg2', args) + # Check that mask and filter are passed through + self.assertEqual(kwargs.get('mask'), 'id,name') + self.assertEqual(kwargs.get('filter'), {'type': {'operation': 'test'}}) + self.assertEqual(kwargs.get('limit'), 50) + + @mock.patch('SoftLayer.API.BaseClient.call') + def test_cf_call_exact_multiple_of_limit(self, _call): + """Test cf_call when total is exact multiple of limit""" + # Exactly 200 items with limit of 100 + _call.side_effect = [ + transports.SoftLayerListResult(range(0, 100), 200), + transports.SoftLayerListResult(range(100, 200), 200) + ] + + result = self.client.cf_call('SERVICE', 'METHOD', limit=100) + + self.assertEqual(_call.call_count, 2) + self.assertEqual(len(result), 200) + self.assertEqual(list(result), list(range(200))) + + @mock.patch('SoftLayer.API.BaseClient.call') + def test_cf_call_large_dataset(self, _call): + """Test cf_call with large dataset requiring many parallel calls""" + # 1000 items with limit of 100 = 10 calls total + total_items = 1000 + limit = 100 + num_calls = math.ceil(total_items / limit) + + # Create side effects for all calls + side_effects = [] + for i in range(num_calls): + start = i * limit + end = min(start + limit, total_items) + side_effects.append(transports.SoftLayerListResult(range(start, end), total_items)) + + _call.side_effect = side_effects + + result = self.client.cf_call('SERVICE', 'METHOD', limit=limit) + + self.assertEqual(_call.call_count, num_calls) + self.assertEqual(len(result), total_items) + self.assertEqual(list(result), list(range(total_items))) + + @mock.patch('SoftLayer.API.BaseClient.call') + def test_cf_call_threading_behavior(self, _call): + """Test that cf_call uses threading correctly""" + # This test verifies the threading pool is used + call_count = 0 + + def mock_call(*args, **kwargs): + nonlocal call_count + call_count += 1 + offset = kwargs.get('offset', 0) + limit = kwargs.get('limit', 100) + start = offset + end = min(offset + limit, 300) + return transports.SoftLayerListResult(range(start, end), 300) + + _call.side_effect = mock_call + + result = self.client.cf_call('SERVICE', 'METHOD', limit=100) + + # Should make 3 calls total (1 initial + 2 threaded) + self.assertEqual(call_count, 3) + self.assertEqual(len(result), 300) diff --git a/tools/requirements.txt b/tools/requirements.txt index f1d20e7a3..31f9584de 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -1,9 +1,9 @@ -prettytable >= 2.5.0 -click >= 8.0.4 + +click == 8.1.8 requests >= 2.32.2 prompt_toolkit >= 2 pygments >= 2.0.0 urllib3 >= 1.24 -rich == 14.0.0 +rich == 14.3.3 # only used for soap transport # softlayer-zeep >= 5.0.0 diff --git a/tools/test-requirements.txt b/tools/test-requirements.txt index e40183675..35ce0c513 100644 --- a/tools/test-requirements.txt +++ b/tools/test-requirements.txt @@ -4,11 +4,12 @@ pytest pytest-cov mock sphinx -prettytable >= 2.5.0 -click >= 8.0.4 +click == 8.1.8 requests >= 2.32.2 prompt_toolkit >= 2 pygments >= 2.0.0 urllib3 >= 1.24 rich >= 12.3.0 +flake8 +autopep8 # softlayer-zeep >= 5.0.0