From c2ce2edd3d9b642ad5c603102edb4b768bb04feb Mon Sep 17 00:00:00 2001 From: psaffrey Date: Mon, 28 Sep 2015 15:36:21 +0100 Subject: [PATCH 01/73] Helper classes for BaseMount and app launch --- src/BaseSpacePy/api/BaseSpaceAPI.py | 92 ++++++++++++++--------------- 1 file changed, 43 insertions(+), 49 deletions(-) diff --git a/src/BaseSpacePy/api/BaseSpaceAPI.py b/src/BaseSpacePy/api/BaseSpaceAPI.py index ba8ccbe..d114d7e 100755 --- a/src/BaseSpacePy/api/BaseSpaceAPI.py +++ b/src/BaseSpacePy/api/BaseSpaceAPI.py @@ -37,7 +37,7 @@ class BaseSpaceAPI(BaseAPI): ''' The main API class used for all communication with the REST server ''' - def __init__(self, clientKey=None, clientSecret=None, apiServer=None, version=None, appSessionId='', AccessToken='', userAgent=None, timeout=10, verbose=0, profile='DEFAULT'): + def __init__(self, clientKey=None, clientSecret=None, apiServer=None, version='v1pre3', appSessionId='', AccessToken='', userAgent=None, timeout=10, verbose=0, profile='default'): ''' The following arguments are required in either the constructor or a config file (~/.basespacepy.cfg): @@ -84,27 +84,12 @@ def _setCredentials(self, clientKey, clientSecret, apiServer, apiVersion, appSes ''' lcl_cred = self._getLocalCredentials(profile) cred = {} - # required credentials - if clientKey is not None: - cred['clientKey'] = clientKey - else: - try: - cred['clientKey'] = lcl_cred['clientKey'] - except KeyError: - raise CredentialsException('Client Key not available - please provide in BaseSpaceAPI constructor or config file') - else: - # set profile name - if 'name' in lcl_cred: - cred['profile'] = lcl_cred['name'] - else: - cred['profile'] = profile - if clientSecret is not None: - cred['clientSecret'] = clientSecret + # set profile name + if 'name' in lcl_cred: + cred['profile'] = lcl_cred['name'] else: - try: - cred['clientSecret'] = lcl_cred['clientSecret'] - except KeyError: - raise CredentialsException('Client Secret not available - please provide in BaseSpaceAPI constructor or config file') + cred['profile'] = profile + # required credentials if apiServer is not None: cred['apiServer'] = apiServer else: @@ -118,8 +103,31 @@ def _setCredentials(self, clientKey, clientSecret, apiServer, apiVersion, appSes try: cred['apiVersion'] = lcl_cred['apiVersion'] except KeyError: - raise CredentialsException('API version available - please provide in BaseSpaceAPI constructor or config file') - # Optional credentials + raise CredentialsException('API version not available - please provide in BaseSpaceAPI constructor or config file') + if accessToken: + cred['accessToken'] = accessToken + elif 'accessToken' in lcl_cred: + try: + cred['accessToken'] = lcl_cred['accessToken'] + except KeyError: + raise CredentialsException('Access token not available - please provide in BaseSpaceAPI constructor or config file') + else: + cred['accessToken'] = accessToken + # Optional credentials + if clientKey is not None: + cred['clientKey'] = clientKey + else: + try: + cred['clientKey'] = lcl_cred['clientKey'] + except KeyError: + cred['clientKey'] = clientKey + if clientSecret is not None: + cred['clientSecret'] = clientSecret + else: + try: + cred['clientSecret'] = lcl_cred['clientSecret'] + except KeyError: + cred['clientSecret'] = clientSecret if appSessionId: cred['appSessionId'] = appSessionId elif 'apiVersion' in lcl_cred: @@ -129,59 +137,45 @@ def _setCredentials(self, clientKey, clientSecret, apiServer, apiVersion, appSes cred['appSessionId'] = appSessionId else: cred['appSessionId'] = appSessionId - - if accessToken: - cred['accessToken'] = accessToken - elif 'accessToken' in lcl_cred: - try: - cred['accessToken'] = lcl_cred['accessToken'] - except KeyError: - cred['accessToken'] = accessToken - else: - cred['accessToken'] = accessToken - + return cred def _getLocalCredentials(self, profile): ''' - Returns credentials from local config file (~/.basespacepy.cfg) + Returns credentials from local config file (~/.basespace/.cfg) If some or all credentials are missing, they aren't included the in the returned dict - :param profile: Profile name from local config file + :param profile: Profile name to use to find local config file :returns: A dictionary with credentials from local config file ''' - config_file = os.path.expanduser('~/.basespacepy.cfg') + config_file = os.path.join(os.path.expanduser('~/.basespace'), "%s.cfg" % profile) + section_name = "DEFAULT" cred = {} config = ConfigParser.SafeConfigParser() if config.read(config_file): - if not config.has_section(profile) and profile.lower() != 'default': - raise CredentialsException("Profile name '%s' not present in config file %s" % (profile, config_file)) - try: - cred['name'] = config.get(profile, "name") - except ConfigParser.NoOptionError: - pass + cred['name'] = profile try: - cred['clientKey'] = config.get(profile, "clientKey") + cred['clientKey'] = config.get(section_name, "clientKey") except ConfigParser.NoOptionError: pass try: - cred['clientSecret'] = config.get(profile, "clientSecret") + cred['clientSecret'] = config.get(section_name, "clientSecret") except ConfigParser.NoOptionError: pass try: - cred['apiServer'] = config.get(profile, "apiServer") + cred['apiServer'] = config.get(section_name, "apiServer") except ConfigParser.NoOptionError: pass try: - cred['apiVersion'] = config.get(profile, "apiVersion") + cred['apiVersion'] = config.get(section_name, "apiVersion") except ConfigParser.NoOptionError: pass try: - cred['appSessionId'] = config.get(profile, "appSessionId") + cred['appSessionId'] = config.get(section_name, "appSessionId") except ConfigParser.NoOptionError: pass try: - cred['accessToken'] = config.get(profile, "accessToken") + cred['accessToken'] = config.get(section_name, "accessToken") except ConfigParser.NoOptionError: pass return cred From 567d8627188fea12432b21534b195a840fd840e6 Mon Sep 17 00:00:00 2001 From: psaffrey Date: Tue, 6 Oct 2015 12:23:42 +0100 Subject: [PATCH 02/73] change to setup requirements name --- src/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setup.py b/src/setup.py index d98dca5..f8fbc5a 100755 --- a/src/setup.py +++ b/src/setup.py @@ -33,7 +33,7 @@ author_email='', packages=['BaseSpacePy.api','BaseSpacePy.model','BaseSpacePy'], package_dir={'BaseSpacePy' : os.path.join(os.path.dirname(__file__),'BaseSpacePy')}, - requires=['pycurl','dateutil'], + install_requires=['pycurl','python-dateutil'], zip_safe=False, ) From 0ea9ba912edebaa62b4d640ce82d01ff0c7940ab Mon Sep 17 00:00:00 2001 From: psaffrey Date: Tue, 6 Oct 2015 14:51:08 +0100 Subject: [PATCH 03/73] Some missing validation for missing paths --- src/BaseSpacePy/api/BaseMountInterface.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/BaseSpacePy/api/BaseMountInterface.py b/src/BaseSpacePy/api/BaseMountInterface.py index 74c8fc2..47868b2 100644 --- a/src/BaseSpacePy/api/BaseMountInterface.py +++ b/src/BaseSpacePy/api/BaseMountInterface.py @@ -1,7 +1,7 @@ """ class to wrap a directory mounted using BaseMount and provide convenience functions to get at the metadata stored there -The currently supported metadata extraction uses the files created by metaBSFS, but is limited to first class entities like projects and samples +The currently supported metadata extraction uses the files created by BaseMount, but is limited to first class entities like projects and samples it will fail (and throw an exception) when pointing at other directories. For some of these it's not really clear what the behaviour *should* be eg. ~/BaseSpace/current_user/Projects/Sloths\ Test/Samples/ which is the owning directory for the "Sloths Test" samples. Should this be a directory of type "project" and id of the "Hyperion Test" project? @@ -32,8 +32,10 @@ def __init__(self, path): def __validate_basemount__(self): """ - Checks whether the chosen directory is a BSFS directory + Checks whether the chosen directory is a BaseMount directory """ + if not os.path.exists(self.path): + return False if os.path.isdir(self.path): for required in REQUIRED_ENTRIES: required_path = os.path.join(self.path, required) @@ -43,7 +45,7 @@ def __validate_basemount__(self): def __resolve_details__(self): """ - pull the useful details out of the . files generated by metaBSFS + pull the useful details out of the . files generated by BaseMount """ if os.path.isdir(self.path): type_file = os.path.join(self.path, ".type") From b3fd0b1e9c5fcaec0889f5fbae5dbaa7acc27fe7 Mon Sep 17 00:00:00 2001 From: psaffrey Date: Tue, 6 Oct 2015 15:02:00 +0100 Subject: [PATCH 04/73] syntax changes for Python2.6 compatibility --- src/BaseSpacePy/api/AppLaunchHelpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BaseSpacePy/api/AppLaunchHelpers.py b/src/BaseSpacePy/api/AppLaunchHelpers.py index 2db5944..a7bbf33 100644 --- a/src/BaseSpacePy/api/AppLaunchHelpers.py +++ b/src/BaseSpacePy/api/AppLaunchHelpers.py @@ -364,7 +364,7 @@ def make_launch_json(self, user_supplied_vars, launch_name, sample_attributes={} if required_vars - supplied_var_names: raise LaunchSpecificationException( "Compulsory variable(s) missing! (%s)" % str(required_vars - supplied_var_names)) - if supplied_var_names - (self.get_variable_requirements() | {"LaunchName"}): + if supplied_var_names - (self.get_variable_requirements() | set("LaunchName")): print "warning! unused variable(s) specified: (%s)" % str( supplied_var_names - self.get_variable_requirements()) all_vars = copy.copy(self.defaults) From ab3014dab2a70e6a5e8b6cb32c3248ca472177e0 Mon Sep 17 00:00:00 2001 From: psaffrey Date: Tue, 6 Oct 2015 15:08:45 +0100 Subject: [PATCH 05/73] syntax changes for Python2.6 compatibility --- src/BaseSpacePy/api/AppLaunchHelpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BaseSpacePy/api/AppLaunchHelpers.py b/src/BaseSpacePy/api/AppLaunchHelpers.py index a7bbf33..165721a 100644 --- a/src/BaseSpacePy/api/AppLaunchHelpers.py +++ b/src/BaseSpacePy/api/AppLaunchHelpers.py @@ -364,7 +364,7 @@ def make_launch_json(self, user_supplied_vars, launch_name, sample_attributes={} if required_vars - supplied_var_names: raise LaunchSpecificationException( "Compulsory variable(s) missing! (%s)" % str(required_vars - supplied_var_names)) - if supplied_var_names - (self.get_variable_requirements() | set("LaunchName")): + if supplied_var_names - (self.get_variable_requirements() | set(["LaunchName"])): print "warning! unused variable(s) specified: (%s)" % str( supplied_var_names - self.get_variable_requirements()) all_vars = copy.copy(self.defaults) From 10e296c8ce2e013d8edde5379de9315726b5290e Mon Sep 17 00:00:00 2001 From: Mauricio Varea Date: Thu, 8 Oct 2015 15:59:01 +0100 Subject: [PATCH 06/73] trying out DEB generation --- src/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setup.py b/src/setup.py index f8fbc5a..ff46f51 100755 --- a/src/setup.py +++ b/src/setup.py @@ -33,7 +33,7 @@ author_email='', packages=['BaseSpacePy.api','BaseSpacePy.model','BaseSpacePy'], package_dir={'BaseSpacePy' : os.path.join(os.path.dirname(__file__),'BaseSpacePy')}, - install_requires=['pycurl','python-dateutil'], + install_requires=['pycurl','python-dateutil','stdeb'], zip_safe=False, ) From 1dd13a4d809f07610737de69ae736d25ef04898d Mon Sep 17 00:00:00 2001 From: Mauricio Varea Date: Thu, 8 Oct 2015 16:58:38 +0100 Subject: [PATCH 07/73] stdeb only used at setup stage --- src/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/setup.py b/src/setup.py index ff46f51..cf94a2e 100755 --- a/src/setup.py +++ b/src/setup.py @@ -33,7 +33,8 @@ author_email='', packages=['BaseSpacePy.api','BaseSpacePy.model','BaseSpacePy'], package_dir={'BaseSpacePy' : os.path.join(os.path.dirname(__file__),'BaseSpacePy')}, - install_requires=['pycurl','python-dateutil','stdeb'], + install_requires=['pycurl','python-dateutil'], + setup_requires=['stdeb'], zip_safe=False, ) From 763b7ce78154fa7486cf62ea302859f471fcf089 Mon Sep 17 00:00:00 2001 From: psaffrey Date: Fri, 9 Oct 2015 13:32:17 +0100 Subject: [PATCH 08/73] additional error checking --- src/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setup.py b/src/setup.py index cf94a2e..2d879d1 100755 --- a/src/setup.py +++ b/src/setup.py @@ -33,7 +33,7 @@ author_email='', packages=['BaseSpacePy.api','BaseSpacePy.model','BaseSpacePy'], package_dir={'BaseSpacePy' : os.path.join(os.path.dirname(__file__),'BaseSpacePy')}, - install_requires=['pycurl','python-dateutil'], + install_requires=['pycurl','python-dateutil','requests'], setup_requires=['stdeb'], zip_safe=False, ) From beb4d9a3c4ac1cdbfbf73fdc8b386e644a8dc7fe Mon Sep 17 00:00:00 2001 From: psaffrey Date: Fri, 9 Oct 2015 13:32:36 +0100 Subject: [PATCH 09/73] Additional error checking --- src/BaseSpacePy/api/AppLaunchHelpers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/BaseSpacePy/api/AppLaunchHelpers.py b/src/BaseSpacePy/api/AppLaunchHelpers.py index 165721a..d861e10 100644 --- a/src/BaseSpacePy/api/AppLaunchHelpers.py +++ b/src/BaseSpacePy/api/AppLaunchHelpers.py @@ -422,6 +422,9 @@ def __init__(self, launch_spec, args, configoptions): self._launch_spec = launch_spec self._args = args self._configoptions = configoptions + varnames = self._launch_spec.get_minimum_requirements() + if len(varnames) != len(self._args): + raise LaunchSpecificationException("Number of arguments does not match specification") def _find_all_entity_names(self, entity_type): """ From c59eb023250baa6582351408b57f41f29ea0b0b0 Mon Sep 17 00:00:00 2001 From: psaffrey Date: Fri, 9 Oct 2015 13:36:27 +0100 Subject: [PATCH 10/73] added authenticate script --- .gitignore | 1 - bin/authenticate.py | 157 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 bin/authenticate.py diff --git a/.gitignore b/.gitignore index 1e0f5da..1463c0f 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,6 @@ build eggs doc_build parts -bin var sdist /scripts diff --git a/bin/authenticate.py b/bin/authenticate.py new file mode 100644 index 0000000..6e8613a --- /dev/null +++ b/bin/authenticate.py @@ -0,0 +1,157 @@ +import json +import time +import sys + +__author__ = 'psaffrey' + +""" +Script that sets up the files in .basespace to contain access tokens. + +One way uses the OAuth flow for web application authentication: + +https://developer.basespace.illumina.com/docs/content/documentation/authentication/obtaining-access-tokens + +to get an access token and put it in the proper place. + +Also supported is obtaining session tokens (cookies), although these are not currently used. + +""" + +import getpass +import requests +import os +import ConfigParser + +SESSION_AUTH_URI = "https://accounts.illumina.com/" +DEFAULT_CONFIG_NAME = "DEFAULT" +SESSION_TOKEN_NAME = "sessionToken" +ACCESS_TOKEN_NAME = "accessToken" +API_SERVER = "https://api.basespace.illumina.com/" +API_VERSION = "v1pre3" +OAUTH_URI = "%s%s/oauthv2/deviceauthorization" % (API_SERVER, API_VERSION) +TOKEN_URI = "%s%s/oauthv2/token" % (API_SERVER, API_VERSION) +WAIT_TIME = 5.0 + +# these are the details for the BaseSpaceCLI app +# shared with BaseMount +CLIENT_ID = "ca2e493333b044a18d65385afaf8eb5d" +CLIENT_SECRET = "282b0f7d4e5d48dfabc7cdfe5b3156a6" +SCOPE="CREATE GLOBAL,BROWSE GLOBAL,CREATE PROJECTS,READ GLOBAL" + +def basespace_session(username, password): + s = requests.session() + payload = {"UserName": username, + "Password": password, + "ReturnUrl": "http://developer.basespace.illumina.com/dashboard"} + r = s.post(url=SESSION_AUTH_URI, + params={'Service': 'basespace'}, + data=payload, + headers={'Content-Type': "application/x-www-form-urlencoded"}, + allow_redirects=False) + return s, r + + +def check_session_details(): + pass + + +def set_session_details(config_path): + username = raw_input("username:") + password = getpass.getpass() + s, r = basespace_session(username, password) + config = parse_config(config_path) + config.set(DEFAULT_CONFIG_NAME, SESSION_TOKEN_NAME, ) + with open(config_path, "w") as fh: + config.write(fh) + + +def parse_config(config_path): + """ + parses the config_path or creates it if it doesn't exist + + :param config_path: path to config file + :return: ConfigParser object + """ + if not os.path.exists(config_path): + config = ConfigParser.SafeConfigParser() + else: + config = ConfigParser.SafeConfigParser() + config.read(config_path) + return config + +def construct_default_config(config): + config.set(DEFAULT_CONFIG_NAME, "apiServer", API_SERVER) + +def set_oauth_details(config_path): + s = requests.session() + # make the initial request + auth_payload = { + "response_type" : "device_code", + "client_id" : CLIENT_ID, + "scope" : SCOPE, + } + try: + r = s.post(url=OAUTH_URI, + data=auth_payload) + except Exception as e: + print "problem communicate with oauth server: %s" % str(e) + raise + # show the URL to the user + auth_url = r.json()["verification_with_code_uri"] + auth_code = r.json()["device_code"] + print "please authenticate here: %s" % auth_url + # poll the token URL until we get the token + token_payload = { + "client_id" : CLIENT_ID, + "client_secret" : CLIENT_SECRET, + "code" : auth_code, + "grant_type" : "device" + } + access_token = None + while 1: + # put the token into the config file + r = s.post(url=TOKEN_URI, + data=token_payload) + if r.status_code == 400: + sys.stdout.write(".") + sys.stdout.flush() + time.sleep(WAIT_TIME) + else: + access_token = r.json()["access_token"] + break + config = parse_config(config_path) + construct_default_config(config) + if not access_token: + raise Exception("problem obtaining token!") + config.set(DEFAULT_CONFIG_NAME, ACCESS_TOKEN_NAME, access_token) + with open(config_path, "w") as fh: + config.write(fh) + + +if __name__ == "__main__": + from argparse import ArgumentParser + + parser = ArgumentParser(description="Derive BaseSpace authentication details") + + parser.add_argument('-c', '--configname', type=str, dest="configname", default="default", help='name of config') + parser.add_argument('-s', '--sessiontoken', default=False, action="store_true", + help='do session auth, instead of regular auth') + + args = parser.parse_args() + + # cross platform way to get home directory + home = os.path.expanduser("~") + config_path = os.path.join(home, ".basespace", "%s.cfg" % args.configname) + + + if args.sessiontoken: + set_session_details(config_path) + else: + try: + if os.path.exists(config_path): + print "config path already exists; not overwriting (%s)" % config_path + sys.exit(1) + set_oauth_details(config_path) + except Exception as e: + print "authentication failed!" + raise From e5d8749d0f2ae5663fdd1c2cdfd308001fe44fd7 Mon Sep 17 00:00:00 2001 From: psaffrey Date: Fri, 9 Oct 2015 13:43:55 +0100 Subject: [PATCH 11/73] Added ability to specify API server --- bin/authenticate.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/bin/authenticate.py b/bin/authenticate.py index 6e8613a..324f525 100644 --- a/bin/authenticate.py +++ b/bin/authenticate.py @@ -26,10 +26,8 @@ DEFAULT_CONFIG_NAME = "DEFAULT" SESSION_TOKEN_NAME = "sessionToken" ACCESS_TOKEN_NAME = "accessToken" -API_SERVER = "https://api.basespace.illumina.com/" +DEFAULT_API_SERVER = "https://api.basespace.illumina.com/" API_VERSION = "v1pre3" -OAUTH_URI = "%s%s/oauthv2/deviceauthorization" % (API_SERVER, API_VERSION) -TOKEN_URI = "%s%s/oauthv2/token" % (API_SERVER, API_VERSION) WAIT_TIME = 5.0 # these are the details for the BaseSpaceCLI app @@ -79,10 +77,12 @@ def parse_config(config_path): config.read(config_path) return config -def construct_default_config(config): - config.set(DEFAULT_CONFIG_NAME, "apiServer", API_SERVER) +def construct_default_config(config, api_server): + config.set(DEFAULT_CONFIG_NAME, "apiServer", api_server) -def set_oauth_details(config_path): +def set_oauth_details(config_path, api_server): + OAUTH_URI = "%s%s/oauthv2/deviceauthorization" % (api_server, API_VERSION) + TOKEN_URI = "%s%s/oauthv2/token" % (api_server, API_VERSION) s = requests.session() # make the initial request auth_payload = { @@ -120,7 +120,7 @@ def set_oauth_details(config_path): access_token = r.json()["access_token"] break config = parse_config(config_path) - construct_default_config(config) + construct_default_config(config, api_server) if not access_token: raise Exception("problem obtaining token!") config.set(DEFAULT_CONFIG_NAME, ACCESS_TOKEN_NAME, access_token) @@ -136,6 +136,7 @@ def set_oauth_details(config_path): parser.add_argument('-c', '--configname', type=str, dest="configname", default="default", help='name of config') parser.add_argument('-s', '--sessiontoken', default=False, action="store_true", help='do session auth, instead of regular auth') + parser.add_argument('-a', '--api-server', default=DEFAULT_API_SERVER, help="choose backend api server") args = parser.parse_args() @@ -151,7 +152,7 @@ def set_oauth_details(config_path): if os.path.exists(config_path): print "config path already exists; not overwriting (%s)" % config_path sys.exit(1) - set_oauth_details(config_path) + set_oauth_details(config_path, args.api_server) except Exception as e: print "authentication failed!" raise From 562e6a3ebf817c6c4ae329bbbdf576a5377190e8 Mon Sep 17 00:00:00 2001 From: psaffrey Date: Tue, 13 Oct 2015 13:17:30 +0100 Subject: [PATCH 12/73] Added authentication script to derive access_token and error message pointing to it --- bin/authenticate.py | 2 + src/BaseSpacePy/api/BaseSpaceAPI.py | 74 ++++++++++------------------- 2 files changed, 27 insertions(+), 49 deletions(-) diff --git a/bin/authenticate.py b/bin/authenticate.py index 324f525..b8a0501 100644 --- a/bin/authenticate.py +++ b/bin/authenticate.py @@ -117,12 +117,14 @@ def set_oauth_details(config_path, api_server): sys.stdout.flush() time.sleep(WAIT_TIME) else: + sys.stdout.write("\n") access_token = r.json()["access_token"] break config = parse_config(config_path) construct_default_config(config, api_server) if not access_token: raise Exception("problem obtaining token!") + print "Success!" config.set(DEFAULT_CONFIG_NAME, ACCESS_TOKEN_NAME, access_token) with open(config_path, "w") as fh: config.write(fh) diff --git a/src/BaseSpacePy/api/BaseSpaceAPI.py b/src/BaseSpacePy/api/BaseSpaceAPI.py index d114d7e..3da8890 100755 --- a/src/BaseSpacePy/api/BaseSpaceAPI.py +++ b/src/BaseSpacePy/api/BaseSpaceAPI.py @@ -79,10 +79,13 @@ def _setCredentials(self, clientKey, clientSecret, apiServer, apiVersion, appSes :param version: the version of the BaseSpace API :param appSessionId: the AppSession Id :param AccessToken: an access token - :param profile: name of profile in config file + :param profile: name of the config file :returns: dictionary with credentials from constructor, config file, or default (for optional args), in this priority order. ''' lcl_cred = self._getLocalCredentials(profile) + my_path = os.path.dirname(os.path.abspath(__file__)) + authenticate = os.path.abspath(os.path.join(my_path, "..", "..", "..", "bin", "authenticate.py")) + authenticate_cmd = "%s --config %s" % (authenticate, profile) cred = {} # set profile name if 'name' in lcl_cred: @@ -90,55 +93,28 @@ def _setCredentials(self, clientKey, clientSecret, apiServer, apiVersion, appSes else: cred['profile'] = profile # required credentials - if apiServer is not None: - cred['apiServer'] = apiServer - else: - try: - cred['apiServer'] = lcl_cred['apiServer'] - except KeyError: - raise CredentialsException('API Server URL not available - please provide in BaseSpaceAPI constructor or config file') - if apiVersion is not None: - cred['apiVersion'] = apiVersion - else: - try: - cred['apiVersion'] = lcl_cred['apiVersion'] - except KeyError: - raise CredentialsException('API version not available - please provide in BaseSpaceAPI constructor or config file') - if accessToken: - cred['accessToken'] = accessToken - elif 'accessToken' in lcl_cred: - try: - cred['accessToken'] = lcl_cred['accessToken'] - except KeyError: - raise CredentialsException('Access token not available - please provide in BaseSpaceAPI constructor or config file') - else: - cred['accessToken'] = accessToken + REQUIRED = ["accessToken", "apiServer", "apiVersion"] + for conf_item in REQUIRED: + local_value = locals()[conf_item] + if local_value: + cred[conf_item] = local_value + else: + try: + cred[conf_item] = lcl_cred[conf_item] + except KeyError: + raise CredentialsException("%s not found or config %s missing. Try running %s" % (conf_item, profile, authenticate_cmd)) # Optional credentials - if clientKey is not None: - cred['clientKey'] = clientKey - else: - try: - cred['clientKey'] = lcl_cred['clientKey'] - except KeyError: - cred['clientKey'] = clientKey - if clientSecret is not None: - cred['clientSecret'] = clientSecret - else: - try: - cred['clientSecret'] = lcl_cred['clientSecret'] - except KeyError: - cred['clientSecret'] = clientSecret - if appSessionId: - cred['appSessionId'] = appSessionId - elif 'apiVersion' in lcl_cred: - try: - cred['appSessionId'] = lcl_cred['appSessionId'] - except KeyError: - cred['appSessionId'] = appSessionId - else: - cred['appSessionId'] = appSessionId - - return cred + OPTIONAL = ["clientKey", "clientSecret", "appSessionId"] + for conf_item in OPTIONAL: + local_value = locals()[conf_item] + if local_value: + cred[conf_item] = local_value + else: + try: + cred[conf_item] = lcl_cred[conf_item] + except KeyError: + cred[conf_item] = local_value + return cred def _getLocalCredentials(self, profile): ''' From 7fa63a2e3603099b3c34666a82d0c29aa28e6d87 Mon Sep 17 00:00:00 2001 From: psaffrey Date: Tue, 13 Oct 2015 13:17:45 +0100 Subject: [PATCH 13/73] Added access token consistency check --- src/BaseSpacePy/api/AppLaunchHelpers.py | 10 +++++++++- src/BaseSpacePy/api/BaseMountInterface.py | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/BaseSpacePy/api/AppLaunchHelpers.py b/src/BaseSpacePy/api/AppLaunchHelpers.py index d861e10..70584a5 100644 --- a/src/BaseSpacePy/api/AppLaunchHelpers.py +++ b/src/BaseSpacePy/api/AppLaunchHelpers.py @@ -400,6 +400,7 @@ def format_minimum_requirements(self): description = ["%s (%s)" % (varname, self.get_property_type(varname)) for varname in minimum_requirements] return " ".join(description) + class LaunchPayload(object): """ Helper class to faciliate users making the user_supplied_vars dictionary @@ -411,7 +412,7 @@ class LaunchPayload(object): LAUNCH_NAME = "LaunchName" - def __init__(self, launch_spec, args, configoptions): + def __init__(self, launch_spec, args, configoptions, access_token): """ :param launch_spec (LaunchSpecification) :param args (list) list or arguments to the app launch. These could be BaseSpace IDs or BaseMount paths @@ -422,6 +423,7 @@ def __init__(self, launch_spec, args, configoptions): self._launch_spec = launch_spec self._args = args self._configoptions = configoptions + self._access_token = access_token varnames = self._launch_spec.get_minimum_requirements() if len(varnames) != len(self._args): raise LaunchSpecificationException("Number of arguments does not match specification") @@ -488,6 +490,12 @@ def to_basespace_id(self, param_name, varval): raise LaunchSpecificationException("Parameter looks like a path, but does not exist: %s" % varval) if os.path.exists(varval): bmi = BaseMountInterface(varval) + # make sure we have a BaseMount access token to compare - old versions won't have one + # also make sure we've been passed an access token - + # if we haven't, access token consistency checking has been disabled. + if bmi.access_token and self._access_token and bmi.access_token != self._access_token: + raise LaunchSpecificationException( + "Access tokens between launch configuration and referenced BaseMount path do not match: %s" % varval) spec_type = self._launch_spec.get_property_bald_type(param_name) basemount_type = bmi.type if spec_type != basemount_type: diff --git a/src/BaseSpacePy/api/BaseMountInterface.py b/src/BaseSpacePy/api/BaseMountInterface.py index 47868b2..8252ae8 100644 --- a/src/BaseSpacePy/api/BaseMountInterface.py +++ b/src/BaseSpacePy/api/BaseMountInterface.py @@ -25,6 +25,7 @@ def __init__(self, path): self.path = path self.id = None self.type = None + self.access_token = None self.name = os.path.basename(path) if not self.__validate_basemount__(): raise BaseMountInterfaceException("Path: %s does not seem to be a BaseMount path" % self.path) @@ -60,13 +61,30 @@ def __resolve_details__(self): if self.type == "file": metadata_path = self.path.replace("Files", "Files.metadata") id_file = os.path.join(metadata_path, ".id") + config_file = os.path.join(os.path.dirname(self.path), ".basemount", "Config.cfg") else: id_file = os.path.join(self.path, ".id") + config_file = os.path.join(self.path, ".basemount", "Config.cfg") + if os.path.isfile(config_file): + # get the access token if we can + self.access_token = self._get_access_token_from_config(config_file) self.id = open(id_file).read().strip() def __str__(self): return "%s : (%s) : (%s)" % (self.path, self.id, self.type) + def _get_access_token_from_config(self, config_path): + from ConfigParser import SafeConfigParser, NoSectionError, NoOptionError + config = SafeConfigParser() + config.read(config_path) + try: + return config.get("DEFAULT", "accessToken") + except NoOptionError, NoSectionError: + raise BaseMountInterfaceException("malformed BaseMount config: %s" % config_path) + + + + def get_meta_data(self): try: with open(os.path.join(self.path, ".json")) as fh: From 1f8d58ca838ce934790d3799c4976b1281750174 Mon Sep 17 00:00:00 2001 From: psaffrey Date: Tue, 13 Oct 2015 13:39:50 +0100 Subject: [PATCH 14/73] Reduced verbosity of upload output --- .../model/MultipartFileTransfer.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/BaseSpacePy/model/MultipartFileTransfer.py b/src/BaseSpacePy/model/MultipartFileTransfer.py index 4555a37..daee09a 100644 --- a/src/BaseSpacePy/model/MultipartFileTransfer.py +++ b/src/BaseSpacePy/model/MultipartFileTransfer.py @@ -342,11 +342,11 @@ def _setup(self): self.exe.add_workers(self.process_count) self.task_total = fileCount - self.start_chunk + 1 - LOGGER.info("Total File Size %s" % Utils.readable_bytes(total_size)) - LOGGER.info("Using File Part Size %d MB" % self.part_size) - LOGGER.info("Processes %d" % self.process_count) - LOGGER.info("File Chunk Count %d" % self.task_total) - LOGGER.info("Start Chunk %d" % self.start_chunk) + LOGGER.debug("Total File Size %s" % Utils.readable_bytes(total_size)) + LOGGER.debug("Using File Part Size %d MB" % self.part_size) + LOGGER.debug("Processes %d" % self.process_count) + LOGGER.debug("File Chunk Count %d" % self.task_total) + LOGGER.debug("Start Chunk %d" % self.start_chunk) def _start_workers(self): ''' @@ -435,11 +435,11 @@ def _setup(self): self.exe.add_workers(self.process_count) self.task_total = self.file_count - self.start_chunk + 1 - LOGGER.info("Total File Size %s" % Utils.readable_bytes(total_bytes)) - LOGGER.info("Using File Part Size %s MB" % str(self.part_size)) - LOGGER.info("Processes %d" % self.process_count) - LOGGER.info("File Chunk Count %d" % self.file_count) - LOGGER.info("Start Chunk %d" % self.start_chunk) + LOGGER.debug("Total File Size %s" % Utils.readable_bytes(total_bytes)) + LOGGER.debug("Using File Part Size %s MB" % str(self.part_size)) + LOGGER.debug("Processes %d" % self.process_count) + LOGGER.debug("File Chunk Count %d" % self.file_count) + LOGGER.debug("Start Chunk %d" % self.start_chunk) def _start_workers(self): ''' From d542e943d7198a848bd4adcc6b5f0fa43d40d955 Mon Sep 17 00:00:00 2001 From: psaffrey Date: Tue, 13 Oct 2015 14:33:25 +0100 Subject: [PATCH 15/73] Suppressing annoying messages in upload --- src/BaseSpacePy/api/APIClient.py | 4 ++++ src/setup.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/src/BaseSpacePy/api/APIClient.py b/src/BaseSpacePy/api/APIClient.py index d068338..d3c56d0 100644 --- a/src/BaseSpacePy/api/APIClient.py +++ b/src/BaseSpacePy/api/APIClient.py @@ -39,6 +39,10 @@ def __forcePostCall__(self, resourcePath, postData, headers): :returns: server response (a string containing json) ''' import requests + # this cleans up the output at the expense of letting the user know they're in an insecure context... + requests.packages.urllib3.disable_warnings() + import logging + logging.getLogger("requests").setLevel(logging.WARNING) # pycurl is hard to get working, so best to cauterise it into only the functions where it is needed # import pycurl # postData = [(p,postData[p]) for p in postData] diff --git a/src/setup.py b/src/setup.py index 2d879d1..dde058e 100755 --- a/src/setup.py +++ b/src/setup.py @@ -33,6 +33,9 @@ author_email='', packages=['BaseSpacePy.api','BaseSpacePy.model','BaseSpacePy'], package_dir={'BaseSpacePy' : os.path.join(os.path.dirname(__file__),'BaseSpacePy')}, + # this line moves closer to a Python configuration that does not issue the SSLContext warning + # it fails because of missing headers when building a dependency + #install_requires=['pycurl','python-dateutil','pyOpenSSL>=0.13','requests','requests[security]'], install_requires=['pycurl','python-dateutil','requests'], setup_requires=['stdeb'], zip_safe=False, From b76e90b4dfea20bef77e39eb7ccd3be9901d0221 Mon Sep 17 00:00:00 2001 From: psaffrey Date: Wed, 14 Oct 2015 13:57:48 +0100 Subject: [PATCH 16/73] Suppressing further annoying messages in upload BASE-17548 --- src/BaseSpacePy/api/APIClient.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/BaseSpacePy/api/APIClient.py b/src/BaseSpacePy/api/APIClient.py index d3c56d0..d16fa82 100644 --- a/src/BaseSpacePy/api/APIClient.py +++ b/src/BaseSpacePy/api/APIClient.py @@ -39,8 +39,11 @@ def __forcePostCall__(self, resourcePath, postData, headers): :returns: server response (a string containing json) ''' import requests - # this cleans up the output at the expense of letting the user know they're in an insecure context... - requests.packages.urllib3.disable_warnings() + # this tries to clean up the output at the expense of letting the user know they're in an insecure context... + try: + requests.packages.urllib3.disable_warnings() + except: + pass import logging logging.getLogger("requests").setLevel(logging.WARNING) # pycurl is hard to get working, so best to cauterise it into only the functions where it is needed From b743e85f71c13d1fdd47c9f7424b5591b1e94208 Mon Sep 17 00:00:00 2001 From: psaffrey Date: Wed, 14 Oct 2015 13:58:16 +0100 Subject: [PATCH 17/73] Added methods to allow getting all appsessions --- src/BaseSpacePy/api/BaseSpaceAPI.py | 13 ++++++++++--- src/BaseSpacePy/model/Project.py | 3 ++- src/BaseSpacePy/model/QueryParameters.py | 1 + 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/BaseSpacePy/api/BaseSpaceAPI.py b/src/BaseSpacePy/api/BaseSpaceAPI.py index 3da8890..7e0d775 100755 --- a/src/BaseSpacePy/api/BaseSpaceAPI.py +++ b/src/BaseSpacePy/api/BaseSpaceAPI.py @@ -207,6 +207,13 @@ def getAppSession(self, Id=None, queryPars=None): queryParams = {} return self.__singleRequest__(AppSessionResponse.AppSessionResponse, resourcePath, method, queryParams, headerParams) + def getAllAppSessions(self, queryPars=None): + queryParams = self._validateQueryParameters(queryPars) + resourcePath = '/users/current/appsessions' + method = 'GET' + headerParams = {} + return self.__listRequest__(AppSession.AppSession, resourcePath, method, queryParams, headerParams) + def __deserializeAppSessionResponse__(self, response): ''' Converts a AppSession response from the API server to an AppSession object. @@ -673,15 +680,15 @@ def getAppResultsByProject(self, Id, queryPars=None, statuses=None): def getSamplesByProject(self, Id, queryPars=None): ''' Returns a list of samples associated with a project with Id - + :param Id: The id of the project :param queryPars: An (optional) object of type QueryParameters for custom sorting and filtering :returns: a list of Sample instances ''' - queryParams = self._validateQueryParameters(queryPars) + queryParams = self._validateQueryParameters(queryPars) resourcePath = '/projects/{Id}/samples' resourcePath = resourcePath.replace('{format}', 'json') - method = 'GET' + method = 'GET' headerParams = {} resourcePath = resourcePath.replace('{Id}',Id) return self.__listRequest__(Sample.Sample,resourcePath, method, queryParams, headerParams) diff --git a/src/BaseSpacePy/model/Project.py b/src/BaseSpacePy/model/Project.py index 80f5cef..75b6514 100644 --- a/src/BaseSpacePy/model/Project.py +++ b/src/BaseSpacePy/model/Project.py @@ -58,10 +58,11 @@ def getAppResults(self, api, queryPars=None, statuses=None): :param api: An instance of BaseSpaceAPI :param queryPars: An (optional) object of type QueryParameters for custom sorting and filtering :param statuses: An optional list of statuses, eg. 'complete' + :return: list of AppResult objects ''' self.isInit() return api.getAppResultsByProject(self.Id, queryPars=queryPars, statuses=statuses) - + def getSamples(self, api, queryPars=None): ''' Returns a list of Sample objects. diff --git a/src/BaseSpacePy/model/QueryParameters.py b/src/BaseSpacePy/model/QueryParameters.py index ddf833e..9d1f445 100644 --- a/src/BaseSpacePy/model/QueryParameters.py +++ b/src/BaseSpacePy/model/QueryParameters.py @@ -2,6 +2,7 @@ from BaseSpacePy.api.BaseSpaceException import UndefinedParameterException, UnknownParameterException, IllegalParameterException, QueryParameterException legal = {'Statuses': [], + 'Status':[], 'SortBy': ['Id', 'Name', 'DateCreated', 'Path', 'Position'], 'Extensions': [], #'Extensions': ['bam', 'vcf'], From 6ee5c66c403fa85e1b7a79d65dca8a5b318791b2 Mon Sep 17 00:00:00 2001 From: psaffrey Date: Fri, 16 Oct 2015 09:37:45 +0100 Subject: [PATCH 18/73] Created authentication class --- src/BaseSpacePy/api/AuthenticationAPI.py | 138 +++++++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 src/BaseSpacePy/api/AuthenticationAPI.py diff --git a/src/BaseSpacePy/api/AuthenticationAPI.py b/src/BaseSpacePy/api/AuthenticationAPI.py new file mode 100644 index 0000000..a53a5f9 --- /dev/null +++ b/src/BaseSpacePy/api/AuthenticationAPI.py @@ -0,0 +1,138 @@ +import sys +import time +import ConfigParser +import getpass +import os +import requests + +__author__ = 'psaffrey' + +""" +Objects to help with creating config files that contain the right details to be used by the BaseSpaceSDK + +One way uses the OAuth flow for web application authentication: + +https://developer.basespace.illumina.com/docs/content/documentation/authentication/obtaining-access-tokens + +to get an access token and put it in the proper place. + +Also partly available here is obtaining session tokens (cookies), although these are not currently used. +""" + +DEFAULT_SCOPE = "CREATE GLOBAL,BROWSE GLOBAL,CREATE PROJECTS,READ GLOBAL" + + +class AuthenticationAPI(object): + DEFAULT_CONFIG_NAME = "DEFAULT" + + def __init__(self, config_path, api_server): + self.config_path = config_path + self.api_server = api_server + self.config = None + self.parse_config() + + def parse_config(self): + """ + parses the config_path or creates it if it doesn't exist + + :param config_path: path to config file + :return: ConfigParser object + """ + if not os.path.exists(self.config_path): + self.config = ConfigParser.SafeConfigParser() + else: + self.config = ConfigParser.SafeConfigParser() + self.config.read(self.config_path) + + def construct_default_config(self, api_server): + self.config.set(self.DEFAULT_CONFIG_NAME, "apiServer", api_server) + + def write_config(self): + with open(self.config_path, "w") as fh: + self.config.write(fh) + + +###### +# the BaseSpaceAPI doesn't support using the session tokens (cookies) at the moment +# but this is here in case it's useful to somebody :) +class SessionAuthentication(AuthenticationAPI): + SESSION_AUTH_URI = "https://accounts.illumina.com/" + SESSION_TOKEN_NAME = "sessionToken" + COOKIE_NAME = "IComLogin" + + def basespace_session(self, username, password): + s = requests.session() + payload = {"UserName": username, + "Password": password, + "ReturnUrl": "http://developer.basespace.illumina.com/dashboard"} + r = s.post(url=self.SESSION_AUTH_URI, + params={'Service': 'basespace'}, + data=payload, + headers={'Content-Type': "application/x-www-form-urlencoded"}, + allow_redirects=False) + return s, r + + def check_session_details(self): + pass + + def set_session_details(self, config_path): + username = raw_input("username:") + password = getpass.getpass() + s, r = self.basespace_session(username, password) + self.config.set(self.DEFAULT_CONFIG_NAME, self.SESSION_TOKEN_NAME, r.cookies[self.COOKIE_NAME]) + self.write_config() + +class OAuthAuthentication(AuthenticationAPI): + WAIT_TIME = 5.0 + ACCESS_TOKEN_NAME = "accessToken" + + def __init__(self, config_path, api_server, api_version): + super(OAuthAuthentication, self).__init__(config_path, api_server) + self.api_version = api_version + + def set_oauth_details(self, client_id, client_secret, scope=DEFAULT_SCOPE): + OAUTH_URI = "%s%s/oauthv2/deviceauthorization" % (self.api_server, self.api_version) + TOKEN_URI = "%s%s/oauthv2/token" % (self.api_server, self.api_version) + s = requests.session() + # make the initial request + auth_payload = { + "response_type": "device_code", + "client_id": client_id, + "scope": scope, + } + try: + r = s.post(url=OAUTH_URI, + data=auth_payload) + except Exception as e: + print "problem communicate with oauth server: %s" % str(e) + raise + # show the URL to the user + auth_url = r.json()["verification_with_code_uri"] + auth_code = r.json()["device_code"] + print "please authenticate here: %s" % auth_url + # poll the token URL until we get the token + token_payload = { + "client_id": client_id, + "client_secret": client_secret, + "code": auth_code, + "grant_type": "device" + } + access_token = None + while 1: + # put the token into the config file + r = s.post(url=TOKEN_URI, + data=token_payload) + if r.status_code == 400: + sys.stdout.write(".") + sys.stdout.flush() + time.sleep(self.WAIT_TIME) + else: + sys.stdout.write("\n") + access_token = r.json()["access_token"] + break + self.construct_default_config(self.api_server) + if not access_token: + raise Exception("problem obtaining token!") + print "Success!" + self.config.set(self.DEFAULT_CONFIG_NAME, self.ACCESS_TOKEN_NAME, access_token) + self.write_config() \ No newline at end of file From c43a80390cb78f808bf476ef31bbc55ae6809b27 Mon Sep 17 00:00:00 2001 From: psaffrey Date: Fri, 16 Oct 2015 17:19:04 +0100 Subject: [PATCH 19/73] Cleaning up authenticate --- src/BaseSpacePy/api/AuthenticationAPI.py | 15 +++++++++++---- src/BaseSpacePy/api/BaseSpaceAPI.py | 4 ++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/BaseSpacePy/api/AuthenticationAPI.py b/src/BaseSpacePy/api/AuthenticationAPI.py index a53a5f9..fa041d9 100644 --- a/src/BaseSpacePy/api/AuthenticationAPI.py +++ b/src/BaseSpacePy/api/AuthenticationAPI.py @@ -5,6 +5,14 @@ import os import requests +# this tries to clean up the output at the expense of letting the user know they're in an insecure context... +try: + requests.packages.urllib3.disable_warnings() +except: + pass +import logging +logging.getLogger("requests").setLevel(logging.WARNING) + __author__ = 'psaffrey' """ @@ -38,10 +46,9 @@ def parse_config(self): :param config_path: path to config file :return: ConfigParser object """ - if not os.path.exists(self.config_path): - self.config = ConfigParser.SafeConfigParser() - else: - self.config = ConfigParser.SafeConfigParser() + self.config = ConfigParser.SafeConfigParser() + self.config.optionxform = str + if os.path.exists(self.config_path): self.config.read(self.config_path) def construct_default_config(self, api_server): diff --git a/src/BaseSpacePy/api/BaseSpaceAPI.py b/src/BaseSpacePy/api/BaseSpaceAPI.py index 7e0d775..d48b9b2 100755 --- a/src/BaseSpacePy/api/BaseSpaceAPI.py +++ b/src/BaseSpacePy/api/BaseSpaceAPI.py @@ -84,7 +84,7 @@ def _setCredentials(self, clientKey, clientSecret, apiServer, apiVersion, appSes ''' lcl_cred = self._getLocalCredentials(profile) my_path = os.path.dirname(os.path.abspath(__file__)) - authenticate = os.path.abspath(os.path.join(my_path, "..", "..", "..", "bin", "authenticate.py")) + authenticate = "bs authenticate" authenticate_cmd = "%s --config %s" % (authenticate, profile) cred = {} # set profile name @@ -102,7 +102,7 @@ def _setCredentials(self, clientKey, clientSecret, apiServer, apiVersion, appSes try: cred[conf_item] = lcl_cred[conf_item] except KeyError: - raise CredentialsException("%s not found or config %s missing. Try running %s" % (conf_item, profile, authenticate_cmd)) + raise CredentialsException("%s not found or config %s missing. Try running \"%s\"" % (conf_item, profile, authenticate_cmd)) # Optional credentials OPTIONAL = ["clientKey", "clientSecret", "appSessionId"] for conf_item in OPTIONAL: From d69c285490c3ba1217cd8aea7bb71f0e1809b021 Mon Sep 17 00:00:00 2001 From: psaffrey Date: Tue, 20 Oct 2015 09:14:48 +0100 Subject: [PATCH 20/73] Refactoring ugly global variables --- src/BaseSpacePy/api/AppLaunchHelpers.py | 47 ++++++++++++++++--------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/src/BaseSpacePy/api/AppLaunchHelpers.py b/src/BaseSpacePy/api/AppLaunchHelpers.py index 70584a5..7a9735a 100644 --- a/src/BaseSpacePy/api/AppLaunchHelpers.py +++ b/src/BaseSpacePy/api/AppLaunchHelpers.py @@ -21,20 +21,11 @@ from BaseMountInterface import BaseMountInterface from BaseSpaceAPI import BaseSpaceAPI -api = BaseSpaceAPI() -API_VERSION = api.version - -SKIP_PROPERTIES = ["app-session-name"] - # if these strings are in the property names, we should not try to capture default values for them. +# these are "global" but are needed by more than one object, so it's the cleanest way for now BS_ENTITIES = ["sample", "project", "appresult", "file"] BS_ENTITY_LIST_NAMES = ["Samples", "Projects", "AppResults", "Files"] -LAUNCH_HEADER = { - "StatusSummary": "AutoLaunch", - "AutoStart": True, -} - class AppSessionMetaData(object): """ @@ -47,6 +38,9 @@ class AppSessionMetaData(object): __metaclass__ = abc.ABCMeta + SKIP_PROPERTIES = ["app-session-name"] + + def __init__(self, appsession_metadata): """ @@ -66,7 +60,7 @@ def get_refined_appsession_properties(self): if property_name.count(".") != 1: continue property_basename = property_name.split(".")[-1] - if property_basename in SKIP_PROPERTIES: + if property_basename in self.SKIP_PROPERTIES: continue if property_basename in BS_ENTITY_LIST_NAMES: continue @@ -161,6 +155,11 @@ class LaunchSpecification(object): Class to help work with a BaseSpace app launch specification, which includes the properties and any defaults """ + LAUNCH_HEADER = { + "StatusSummary": "AutoLaunch", + "AutoStart": True, + } + def __init__(self, properties, defaults): self.properties = properties self.property_lookup = dict((self.clean_name(property_["Name"]), property_) for property_ in self.properties) @@ -177,6 +176,20 @@ def clean_name(parameter_name): assert prefix == "Input" return cleaned_name + def process_parameter(self, param, varname): + # if option is prefixed with an @, it's a file (or process substitution with <() ) + # so we should read inputs from there + if param.startswith("@"): + assert self.is_list_property(varname), "cannot specify non-list parameter with file" + with open(param[1:]) as fh: + processed_param = [line.strip() for line in fh] + else: + if self.is_list_property(varname): + processed_param = param.split(",") + else: + processed_param = param + return processed_param + def resolve_list_variables(self, var_dict): """ ensure each variable has the right list type @@ -240,7 +253,7 @@ def make_sample_attribute_entry(sampleid, wrapped_sampleid, sample_attributes): this_sample_attributes.append(attribute_entry) return this_sample_attributes - def populate_properties(self, var_dict, sample_attributes={}): + def populate_properties(self, var_dict, api_version, sample_attributes={}): """ Uses the base properties of the object and an instantiation of those properties (var_dict) build a dictionary that represents the launch payload @@ -270,14 +283,14 @@ def populate_properties(self, var_dict, sample_attributes={}): if "[]" in property_type: processed_value = [] for one_val in property_value: - wrapped_value = "%s/%ss/%s" % (API_VERSION, bald_type, one_val) + wrapped_value = "%s/%ss/%s" % (api_version, bald_type, one_val) processed_value.append(wrapped_value) if sample_attributes and bald_type == "sample": one_sample_attributes = self.make_sample_attribute_entry(one_val, wrapped_value, sample_attributes) all_sample_attributes["items"].append(one_sample_attributes) else: - processed_value = "%s/%ss/%s" % (API_VERSION, bald_type, property_value) + processed_value = "%s/%ss/%s" % (api_version, bald_type, property_value) if sample_attributes and bald_type == "sample": one_sample_attributes = self.make_sample_attribute_entry(property_value, processed_value, sample_attributes) @@ -343,7 +356,7 @@ def count_list_properties(self): """ return [self.is_list_property(property_name) for property_name in self.get_minimum_requirements()].count(True) - def make_launch_json(self, user_supplied_vars, launch_name, sample_attributes={}, agent_id=""): + def make_launch_json(self, user_supplied_vars, launch_name, api_version, sample_attributes={}, agent_id=""): """ build the launch payload (a json blob as a string) based on the supplied mapping from property name to value @@ -355,7 +368,7 @@ def make_launch_json(self, user_supplied_vars, launch_name, sample_attributes={} :param agent_id: an AgentId to be attached to the launch, if specifed """ # build basic headers - launch_dict = copy.copy(LAUNCH_HEADER) + launch_dict = copy.copy(self.LAUNCH_HEADER) launch_dict["Name"] = launch_name if agent_id: launch_dict["AgentId"] = agent_id @@ -370,7 +383,7 @@ def make_launch_json(self, user_supplied_vars, launch_name, sample_attributes={} all_vars = copy.copy(self.defaults) all_vars.update(user_supplied_vars) self.resolve_list_variables(all_vars) - properties_dict = self.populate_properties(all_vars, sample_attributes) + properties_dict = self.populate_properties(all_vars, api_version, sample_attributes) launch_dict["Properties"] = properties_dict return json.dumps(launch_dict) From ae98895f0a28100c67f9fe9f2cf979cfe1e10306 Mon Sep 17 00:00:00 2001 From: Mauricio Varea Date: Thu, 22 Oct 2015 15:13:47 +0100 Subject: [PATCH 21/73] Improving exception messages --- src/BaseSpacePy/api/BaseSpaceException.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/BaseSpacePy/api/BaseSpaceException.py b/src/BaseSpacePy/api/BaseSpaceException.py index c4bba4a..579cffc 100644 --- a/src/BaseSpacePy/api/BaseSpaceException.py +++ b/src/BaseSpacePy/api/BaseSpaceException.py @@ -49,43 +49,43 @@ def __str__(self): class UploadPartSizeException(Exception): def __init__(self, value): - self.parameter = 'Upload part size invalid: ' + value + self.parameter = 'Upload part size is invalid: ' + value def __str__(self): return repr(self.parameter) class CredentialsException(Exception): def __init__(self, value): - self.parameter = 'Error with BaseSpace credentials: ' + value + self.parameter = 'Invalid BaseSpace credentials: ' + value def __str__(self): return repr(self.parameter) class QueryParameterException(Exception): def __init__(self, value): - self.parameter = 'Error with query parameter: ' + value + self.parameter = 'Invalid query parameter: ' + value def __str__(self): return repr(self.parameter) class AppSessionException(Exception): def __init__(self, value): - self.parameter = 'Error with AppSession: ' + value + self.parameter = 'AppSession error: ' + value def __str__(self): return repr(self.parameter) class ModelNotSupportedException(Exception): def __init__(self, value): - self.parameter = 'Model not supported: ' + value + self.parameter = 'Unsupported model: ' + value def __str__(self): return repr(self.parameter) class OAuthException(Exception): def __init__(self, value): - self.parameter = 'Error with OAuth: ' + value + self.parameter = 'Could not authenticate with OAuth: ' + value def __str__(self): return repr(self.parameter) class RestMethodException(Exception): def __init__(self, value): - self.parameter = 'Error with REST API method: ' + value + self.parameter = 'Problem with REST API method: ' + value def __str__(self): return repr(self.parameter) From ee69471e252acd208e25663694025094fae7aded Mon Sep 17 00:00:00 2001 From: psaffrey Date: Fri, 23 Oct 2015 13:44:26 +0100 Subject: [PATCH 22/73] Handle "do not accept" properly --- src/BaseSpacePy/api/AuthenticationAPI.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/BaseSpacePy/api/AuthenticationAPI.py b/src/BaseSpacePy/api/AuthenticationAPI.py index fa041d9..9127b45 100644 --- a/src/BaseSpacePy/api/AuthenticationAPI.py +++ b/src/BaseSpacePy/api/AuthenticationAPI.py @@ -130,6 +130,9 @@ def set_oauth_details(self, client_id, client_secret, scope=DEFAULT_SCOPE): r = s.post(url=TOKEN_URI, data=token_payload) if r.status_code == 400: + if r.content["error"] == "access_denied": + sys.stdout.write("\n") + break sys.stdout.write(".") sys.stdout.flush() time.sleep(self.WAIT_TIME) From e1ee1bbf32f1a924d4ca291e128fb843373dd363 Mon Sep 17 00:00:00 2001 From: psaffrey Date: Fri, 23 Oct 2015 17:08:28 +0100 Subject: [PATCH 23/73] bug with error message resolution for do-not-accept --- src/BaseSpacePy/api/AuthenticationAPI.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BaseSpacePy/api/AuthenticationAPI.py b/src/BaseSpacePy/api/AuthenticationAPI.py index 9127b45..942d858 100644 --- a/src/BaseSpacePy/api/AuthenticationAPI.py +++ b/src/BaseSpacePy/api/AuthenticationAPI.py @@ -130,7 +130,7 @@ def set_oauth_details(self, client_id, client_secret, scope=DEFAULT_SCOPE): r = s.post(url=TOKEN_URI, data=token_payload) if r.status_code == 400: - if r.content["error"] == "access_denied": + if r.json()["error"] == "access_denied": sys.stdout.write("\n") break sys.stdout.write(".") From fa46d22a1aa255e084ec147d303d1c7c6fc372a8 Mon Sep 17 00:00:00 2001 From: psaffrey Date: Fri, 6 Nov 2015 13:36:40 +0000 Subject: [PATCH 24/73] tweaked exception message --- src/BaseSpacePy/api/BaseMountInterface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BaseSpacePy/api/BaseMountInterface.py b/src/BaseSpacePy/api/BaseMountInterface.py index 8252ae8..35f3fab 100644 --- a/src/BaseSpacePy/api/BaseMountInterface.py +++ b/src/BaseSpacePy/api/BaseMountInterface.py @@ -28,7 +28,7 @@ def __init__(self, path): self.access_token = None self.name = os.path.basename(path) if not self.__validate_basemount__(): - raise BaseMountInterfaceException("Path: %s does not seem to be a BaseMount path" % self.path) + raise BaseMountInterfaceException("Path: %s does not seem to be a BaseMount entity path" % self.path) self.__resolve_details__() def __validate_basemount__(self): From 21e30f325f5a2b0d10fd98cfe1f20e2cad6f01a6 Mon Sep 17 00:00:00 2001 From: Mauricio Varea Date: Mon, 9 Nov 2015 11:53:25 +0000 Subject: [PATCH 25/73] fixed package naming and dependencies --- src/setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/setup.py b/src/setup.py index dde058e..bf27bb3 100755 --- a/src/setup.py +++ b/src/setup.py @@ -21,23 +21,23 @@ from distutils.core import setup -setup(name='BaseSpacePy', +setup(name='basespace-python-sdk', description='A Python SDK for connecting to Illumina BaseSpace data', author='Illumina', - version='0.3', + version='0.4', long_description=""" BaseSpacePy is a Python based SDK to be used in the development of Apps and scripts for working with Illumina's BaseSpace cloud-computing solution for next-gen sequencing data analysis. The primary purpose of the SDK is to provide an easy-to-use Python environment enabling developers to authenticate a user, retrieve data, and upload data/results from their own analysis to BaseSpace.""", - author_email='', + author_email='techsupport@illumina.com', packages=['BaseSpacePy.api','BaseSpacePy.model','BaseSpacePy'], package_dir={'BaseSpacePy' : os.path.join(os.path.dirname(__file__),'BaseSpacePy')}, # this line moves closer to a Python configuration that does not issue the SSLContext warning # it fails because of missing headers when building a dependency #install_requires=['pycurl','python-dateutil','pyOpenSSL>=0.13','requests','requests[security]'], install_requires=['pycurl','python-dateutil','requests'], - setup_requires=['stdeb'], + #setup_requires=['rpm-build','stdeb'], zip_safe=False, ) From e5637c88647ed18b6ac00fc716305270245fcc5e Mon Sep 17 00:00:00 2001 From: psaffrey Date: Mon, 9 Nov 2015 11:59:05 +0000 Subject: [PATCH 26/73] removed spurious print statement in authentication --- src/BaseSpacePy/api/AuthenticationAPI.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/BaseSpacePy/api/AuthenticationAPI.py b/src/BaseSpacePy/api/AuthenticationAPI.py index 942d858..255e923 100644 --- a/src/BaseSpacePy/api/AuthenticationAPI.py +++ b/src/BaseSpacePy/api/AuthenticationAPI.py @@ -111,7 +111,6 @@ def set_oauth_details(self, client_id, client_secret, scope=DEFAULT_SCOPE): r = s.post(url=OAUTH_URI, data=auth_payload) except Exception as e: - print "problem communicate with oauth server: %s" % str(e) raise # show the URL to the user auth_url = r.json()["verification_with_code_uri"] From 6f98de1b6e0cb3981f86d27c945b2727b5f8eb1a Mon Sep 17 00:00:00 2001 From: psaffrey Date: Mon, 9 Nov 2015 12:17:07 +0000 Subject: [PATCH 27/73] better exception handling for authentication failures --- src/BaseSpacePy/api/AuthenticationAPI.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/BaseSpacePy/api/AuthenticationAPI.py b/src/BaseSpacePy/api/AuthenticationAPI.py index 255e923..e72412c 100644 --- a/src/BaseSpacePy/api/AuthenticationAPI.py +++ b/src/BaseSpacePy/api/AuthenticationAPI.py @@ -29,6 +29,8 @@ DEFAULT_SCOPE = "CREATE GLOBAL,BROWSE GLOBAL,CREATE PROJECTS,READ GLOBAL" +class AuthenticationException(Exception): + pass class AuthenticationAPI(object): DEFAULT_CONFIG_NAME = "DEFAULT" @@ -111,10 +113,14 @@ def set_oauth_details(self, client_id, client_secret, scope=DEFAULT_SCOPE): r = s.post(url=OAUTH_URI, data=auth_payload) except Exception as e: - raise + raise AuthenticationException("Failed to communicate with server: %s" % str(e)) # show the URL to the user - auth_url = r.json()["verification_with_code_uri"] - auth_code = r.json()["device_code"] + try: + payload = r.json() + except ValueError: + raise AuthenticationException("bad payload from server - perhaps you should use https instead of http?") + auth_url = payload["verification_with_code_uri"] + auth_code = payload["device_code"] print "please authenticate here: %s" % auth_url # poll the token URL until we get the token token_payload = { From ed44e33c2c9b44e2b83c23ac5e70723337a4cb7e Mon Sep 17 00:00:00 2001 From: Mauricio Varea Date: Mon, 9 Nov 2015 18:21:31 +0000 Subject: [PATCH 28/73] added forgotten file --- src/setup.cfg | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 src/setup.cfg diff --git a/src/setup.cfg b/src/setup.cfg new file mode 100644 index 0000000..a8ab3a7 --- /dev/null +++ b/src/setup.cfg @@ -0,0 +1,8 @@ +[bdist_rpm] + +build_requires = rpm-build +requires = python >= 2.6 + python-pycurl + python-dateutil + python-requests +no-autoreq = yes From 2d7f85c3116c813e2f3f9f7cde59eb6c3047f58a Mon Sep 17 00:00:00 2001 From: psaffrey Date: Tue, 10 Nov 2015 15:49:21 +0000 Subject: [PATCH 29/73] removed sp --- src/BaseSpacePy/api/BaseSpaceAPI.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/BaseSpacePy/api/BaseSpaceAPI.py b/src/BaseSpacePy/api/BaseSpaceAPI.py index d48b9b2..59c406f 100755 --- a/src/BaseSpacePy/api/BaseSpaceAPI.py +++ b/src/BaseSpacePy/api/BaseSpaceAPI.py @@ -85,7 +85,8 @@ def _setCredentials(self, clientKey, clientSecret, apiServer, apiVersion, appSes lcl_cred = self._getLocalCredentials(profile) my_path = os.path.dirname(os.path.abspath(__file__)) authenticate = "bs authenticate" - authenticate_cmd = "%s --config %s" % (authenticate, profile) + if profile != "default": + authenticate_cmd = "%s --config %s" % (authenticate, profile) cred = {} # set profile name if 'name' in lcl_cred: From 51f86db46c4ef3f1b739c63a9c6eec911ee8518c Mon Sep 17 00:00:00 2001 From: psaffrey Date: Wed, 11 Nov 2015 14:28:33 +0000 Subject: [PATCH 30/73] Added getUserProjectByName method --- bin/authenticate.py | 1 + src/BaseSpacePy/api/AppLaunchHelpers.py | 3 +-- src/BaseSpacePy/api/BaseSpaceAPI.py | 16 ++++++++++++++-- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/bin/authenticate.py b/bin/authenticate.py index b8a0501..0040fa8 100644 --- a/bin/authenticate.py +++ b/bin/authenticate.py @@ -58,6 +58,7 @@ def set_session_details(config_path): password = getpass.getpass() s, r = basespace_session(username, password) config = parse_config(config_path) + import pdb; pdb.set_trace() config.set(DEFAULT_CONFIG_NAME, SESSION_TOKEN_NAME, ) with open(config_path, "w") as fh: config.write(fh) diff --git a/src/BaseSpacePy/api/AppLaunchHelpers.py b/src/BaseSpacePy/api/AppLaunchHelpers.py index 7a9735a..69ed8d3 100644 --- a/src/BaseSpacePy/api/AppLaunchHelpers.py +++ b/src/BaseSpacePy/api/AppLaunchHelpers.py @@ -19,7 +19,6 @@ import os from BaseMountInterface import BaseMountInterface -from BaseSpaceAPI import BaseSpaceAPI # if these strings are in the property names, we should not try to capture default values for them. # these are "global" but are needed by more than one object, so it's the cleanest way for now @@ -501,6 +500,7 @@ def to_basespace_id(self, param_name, varval): """ if varval.startswith("/") and not os.path.exists(varval): raise LaunchSpecificationException("Parameter looks like a path, but does not exist: %s" % varval) + spec_type = self._launch_spec.get_property_bald_type(param_name) if os.path.exists(varval): bmi = BaseMountInterface(varval) # make sure we have a BaseMount access token to compare - old versions won't have one @@ -509,7 +509,6 @@ def to_basespace_id(self, param_name, varval): if bmi.access_token and self._access_token and bmi.access_token != self._access_token: raise LaunchSpecificationException( "Access tokens between launch configuration and referenced BaseMount path do not match: %s" % varval) - spec_type = self._launch_spec.get_property_bald_type(param_name) basemount_type = bmi.type if spec_type != basemount_type: raise LaunchSpecificationException( diff --git a/src/BaseSpacePy/api/BaseSpaceAPI.py b/src/BaseSpacePy/api/BaseSpaceAPI.py index 59c406f..f149e40 100755 --- a/src/BaseSpacePy/api/BaseSpaceAPI.py +++ b/src/BaseSpacePy/api/BaseSpaceAPI.py @@ -84,7 +84,7 @@ def _setCredentials(self, clientKey, clientSecret, apiServer, apiVersion, appSes ''' lcl_cred = self._getLocalCredentials(profile) my_path = os.path.dirname(os.path.abspath(__file__)) - authenticate = "bs authenticate" + authenticate_cmd = "bs authenticate" if profile != "default": authenticate_cmd = "%s --config %s" % (authenticate, profile) cred = {} @@ -583,7 +583,19 @@ def getProjectByUser(self, queryPars=None): method = 'GET' headerParams = {} return self.__listRequest__(Project.Project,resourcePath, method, queryParams, headerParams) - + + def getUserProjectByName(self, projectName): + ''' + + :return: project matching the provided name + ''' + projects = self.getProjectByUser(qp({"Name":projectName})) + if len(projects) == 0: + raise ValueError("No such project: %s" % projectName) + if len(projects) > 1: + raise ValueError("More than one matching projects: %s" % projectName) + return projects[0] + def getAccessibleRunsByUser(self, queryPars=None): ''' Returns a list of accessible runs for the current User From 2459f38e04d5ede34fa589a2d9e6913041f8601b Mon Sep 17 00:00:00 2001 From: psaffrey Date: Wed, 11 Nov 2015 14:29:00 +0000 Subject: [PATCH 31/73] Removed spurious authenticate command --- bin/authenticate.py | 161 -------------------------------------------- 1 file changed, 161 deletions(-) delete mode 100644 bin/authenticate.py diff --git a/bin/authenticate.py b/bin/authenticate.py deleted file mode 100644 index 0040fa8..0000000 --- a/bin/authenticate.py +++ /dev/null @@ -1,161 +0,0 @@ -import json -import time -import sys - -__author__ = 'psaffrey' - -""" -Script that sets up the files in .basespace to contain access tokens. - -One way uses the OAuth flow for web application authentication: - -https://developer.basespace.illumina.com/docs/content/documentation/authentication/obtaining-access-tokens - -to get an access token and put it in the proper place. - -Also supported is obtaining session tokens (cookies), although these are not currently used. - -""" - -import getpass -import requests -import os -import ConfigParser - -SESSION_AUTH_URI = "https://accounts.illumina.com/" -DEFAULT_CONFIG_NAME = "DEFAULT" -SESSION_TOKEN_NAME = "sessionToken" -ACCESS_TOKEN_NAME = "accessToken" -DEFAULT_API_SERVER = "https://api.basespace.illumina.com/" -API_VERSION = "v1pre3" -WAIT_TIME = 5.0 - -# these are the details for the BaseSpaceCLI app -# shared with BaseMount -CLIENT_ID = "ca2e493333b044a18d65385afaf8eb5d" -CLIENT_SECRET = "282b0f7d4e5d48dfabc7cdfe5b3156a6" -SCOPE="CREATE GLOBAL,BROWSE GLOBAL,CREATE PROJECTS,READ GLOBAL" - -def basespace_session(username, password): - s = requests.session() - payload = {"UserName": username, - "Password": password, - "ReturnUrl": "http://developer.basespace.illumina.com/dashboard"} - r = s.post(url=SESSION_AUTH_URI, - params={'Service': 'basespace'}, - data=payload, - headers={'Content-Type': "application/x-www-form-urlencoded"}, - allow_redirects=False) - return s, r - - -def check_session_details(): - pass - - -def set_session_details(config_path): - username = raw_input("username:") - password = getpass.getpass() - s, r = basespace_session(username, password) - config = parse_config(config_path) - import pdb; pdb.set_trace() - config.set(DEFAULT_CONFIG_NAME, SESSION_TOKEN_NAME, ) - with open(config_path, "w") as fh: - config.write(fh) - - -def parse_config(config_path): - """ - parses the config_path or creates it if it doesn't exist - - :param config_path: path to config file - :return: ConfigParser object - """ - if not os.path.exists(config_path): - config = ConfigParser.SafeConfigParser() - else: - config = ConfigParser.SafeConfigParser() - config.read(config_path) - return config - -def construct_default_config(config, api_server): - config.set(DEFAULT_CONFIG_NAME, "apiServer", api_server) - -def set_oauth_details(config_path, api_server): - OAUTH_URI = "%s%s/oauthv2/deviceauthorization" % (api_server, API_VERSION) - TOKEN_URI = "%s%s/oauthv2/token" % (api_server, API_VERSION) - s = requests.session() - # make the initial request - auth_payload = { - "response_type" : "device_code", - "client_id" : CLIENT_ID, - "scope" : SCOPE, - } - try: - r = s.post(url=OAUTH_URI, - data=auth_payload) - except Exception as e: - print "problem communicate with oauth server: %s" % str(e) - raise - # show the URL to the user - auth_url = r.json()["verification_with_code_uri"] - auth_code = r.json()["device_code"] - print "please authenticate here: %s" % auth_url - # poll the token URL until we get the token - token_payload = { - "client_id" : CLIENT_ID, - "client_secret" : CLIENT_SECRET, - "code" : auth_code, - "grant_type" : "device" - } - access_token = None - while 1: - # put the token into the config file - r = s.post(url=TOKEN_URI, - data=token_payload) - if r.status_code == 400: - sys.stdout.write(".") - sys.stdout.flush() - time.sleep(WAIT_TIME) - else: - sys.stdout.write("\n") - access_token = r.json()["access_token"] - break - config = parse_config(config_path) - construct_default_config(config, api_server) - if not access_token: - raise Exception("problem obtaining token!") - print "Success!" - config.set(DEFAULT_CONFIG_NAME, ACCESS_TOKEN_NAME, access_token) - with open(config_path, "w") as fh: - config.write(fh) - - -if __name__ == "__main__": - from argparse import ArgumentParser - - parser = ArgumentParser(description="Derive BaseSpace authentication details") - - parser.add_argument('-c', '--configname', type=str, dest="configname", default="default", help='name of config') - parser.add_argument('-s', '--sessiontoken', default=False, action="store_true", - help='do session auth, instead of regular auth') - parser.add_argument('-a', '--api-server', default=DEFAULT_API_SERVER, help="choose backend api server") - - args = parser.parse_args() - - # cross platform way to get home directory - home = os.path.expanduser("~") - config_path = os.path.join(home, ".basespace", "%s.cfg" % args.configname) - - - if args.sessiontoken: - set_session_details(config_path) - else: - try: - if os.path.exists(config_path): - print "config path already exists; not overwriting (%s)" % config_path - sys.exit(1) - set_oauth_details(config_path, args.api_server) - except Exception as e: - print "authentication failed!" - raise From a2ce2dbee63256488c73a8ed7028ea1e46e77e9c Mon Sep 17 00:00:00 2001 From: psaffrey Date: Wed, 11 Nov 2015 14:31:00 +0000 Subject: [PATCH 32/73] bug in authenticate command suggestion --- src/BaseSpacePy/api/BaseSpaceAPI.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BaseSpacePy/api/BaseSpaceAPI.py b/src/BaseSpacePy/api/BaseSpaceAPI.py index f149e40..294e2f2 100755 --- a/src/BaseSpacePy/api/BaseSpaceAPI.py +++ b/src/BaseSpacePy/api/BaseSpaceAPI.py @@ -86,7 +86,7 @@ def _setCredentials(self, clientKey, clientSecret, apiServer, apiVersion, appSes my_path = os.path.dirname(os.path.abspath(__file__)) authenticate_cmd = "bs authenticate" if profile != "default": - authenticate_cmd = "%s --config %s" % (authenticate, profile) + authenticate_cmd = "%s --config %s" % (authenticate_cmd, profile) cred = {} # set profile name if 'name' in lcl_cred: From 8d70c51aaaea986ad1bd04702a6336f74bef6e0b Mon Sep 17 00:00:00 2001 From: Lilian Janin Date: Wed, 11 Nov 2015 15:32:43 +0000 Subject: [PATCH 33/73] Fix for ubuntu build --- src/setup.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/src/setup.cfg b/src/setup.cfg index a8ab3a7..70be50d 100644 --- a/src/setup.cfg +++ b/src/setup.cfg @@ -1,6 +1,5 @@ [bdist_rpm] -build_requires = rpm-build requires = python >= 2.6 python-pycurl python-dateutil From dfab7ea8f11ac8f8018bf4d54dc6487aedd287ab Mon Sep 17 00:00:00 2001 From: psaffrey Date: Thu, 12 Nov 2015 10:47:06 +0000 Subject: [PATCH 34/73] Rejigging applaunch helper interface --- src/BaseSpacePy/api/AppLaunchHelpers.py | 15 +++++++++++++-- src/BaseSpacePy/api/BaseSpaceAPI.py | 4 ++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/BaseSpacePy/api/AppLaunchHelpers.py b/src/BaseSpacePy/api/AppLaunchHelpers.py index 69ed8d3..cd65f71 100644 --- a/src/BaseSpacePy/api/AppLaunchHelpers.py +++ b/src/BaseSpacePy/api/AppLaunchHelpers.py @@ -19,6 +19,7 @@ import os from BaseMountInterface import BaseMountInterface +from BaseSpaceException import ServerResponseException # if these strings are in the property names, we should not try to capture default values for them. # these are "global" but are needed by more than one object, so it's the cleanest way for now @@ -424,7 +425,7 @@ class LaunchPayload(object): LAUNCH_NAME = "LaunchName" - def __init__(self, launch_spec, args, configoptions, access_token): + def __init__(self, launch_spec, args, configoptions, api, disable_consistency_checking=True): """ :param launch_spec (LaunchSpecification) :param args (list) list or arguments to the app launch. These could be BaseSpace IDs or BaseMount paths @@ -435,7 +436,8 @@ def __init__(self, launch_spec, args, configoptions, access_token): self._launch_spec = launch_spec self._args = args self._configoptions = configoptions - self._access_token = access_token + self._api = api + self._access_token = None if disable_consistency_checking else api.apiClient.apiKey varnames = self._launch_spec.get_minimum_requirements() if len(varnames) != len(self._args): raise LaunchSpecificationException("Number of arguments does not match specification") @@ -458,6 +460,15 @@ def _find_all_entity_names(self, entity_type): if os.path.exists(entry): bmi = BaseMountInterface(entry) entity_names.append(bmi.name) + # if this is not a BaseMount path, try to resolve an entity name using the API + # note we're relying on the regular naming of the API to provide the right method name + try: + method_name = "get%sById" % entity_type.title() + method = getattr(self._api, method_name) + entity_names.append(method(entry).Name) + except (AttributeError, ServerResponseException): + pass + return entity_names def derive_launch_name(self, app_name): diff --git a/src/BaseSpacePy/api/BaseSpaceAPI.py b/src/BaseSpacePy/api/BaseSpaceAPI.py index 294e2f2..c7b3383 100755 --- a/src/BaseSpacePy/api/BaseSpaceAPI.py +++ b/src/BaseSpacePy/api/BaseSpaceAPI.py @@ -591,9 +591,9 @@ def getUserProjectByName(self, projectName): ''' projects = self.getProjectByUser(qp({"Name":projectName})) if len(projects) == 0: - raise ValueError("No such project: %s" % projectName) + raise ServerResponseException("No such project: %s" % projectName) if len(projects) > 1: - raise ValueError("More than one matching projects: %s" % projectName) + raise ServerResponseException("More than one matching projects: %s" % projectName) return projects[0] def getAccessibleRunsByUser(self, queryPars=None): From b06f21aae671078bc1da4c7ae1db2aa3e01aada3 Mon Sep 17 00:00:00 2001 From: psaffrey Date: Thu, 12 Nov 2015 16:39:17 +0000 Subject: [PATCH 35/73] Additional method to pull down applications --- src/BaseSpacePy/api/BaseSpaceAPI.py | 9 +++++++++ src/BaseSpacePy/model/Application.py | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/BaseSpacePy/api/BaseSpaceAPI.py b/src/BaseSpacePy/api/BaseSpaceAPI.py index c7b3383..e9d29e7 100755 --- a/src/BaseSpacePy/api/BaseSpaceAPI.py +++ b/src/BaseSpacePy/api/BaseSpaceAPI.py @@ -1485,3 +1485,12 @@ def getResourceProperties(self, resourceType, resourceId): headerParams = {} return self.__singleRequest__(PropertiesResponse.PropertiesResponse, resourcePath, method, queryParams, headerParams) + + def getApplications(self, queryPars=None): + resourcePath = '/applications' + method = 'GET' + headerParams = {} + if not queryPars: + queryPars = qp({"Limit": 1000}) + queryParams = self._validateQueryParameters(queryPars) + return self.__listRequest__(Application.Application, resourcePath, method, queryParams, headerParams) \ No newline at end of file diff --git a/src/BaseSpacePy/model/Application.py b/src/BaseSpacePy/model/Application.py index d2b4f02..12a4ac9 100644 --- a/src/BaseSpacePy/model/Application.py +++ b/src/BaseSpacePy/model/Application.py @@ -9,5 +9,6 @@ def __init__(self): 'HrefLogo': 'str', 'HomepageUri': 'str', 'ShortDescription': 'str', - 'DateCreated': 'datetime' + 'DateCreated': 'datetime', + 'VersionNumber': 'str' } From 7cd1104079f8dc60d6e84dd37bdca5aea3b9e51c Mon Sep 17 00:00:00 2001 From: psaffrey Date: Fri, 13 Nov 2015 17:17:49 +0000 Subject: [PATCH 36/73] Added individual app method --- src/BaseSpacePy/api/BaseSpaceAPI.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/BaseSpacePy/api/BaseSpaceAPI.py b/src/BaseSpacePy/api/BaseSpaceAPI.py index e9d29e7..82ceb10 100755 --- a/src/BaseSpacePy/api/BaseSpaceAPI.py +++ b/src/BaseSpacePy/api/BaseSpaceAPI.py @@ -1487,10 +1487,28 @@ def getResourceProperties(self, resourceType, resourceId): resourcePath, method, queryParams, headerParams) def getApplications(self, queryPars=None): + ''' + Get details about all apps. + Note that each app will only have a single entry, even if it has many versions + :param queryPars: query parameters. Will default to a limit of 1000, so all are gained + :return: list of model.Application.Application objects + ''' resourcePath = '/applications' method = 'GET' headerParams = {} if not queryPars: queryPars = qp({"Limit": 1000}) queryParams = self._validateQueryParameters(queryPars) - return self.__listRequest__(Application.Application, resourcePath, method, queryParams, headerParams) \ No newline at end of file + return self.__listRequest__(Application.Application, resourcePath, method, queryParams, headerParams) + + def getApplicationById(self, Id): + ''' + Get a single app by ID + :return: App object + :raises: ServerResponseException if there is no such app + ''' + resourcePath = '/applications/%s' % Id + method = 'GET' + headerParams = {} + queryParams = {} + return self.__singleRequest__(Application.Application, resourcePath, method, queryParams, headerParams) \ No newline at end of file From fb541914ce934c265c5003f89874bd6edd986cad Mon Sep 17 00:00:00 2001 From: psaffrey Date: Fri, 13 Nov 2015 17:18:06 +0000 Subject: [PATCH 37/73] Add app version methods --- src/BaseSpacePy/api/AppLaunchHelpers.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/BaseSpacePy/api/AppLaunchHelpers.py b/src/BaseSpacePy/api/AppLaunchHelpers.py index cd65f71..8c3dfd6 100644 --- a/src/BaseSpacePy/api/AppLaunchHelpers.py +++ b/src/BaseSpacePy/api/AppLaunchHelpers.py @@ -126,6 +126,9 @@ def get_app_name(self): def get_app_id(self): return self.asm.Application.Id + def get_app_version(self): + return self.asm.Application.VersionNumber + @staticmethod def unpack_bs_property(bs_property, attribute): return getattr(bs_property, attribute) @@ -141,6 +144,9 @@ def get_app_name(self): def get_app_id(self): return self.asm["Response"]["Application"]["Id"] + def get_app_version(self): + return self.asm["Response"]["Application"]["VersionNumber"] + @staticmethod def unpack_bs_property(bs_property, attribute): return bs_property[attribute] From b112dbb0e6a64d8ab905fc0344af5e516a7d5018 Mon Sep 17 00:00:00 2001 From: psaffrey Date: Tue, 1 Dec 2015 16:13:02 +0000 Subject: [PATCH 38/73] added to the allowed query parameters --- src/BaseSpacePy/model/QueryParameters.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/BaseSpacePy/model/QueryParameters.py b/src/BaseSpacePy/model/QueryParameters.py index 9d1f445..f5570e2 100644 --- a/src/BaseSpacePy/model/QueryParameters.py +++ b/src/BaseSpacePy/model/QueryParameters.py @@ -12,7 +12,10 @@ 'Name': [], 'StartPos':[], 'EndPos':[], - 'Format':[] + 'Format':[], + 'include':[], + 'propertyFilters':[], + 'userCreatedBy':[] #'Format': ['txt', 'json', 'vcf'], } From b76666415479ebda9bc39fb6b79047ff822444e7 Mon Sep 17 00:00:00 2001 From: psaffrey Date: Wed, 2 Dec 2015 15:16:33 +0000 Subject: [PATCH 39/73] strip quotes from parameters in launch helper --- src/BaseSpacePy/api/AppLaunchHelpers.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/BaseSpacePy/api/AppLaunchHelpers.py b/src/BaseSpacePy/api/AppLaunchHelpers.py index 8c3dfd6..50bf74f 100644 --- a/src/BaseSpacePy/api/AppLaunchHelpers.py +++ b/src/BaseSpacePy/api/AppLaunchHelpers.py @@ -504,7 +504,15 @@ def is_valid_basespace_id(self, varname, basespace_id): To validate other kinds of ID, we should (TODO!) resolve the type based on the varname and use the SDK to look it up. """ - return True + vartype = self._launch_spec.get_property_bald_type(varname) + if vartype == "Sample": + self._api.getSampleById(basespace_id) + elif vartype == "Project": + self._api.getProjectById(basespace_id) + elif vartype == "AppResult": + self._api.getAppResultById(basespace_id) + else: + return True def to_basespace_id(self, param_name, varval): """ @@ -532,8 +540,13 @@ def to_basespace_id(self, param_name, varval): "wrong type of BaseMount path selected: %s needs to be of type %s" % (varval, spec_type)) bid = bmi.id else: - bid = varval - assert self.is_valid_basespace_id(param_name, bid) + # strip off quotes, which will be what comes in from bs list samples -f csv + bid = varval.strip('"') + # skip this step for now - it could be really expensive for big launches + # try: + # self.is_valid_basespace_id(param_name, bid) + # except ServerResponseException as e: + # raise LaunchSpecificationException("invalid BaseSpace ID '%s' for var: %s (%s)" % (varval, param_name, str(e))) return bid def get_args(self): From 6ac514b03ebf05d1339cb0c4cc7f532540fcba51 Mon Sep 17 00:00:00 2001 From: psaffrey Date: Wed, 2 Dec 2015 15:40:52 +0000 Subject: [PATCH 40/73] Further stripping of quoted arguments --- src/BaseSpacePy/api/AppLaunchHelpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/BaseSpacePy/api/AppLaunchHelpers.py b/src/BaseSpacePy/api/AppLaunchHelpers.py index 50bf74f..b5c209f 100644 --- a/src/BaseSpacePy/api/AppLaunchHelpers.py +++ b/src/BaseSpacePy/api/AppLaunchHelpers.py @@ -468,6 +468,7 @@ def _find_all_entity_names(self, entity_type): entity_names.append(bmi.name) # if this is not a BaseMount path, try to resolve an entity name using the API # note we're relying on the regular naming of the API to provide the right method name + entry = entry.strip('"') try: method_name = "get%sById" % entity_type.title() method = getattr(self._api, method_name) From f46bbf8e4e5493a16653b3619cc5db66a40739e6 Mon Sep 17 00:00:00 2001 From: psaffrey Date: Thu, 3 Dec 2015 11:10:21 +0000 Subject: [PATCH 41/73] changes to applaunch helpers to make accessing property information more generic --- src/BaseSpacePy/api/AppLaunchHelpers.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/BaseSpacePy/api/AppLaunchHelpers.py b/src/BaseSpacePy/api/AppLaunchHelpers.py index b5c209f..bec670d 100644 --- a/src/BaseSpacePy/api/AppLaunchHelpers.py +++ b/src/BaseSpacePy/api/AppLaunchHelpers.py @@ -393,8 +393,7 @@ def make_launch_json(self, user_supplied_vars, launch_name, api_version, sample_ launch_dict["Properties"] = properties_dict return json.dumps(launch_dict) - def format_property_information(self): - lines = ["\t".join(["Name", "Type", "Default"])] + def property_information_generator(self): minimum_requirements = self.get_minimum_requirements() for property_ in sorted(self.properties): property_name = self.clean_name(property_["Name"]) @@ -404,8 +403,11 @@ def format_property_information(self): output = [property_name, property_type] if property_name in self.defaults: output.append(str(self.defaults[property_name])) - lines.append("\t".join(output)) - return "\n".join(lines) + yield output + + def format_property_information(self): + header = ["\t".join(["Name", "Type", "Default"])] + return "\n".join(header + [ "\t".join(line) for line in self.property_information_generator() ]) def dump_property_information(self): """ From 30329c3e112c0a6dfef6ed8c037993b3760dcdcf Mon Sep 17 00:00:00 2001 From: psaffrey Date: Mon, 7 Dec 2015 10:44:43 +0000 Subject: [PATCH 42/73] Added comment to crucial method --- src/BaseSpacePy/api/AppLaunchHelpers.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/BaseSpacePy/api/AppLaunchHelpers.py b/src/BaseSpacePy/api/AppLaunchHelpers.py index bec670d..c61f67b 100644 --- a/src/BaseSpacePy/api/AppLaunchHelpers.py +++ b/src/BaseSpacePy/api/AppLaunchHelpers.py @@ -49,6 +49,11 @@ def __init__(self, appsession_metadata): self.asm = appsession_metadata def get_refined_appsession_properties(self): + """ + Unpacks the properties from an appsession and refines them ready to make a launch specification + + :return: + """ appsession_properties = self.get_properties() properties = [] defaults = {} From e0d4d35d43807debed487870253df04347acae63 Mon Sep 17 00:00:00 2001 From: psaffrey Date: Mon, 7 Dec 2015 10:49:23 +0000 Subject: [PATCH 43/73] Added more documentation to crucial method --- src/BaseSpacePy/api/AppLaunchHelpers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/BaseSpacePy/api/AppLaunchHelpers.py b/src/BaseSpacePy/api/AppLaunchHelpers.py index c61f67b..f5c8c28 100644 --- a/src/BaseSpacePy/api/AppLaunchHelpers.py +++ b/src/BaseSpacePy/api/AppLaunchHelpers.py @@ -52,7 +52,8 @@ def get_refined_appsession_properties(self): """ Unpacks the properties from an appsession and refines them ready to make a launch specification - :return: + :return: properties (list of dict of "Name" and "Type") + defaults (dict from property name to default value) """ appsession_properties = self.get_properties() properties = [] From 9005670f59c55327617ab5b6dccf624480cf66ed Mon Sep 17 00:00:00 2001 From: psaffrey Date: Wed, 9 Dec 2015 12:14:35 +0000 Subject: [PATCH 44/73] Increased logging level for multipart upload --- .../model/MultipartFileTransfer.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/BaseSpacePy/model/MultipartFileTransfer.py b/src/BaseSpacePy/model/MultipartFileTransfer.py index daee09a..ecda2bf 100644 --- a/src/BaseSpacePy/model/MultipartFileTransfer.py +++ b/src/BaseSpacePy/model/MultipartFileTransfer.py @@ -13,6 +13,7 @@ LOGGER = logging.getLogger(__name__) + class UploadTask(object): ''' Uploads a piece of a large local file. @@ -74,7 +75,7 @@ def execute(self, lock): return self def __str__(self): - return 'File piece %d of %d, total file size %s' % (self.piece, self.total_pieces, Utils.readable_bytes(self.total_size)) + return 'File piece %d of %d, total %s' % (self.piece+1, self.total_pieces, Utils.readable_bytes(self.total_size)) class DownloadTask(object): @@ -180,7 +181,8 @@ def run(self): else: # attempt to run tasks, with retry LOGGER.debug('Worker %s processing task: %s' % (self.name, str(next_task))) - for i in xrange(1, self.retries + 1): + logging.info('%s' % str(next_task)) + for i in xrange(1, self.retries + 1): if self.halt.is_set(): LOGGER.debug('Worker %s exiting, found halt signal' % self.name) self.task_queue.task_done() @@ -230,7 +232,7 @@ class Executor(object): ''' def __init__(self): self.tasks = multiprocessing.JoinableQueue() - self.result_queue = multiprocessing.Queue() + self.result_queue = multiprocessing.Queue() self.halt_event = multiprocessing.Event() self.lock = multiprocessing.Lock() @@ -284,7 +286,7 @@ class MultipartUpload(object): ''' Uploads a (large) file by uploading file parts in separate processes. ''' - def __init__(self, api, local_path, bs_file, process_count, part_size, temp_dir): + def __init__(self, api, local_path, bs_file, process_count, part_size, temp_dir, logger=None): ''' Create a multipart upload object @@ -301,9 +303,9 @@ def __init__(self, api, local_path, bs_file, process_count, part_size, temp_dir) self.process_count = process_count self.part_size = part_size self.temp_dir = temp_dir - + self.start_chunk = 0 - + def upload(self): ''' Start the upload, then when complete retrieve and return the file object from @@ -332,7 +334,7 @@ def _setup(self): cmd = ['split', '-a', '4', '-d', '-b', str(chunk_size), self.local_path, prefix] rc = call(cmd) if rc != 0: - err_msg = "Splitting local file failed: %s" % str.local_path + err_msg = "Splitting local file failed: %s" % self.local_path raise MultiProcessingTaskFailedException(err_msg) self.exe = Executor() @@ -342,8 +344,8 @@ def _setup(self): self.exe.add_workers(self.process_count) self.task_total = fileCount - self.start_chunk + 1 - LOGGER.debug("Total File Size %s" % Utils.readable_bytes(total_size)) - LOGGER.debug("Using File Part Size %d MB" % self.part_size) + logging.info("Total File Size %s" % Utils.readable_bytes(total_size)) + logging.info("Using File Part Size %d MB" % self.part_size) LOGGER.debug("Processes %d" % self.process_count) LOGGER.debug("File Chunk Count %d" % self.task_total) LOGGER.debug("Start Chunk %d" % self.start_chunk) From b531adc5fe4b489f6149477966a49f35cb4d5b2c Mon Sep 17 00:00:00 2001 From: psaffrey Date: Wed, 9 Dec 2015 12:41:17 +0000 Subject: [PATCH 45/73] working with logging levels in multipart --- src/BaseSpacePy/model/MultipartFileTransfer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/BaseSpacePy/model/MultipartFileTransfer.py b/src/BaseSpacePy/model/MultipartFileTransfer.py index ecda2bf..484689c 100644 --- a/src/BaseSpacePy/model/MultipartFileTransfer.py +++ b/src/BaseSpacePy/model/MultipartFileTransfer.py @@ -181,7 +181,7 @@ def run(self): else: # attempt to run tasks, with retry LOGGER.debug('Worker %s processing task: %s' % (self.name, str(next_task))) - logging.info('%s' % str(next_task)) + LOGGER.info('%s' % str(next_task)) for i in xrange(1, self.retries + 1): if self.halt.is_set(): LOGGER.debug('Worker %s exiting, found halt signal' % self.name) @@ -344,8 +344,8 @@ def _setup(self): self.exe.add_workers(self.process_count) self.task_total = fileCount - self.start_chunk + 1 - logging.info("Total File Size %s" % Utils.readable_bytes(total_size)) - logging.info("Using File Part Size %d MB" % self.part_size) + LOGGER.info("Total File Size %s" % Utils.readable_bytes(total_size)) + LOGGER.info("Using File Part Size %d MB" % self.part_size) LOGGER.debug("Processes %d" % self.process_count) LOGGER.debug("File Chunk Count %d" % self.task_total) LOGGER.debug("Start Chunk %d" % self.start_chunk) From 62b9899d51d122b835a9038232428e3fbfa5ee7d Mon Sep 17 00:00:00 2001 From: psaffrey Date: Thu, 10 Dec 2015 10:03:48 +0000 Subject: [PATCH 46/73] Changes to support alternative directories for temp files --- src/BaseSpacePy/api/BaseSpaceAPI.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/BaseSpacePy/api/BaseSpaceAPI.py b/src/BaseSpacePy/api/BaseSpaceAPI.py index 82ceb10..978af4b 100755 --- a/src/BaseSpacePy/api/BaseSpaceAPI.py +++ b/src/BaseSpacePy/api/BaseSpaceAPI.py @@ -13,6 +13,7 @@ import ConfigParser import urlparse import logging +import getpass from BaseSpacePy.api.APIClient import APIClient from BaseSpacePy.api.BaseAPI import BaseAPI @@ -62,7 +63,9 @@ def __init__(self, clientKey=None, clientSecret=None, apiServer=None, version='v self.profile = cred['profile'] # TODO this replacement won't work for all environments self.weburl = cred['apiServer'].replace('api.','') - + + self.tempdir = cred['tempDirectoryBaseName'] if 'tempDirectoryBaseName' in cred else None + apiServerAndVersion = urlparse.urljoin(cred['apiServer'], cred['apiVersion']) super(BaseSpaceAPI, self).__init__(cred['accessToken'], apiServerAndVersion, userAgent, timeout, verbose) @@ -72,6 +75,7 @@ def _setCredentials(self, clientKey, clientSecret, apiServer, apiVersion, appSes for each credential. If clientKey was provided only in config file, include 'name' (in return dict) with profile name. Raises exception if required creds aren't provided (clientKey, clientSecret, apiServer, apiVersion). + also gets the tempdir to use, if any is specified :param clientKey: the client key of the user's app :param clientSecret: the client secret of the user's app @@ -115,6 +119,8 @@ def _setCredentials(self, clientKey, clientSecret, apiServer, apiVersion, appSes cred[conf_item] = lcl_cred[conf_item] except KeyError: cred[conf_item] = local_value + if "tempDirectoryBaseName" in lcl_cred: + cred["tempDirectoryBaseName"] = lcl_cred["tempDirectoryBaseName"] return cred def _getLocalCredentials(self, profile): @@ -155,6 +161,10 @@ def _getLocalCredentials(self, profile): cred['accessToken'] = config.get(section_name, "accessToken") except ConfigParser.NoOptionError: pass + try: + cred['tempDirectoryBaseName'] = config.get(section_name, "tempDirectoryBaseName") + except ConfigParser.NoOptionError: + pass return cred def getAppSessionById(self, Id): @@ -1163,7 +1173,12 @@ def multipartFileUpload(self, resourceType, resourceId, localPath, fileName, dir if partSize <= 5 or partSize > 25: raise UploadPartSizeException("Multipart upload partSize must be >5 MB and <=25 MB") if tempDir is None: - tempDir = mkdtemp() + if self.tempdir: + username = getpass.getuser() + suffix = "pythonsdk_%s" % username + tempDir = mkdtemp(prefix=self.tempdir, suffix=username) + else: + tempDir = mkdtemp() bsFile = self.__initiateMultipartFileUpload__(resourceType, resourceId, fileName, directory, contentType) myMpu = mpu(self, localPath, bsFile, processCount, partSize, temp_dir=tempDir) return myMpu.upload() @@ -1186,7 +1201,10 @@ def multipartFileUploadSample(self, Id, localPath, fileName, directory, contentT if partSize <= 5 or partSize > 25: raise UploadPartSizeException("Multipart upload partSize must be >5 MB and <=25 MB") if tempDir is None: - tempDir = mkdtemp() + if self.tempdir: + tempDir = self.tempdir + else: + tempDir = mkdtemp() bsFile = self.__initiateMultipartFileUploadSample__(Id, fileName, directory, contentType) myMpu = mpu(self, localPath, bsFile, processCount, partSize, temp_dir=tempDir) return myMpu.upload() From be5e94e5639974ea61f5d1f98cd9e7dcc465446b Mon Sep 17 00:00:00 2001 From: psaffrey Date: Thu, 10 Dec 2015 10:04:36 +0000 Subject: [PATCH 47/73] automatically get paged samples/projects --- src/BaseSpacePy/api/BaseAPI.py | 39 ++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/src/BaseSpacePy/api/BaseAPI.py b/src/BaseSpacePy/api/BaseAPI.py index da2de39..c9ec8ba 100644 --- a/src/BaseSpacePy/api/BaseAPI.py +++ b/src/BaseSpacePy/api/BaseAPI.py @@ -12,7 +12,7 @@ from BaseSpacePy.api.APIClient import APIClient from BaseSpacePy.api.BaseSpaceException import * from BaseSpacePy.model import * - +from itertools import chain class BaseAPI(object): ''' @@ -103,18 +103,31 @@ def __listRequest__(self, myModel, resourcePath, method, queryParams, headerPara print ' # Path: ' + str(resourcePath) print ' # QPars: ' + str(queryParams) print ' # Hdrs: ' + str(headerParams) - response = self.apiClient.callAPI(resourcePath, method, queryParams, None, headerParams) - if self.verbose: - self.__json_print__(' # Response: ',response) - if not response: - raise ServerResponseException('No response returned') - if response['ResponseStatus'].has_key('ErrorCode'): - raise ServerResponseException(str(response['ResponseStatus']['ErrorCode'] + ": " + response['ResponseStatus']['Message'])) - elif response['ResponseStatus'].has_key('Message'): - raise ServerResponseException(str(response['ResponseStatus']['Message'])) - - respObj = self.apiClient.deserialize(response, ListResponse.ListResponse) - return [self.apiClient.deserialize(c, myModel) for c in respObj._convertToObjectList()] + number_received = 0 + total_number = None + responses = [] + queryParams["Limit"] = 1024 + while total_number is None or number_received < total_number: + queryParams["Offset"] = number_received + response = self.apiClient.callAPI(resourcePath, method, queryParams, None, headerParams) + if self.verbose: + self.__json_print__(' # Response: ',response) + if not response: + raise ServerResponseException('No response returned') + if response['ResponseStatus'].has_key('ErrorCode'): + raise ServerResponseException(str(response['ResponseStatus']['ErrorCode'] + ": " + response['ResponseStatus']['Message'])) + elif response['ResponseStatus'].has_key('Message'): + raise ServerResponseException(str(response['ResponseStatus']['Message'])) + + respObj = self.apiClient.deserialize(response, ListResponse.ListResponse) + if total_number is None: + total_number = respObj.Response.TotalCount + elif total_number != respObj.Response.TotalCount: + raise ServerResponseException("Inconsistent values in large entity query") + responses.append(respObj) + number_received += respObj.Response.DisplayedCount + + return [self.apiClient.deserialize(c, myModel) for c in chain(*[ ro._convertToObjectList() for ro in responses ])] def __makeCurlRequest__(self, data, url): ''' From 6e57adb61d49e6c8f0f2e5701a0e078acd144c75 Mon Sep 17 00:00:00 2001 From: psaffrey Date: Wed, 23 Dec 2015 09:58:45 +0000 Subject: [PATCH 48/73] Replaced multi-part upload split-and-curl using Python requests and live file-part extraction; removed tmpdir stuff in toplevel class (not needed anymore); fixed unit tests --- src/BaseSpacePy/api/APIClient.py | 23 +++- src/BaseSpacePy/api/BaseAPI.py | 31 +++-- src/BaseSpacePy/api/BaseSpaceAPI.py | 39 ++---- .../model/MultipartFileTransfer.py | 58 ++++----- test/unit_tests.py | 111 ++++++++---------- 5 files changed, 126 insertions(+), 136 deletions(-) diff --git a/src/BaseSpacePy/api/APIClient.py b/src/BaseSpacePy/api/APIClient.py index d16fa82..9c43ff4 100644 --- a/src/BaseSpacePy/api/APIClient.py +++ b/src/BaseSpacePy/api/APIClient.py @@ -65,7 +65,7 @@ def __forcePostCall__(self, resourcePath, postData, headers): response = requests.post(resourcePath, data=json.dumps(postData), headers=headers) return response.text - def __putCall__(self, resourcePath, headers, transFile): + def __putCall__(self, resourcePath, headers, data): ''' Performs a REST PUT call to the API server. @@ -74,10 +74,21 @@ def __putCall__(self, resourcePath, headers, transFile): :param transFile: the name of the file containing only data to be PUT :returns: server response (a string containing upload status message (from curl?) followed by json response) ''' - headerPrep = [k + ':' + headers[k] for k in headers.keys()] - cmd = 'curl -H "x-access-token:' + self.apiKey + '" -H "Content-MD5:' + headers['Content-MD5'].strip() +'" -T "'+ transFile +'" -X PUT ' + resourcePath - p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) - return p.stdout.read() + # headerPrep = [k + ':' + headers[k] for k in headers.keys()] + # cmd = 'curl -H "x-access-token:' + self.apiKey + '" -H "Content-MD5:' + headers['Content-MD5'].strip() +'" -T "'+ transFile +'" -X PUT ' + resourcePath + # p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) + # output = p.stdout.read() + # print output + # return output + import requests + put_headers = { + 'Content-MD5' : headers['Content-MD5'].strip(), + 'x-access-token': self.apiKey + } + put_val = requests.put(resourcePath, data, headers=put_headers) + if put_val.status_code != 200: + raise ServerResponseException("Multi-part upload: Server return code %s with error %s" % (put_val.status_code, put_val.reason)) + return put_val.text def callAPI(self, resourcePath, method, queryParams, postData, headerParams=None, forcePost=False): ''' @@ -145,7 +156,7 @@ def callAPI(self, resourcePath, method, queryParams, postData, headerParams=None if method == 'DELETE': raise NotImplementedError("DELETE REST API calls aren't currently supported") response = self.__putCall__(url, headers, data) - response = response.split()[-1] # discard upload status msg (from curl put?) + response = response.split()[-1] # discard upload status msg (from curl put?) else: raise RestMethodException('Method ' + method + ' is not recognized.') diff --git a/src/BaseSpacePy/api/BaseAPI.py b/src/BaseSpacePy/api/BaseAPI.py index c9ec8ba..d0180dd 100644 --- a/src/BaseSpacePy/api/BaseAPI.py +++ b/src/BaseSpacePy/api/BaseAPI.py @@ -19,7 +19,7 @@ class BaseAPI(object): Parent class for BaseSpaceAPI and BillingAPI classes ''' - def __init__(self, AccessToken, apiServerAndVersion, userAgent, timeout=10, verbose=False): + def __init__(self, AccessToken, apiServerAndVersion, userAgent=None, timeout=10, verbose=False): ''' :param AccessToken: the current access token :param apiServerAndVersion: the api server URL with api version @@ -83,16 +83,20 @@ def __singleRequest__(self, myModel, resourcePath, method, queryParams, headerPa else: return responseObject - def __listRequest__(self, myModel, resourcePath, method, queryParams, headerParams): + def __listRequest__(self, myModel, resourcePath, method, queryParams, headerParams, sort=True): ''' Call a REST API that returns a list and deserialize response into a list of objects of the provided model. Handles errors from server. + Sorting by date for each call is the default, so that if a new item is created while we're paging through + we'll pick it up at the end. However, this should be switched off for some calls (like variants) + :param myModel: a Model type to return a list of :param resourcePath: the api url path to call (without server and version) :param method: the REST method type, eg. GET :param queryParams: a dictionary of query parameters :param headerParams: a dictionary of header parameters + :param sort: sort the outputs from the API to prevent race-conditions :raises ServerResponseException: if server returns an error or has no response :returns: a list of instances of the provided model @@ -106,7 +110,14 @@ def __listRequest__(self, myModel, resourcePath, method, queryParams, headerPara number_received = 0 total_number = None responses = [] - queryParams["Limit"] = 1024 + # if the user explicitly sets a Limit in queryParams, just make one call with that limit + justOne = False + if "Limit" in queryParams: + justOne = True + else: + queryParams["Limit"] = 1024 + if sort: + queryParams["SortBy"] = "DateCreated" while total_number is None or number_received < total_number: queryParams["Offset"] = number_received response = self.apiClient.callAPI(resourcePath, method, queryParams, None, headerParams) @@ -120,11 +131,17 @@ def __listRequest__(self, myModel, resourcePath, method, queryParams, headerPara raise ServerResponseException(str(response['ResponseStatus']['Message'])) respObj = self.apiClient.deserialize(response, ListResponse.ListResponse) - if total_number is None: - total_number = respObj.Response.TotalCount - elif total_number != respObj.Response.TotalCount: - raise ServerResponseException("Inconsistent values in large entity query") responses.append(respObj) + if justOne: + break + # if a TotalCount is not an attribute, assume we have all of them (eg. variantsets) + if not hasattr(respObj.Response, "TotalCount"): + break + # allow the total number to change on each call + # to catch the race condition where a new entity appears while we're calling + total_number = respObj.Response.TotalCount + if respObj.Response.DisplayedCount == 0: + raise ServerResponseException("Paged query returned no results") number_received += respObj.Response.DisplayedCount return [self.apiClient.deserialize(c, myModel) for c in chain(*[ ro._convertToObjectList() for ro in responses ])] diff --git a/src/BaseSpacePy/api/BaseSpaceAPI.py b/src/BaseSpacePy/api/BaseSpaceAPI.py index 978af4b..bb00457 100755 --- a/src/BaseSpacePy/api/BaseSpaceAPI.py +++ b/src/BaseSpacePy/api/BaseSpaceAPI.py @@ -64,8 +64,6 @@ def __init__(self, clientKey=None, clientSecret=None, apiServer=None, version='v # TODO this replacement won't work for all environments self.weburl = cred['apiServer'].replace('api.','') - self.tempdir = cred['tempDirectoryBaseName'] if 'tempDirectoryBaseName' in cred else None - apiServerAndVersion = urlparse.urljoin(cred['apiServer'], cred['apiVersion']) super(BaseSpaceAPI, self).__init__(cred['accessToken'], apiServerAndVersion, userAgent, timeout, verbose) @@ -75,7 +73,6 @@ def _setCredentials(self, clientKey, clientSecret, apiServer, apiVersion, appSes for each credential. If clientKey was provided only in config file, include 'name' (in return dict) with profile name. Raises exception if required creds aren't provided (clientKey, clientSecret, apiServer, apiVersion). - also gets the tempdir to use, if any is specified :param clientKey: the client key of the user's app :param clientSecret: the client secret of the user's app @@ -92,11 +89,13 @@ def _setCredentials(self, clientKey, clientSecret, apiServer, apiVersion, appSes if profile != "default": authenticate_cmd = "%s --config %s" % (authenticate_cmd, profile) cred = {} - # set profile name - if 'name' in lcl_cred: - cred['profile'] = lcl_cred['name'] - else: - cred['profile'] = profile + # if access tokens have not been provided through the constructor, + # set a profile name + if not accessToken: + if 'name' in lcl_cred: + cred['profile'] = lcl_cred['name'] + else: + cred['profile'] = profile # required credentials REQUIRED = ["accessToken", "apiServer", "apiVersion"] for conf_item in REQUIRED: @@ -119,8 +118,6 @@ def _setCredentials(self, clientKey, clientSecret, apiServer, apiVersion, appSes cred[conf_item] = lcl_cred[conf_item] except KeyError: cred[conf_item] = local_value - if "tempDirectoryBaseName" in lcl_cred: - cred["tempDirectoryBaseName"] = lcl_cred["tempDirectoryBaseName"] return cred def _getLocalCredentials(self, profile): @@ -132,6 +129,8 @@ def _getLocalCredentials(self, profile): :returns: A dictionary with credentials from local config file ''' config_file = os.path.join(os.path.expanduser('~/.basespace'), "%s.cfg" % profile) + if not os.path.exists(config_file): + raise CredentialsException("Could not find config file: %s" % config_file) section_name = "DEFAULT" cred = {} config = ConfigParser.SafeConfigParser() @@ -161,10 +160,6 @@ def _getLocalCredentials(self, profile): cred['accessToken'] = config.get(section_name, "accessToken") except ConfigParser.NoOptionError: pass - try: - cred['tempDirectoryBaseName'] = config.get(section_name, "tempDirectoryBaseName") - except ConfigParser.NoOptionError: - pass return cred def getAppSessionById(self, Id): @@ -836,7 +831,7 @@ def getAvailableGenomes(self, queryPars=None): method = 'GET' headerParams = {} return self.__listRequest__(GenomeV1.GenomeV1, - resourcePath, method, queryParams, headerParams) + resourcePath, method, queryParams, headerParams, sort=False) def getIntervalCoverage(self, Id, Chrom, StartPos, EndPos): ''' @@ -902,7 +897,7 @@ def filterVariantSet(self,Id, Chrom, StartPos, EndPos, Format='json', queryPars= if Format == 'vcf': raise NotImplementedError("Returning native VCF format isn't yet supported by BaseSpacePy") else: - return self.__listRequest__(Variant.Variant, resourcePath, method, queryParams, headerParams) + return self.__listRequest__(Variant.Variant, resourcePath, method, queryParams, headerParams, sort=False) def getVariantMetadata(self, Id, Format='json'): ''' @@ -1152,7 +1147,7 @@ def __finalizeMultipartFileUpload__(self, Id): return self.__singleRequest__(FileResponse.FileResponse, resourcePath, method, queryParams, headerParams, postData=postData, forcePost=1) - def multipartFileUpload(self, resourceType, resourceId, localPath, fileName, directory, contentType, tempDir=None, processCount=10, partSize=25): + def multipartFileUpload(self, resourceType, resourceId, localPath, fileName, directory, contentType, processCount=10, partSize=25): ''' Method for multi-threaded file-upload for parallel transfer of very large files (currently only runs on unix systems) @@ -1162,7 +1157,6 @@ def multipartFileUpload(self, resourceType, resourceId, localPath, fileName, dir :param fileName: The desired filename on the server :param directory: The desired directory name on the server (empty string will place it in the root directory) :param contentType: The content type of the file - :param tempdir: (optional) Temp directory to use for temporary file chunks to upload :param processCount: (optional) The number of processes to be used, default 10 :param partSize: (optional) The size in MB of individual upload parts (must be >5 Mb and <=25 Mb), default 25 :returns: a File instance, which has been updated after the upload has completed. @@ -1172,15 +1166,8 @@ def multipartFileUpload(self, resourceType, resourceId, localPath, fileName, dir # First create file object in BaseSpace, then create multipart upload object and start upload if partSize <= 5 or partSize > 25: raise UploadPartSizeException("Multipart upload partSize must be >5 MB and <=25 MB") - if tempDir is None: - if self.tempdir: - username = getpass.getuser() - suffix = "pythonsdk_%s" % username - tempDir = mkdtemp(prefix=self.tempdir, suffix=username) - else: - tempDir = mkdtemp() bsFile = self.__initiateMultipartFileUpload__(resourceType, resourceId, fileName, directory, contentType) - myMpu = mpu(self, localPath, bsFile, processCount, partSize, temp_dir=tempDir) + myMpu = mpu(self, localPath, bsFile, processCount, partSize) return myMpu.upload() def multipartFileUploadSample(self, Id, localPath, fileName, directory, contentType, tempDir=None, processCount=10, partSize=25): diff --git a/src/BaseSpacePy/model/MultipartFileTransfer.py b/src/BaseSpacePy/model/MultipartFileTransfer.py index 484689c..fc5adc3 100644 --- a/src/BaseSpacePy/model/MultipartFileTransfer.py +++ b/src/BaseSpacePy/model/MultipartFileTransfer.py @@ -18,14 +18,14 @@ class UploadTask(object): ''' Uploads a piece of a large local file. ''' - def __init__(self, api, bs_file_id, piece, total_pieces, local_path, total_size, temp_dir): + def __init__(self, api, bs_file_id, piece, total_pieces, local_path, total_size, chunk_size): self.api = api self.bs_file_id = bs_file_id # the BaseSpace File Id self.piece = piece # piece number self.total_pieces = total_pieces # out of total piece count self.local_path = local_path # the path of the local file to be uploaded, including file name self.total_size = total_size # total file size of upload, for reporting - self.temp_dir = temp_dir # temp location to store file chunks for upload + self.chunk_size = chunk_size # chunk size # tasks must implement these attributes and execute() self.success = False @@ -33,29 +33,18 @@ def __init__(self, api, bs_file_id, piece, total_pieces, local_path, total_size, def execute(self, lock): ''' - Upload a piece of the target file, first splitting the local file into a temp file. Calculate md5 of file piece and pass to upload method. Lock is not used (but needed since worker sends it for multipart download) ''' try: fname = os.path.basename(self.local_path) - # this relies on the way the calling function has split the file - # but we still need to pass around the piece numbers because the BaseSpace API needs them - # to reassemble the file at the other end - # the zfill(4) is to make sure we have a zero padded suffix that split -a 4 -d will make - transFile = os.path.join(self.temp_dir, fname + str(self.piece).zfill(4)) - #cmd = ['split', '-d', '-n', str(self.piece) + '/' + str(self.total_pieces), self.local_path] - #with open(transFile, "w") as fp: - # rc = call(cmd, stdout=fp) - # if rc != 0: - # self.sucess = False - # self.err_msg = "Splitting local file failed for piece %s" % str(self.piece) - # return self - with open(transFile, "r") as f: - out = f.read() - self.md5 = hashlib.md5(out).digest().encode('base64') + chunk_data = "" + with open(self.local_path) as fh: + fh.seek(self.piece * self.chunk_size) + chunk_data = fh.read(self.chunk_size) + self.md5 = hashlib.md5(chunk_data).digest().encode('base64') try: - res = self.api.__uploadMultipartUnit__(self.bs_file_id,self.piece+1,self.md5,transFile) + res = self.api.__uploadMultipartUnit__(self.bs_file_id,self.piece+1,self.md5,chunk_data) except Exception as e: self.success = False self.err_msg = str(e) @@ -66,8 +55,6 @@ def execute(self, lock): else: self.success = False self.err_msg = "Error - empty response from uploading file piece or missing ETag in response" - if self.success: - os.remove(transFile) # capture exception, since unpickleable exceptions may block except Exception as e: self.success = False @@ -286,7 +273,7 @@ class MultipartUpload(object): ''' Uploads a (large) file by uploading file parts in separate processes. ''' - def __init__(self, api, local_path, bs_file, process_count, part_size, temp_dir, logger=None): + def __init__(self, api, local_path, bs_file, process_count, part_size, logger=None): ''' Create a multipart upload object @@ -295,14 +282,12 @@ def __init__(self, api, local_path, bs_file, process_count, part_size, temp_dir, :param bs_file: the File object of the newly created BaseSpace File to upload :param process_count: the number of process to use for uploading :param part_size: in MB, the size of each uploaded part - :param temp_dir: temp directory to store file pieces for upload ''' self.api = api self.local_path = local_path self.remote_file = bs_file self.process_count = process_count self.part_size = part_size - self.temp_dir = temp_dir self.start_chunk = 0 @@ -319,27 +304,28 @@ def _setup(self): ''' Determine number of file pieces to upload, add upload tasks to work queue ''' - logfile = os.path.join(self.temp_dir, "main.log") - total_size = os.path.getsize(self.local_path) - fileCount = int(total_size/(self.part_size*1024*1024)) + 1 + from math import ceil + total_size = os.path.getsize(self.local_path) + # round up to get a number of chunks that will be enough for the whole file + fileCount = int(ceil(total_size/float(self.part_size*1024*1024))) - chunk_size = (total_size / fileCount) + 1 + chunk_size = self.part_size*1024*1024 assert chunk_size * fileCount > total_size - fname = os.path.basename(self.local_path) - prefix = os.path.join(self.temp_dir, fname) + # fname = os.path.basename(self.local_path) + # prefix = os.path.join(self.temp_dir, fname) # -a 4 always use 4 digit sufixes, to make sure we can predict the filenames # -d use digits as suffixes, not letters # -b chunk size (in bytes) - cmd = ['split', '-a', '4', '-d', '-b', str(chunk_size), self.local_path, prefix] - rc = call(cmd) - if rc != 0: - err_msg = "Splitting local file failed: %s" % self.local_path - raise MultiProcessingTaskFailedException(err_msg) + # cmd = ['split', '-a', '4', '-d', '-b', str(chunk_size), self.local_path, prefix] + # rc = call(cmd) + # if rc != 0: + # err_msg = "Splitting local file failed: %s" % self.local_path + # raise MultiProcessingTaskFailedException(err_msg) self.exe = Executor() for i in xrange(self.start_chunk, fileCount): - t = UploadTask(self.api, self.remote_file.Id, i, fileCount, self.local_path, total_size, self.temp_dir) + t = UploadTask(self.api, self.remote_file.Id, i, fileCount, self.local_path, total_size, chunk_size) self.exe.add_task(t) self.exe.add_workers(self.process_count) self.task_total = fileCount - self.start_chunk + 1 diff --git a/test/unit_tests.py b/test/unit_tests.py index d71e95d..be807ac 100644 --- a/test/unit_tests.py +++ b/test/unit_tests.py @@ -21,8 +21,8 @@ # Dependencies: # ============ -# 1. Create a profile named 'unit_tests' in ~/.basespacepy.cfg that has the credentials for an app on https://portal-hoth.illumina.com; -# (there should also be a 'DEFALT' profile in the config file) +# 1. Create a config file in ~/.basespace/unit_tests.cfg that has the credentials for an app on https://portal-hoth.illumina.com; +# you can do this with: bs -c unit_tests authenticate --api-server https://api.cloud-hoth.illumina.com/ # 2. Import the following data from Public Dataset 'MiSeq B. cereus demo data' on cloud-hoth.illumina.com: # 2.a. Project name 'BaseSpaceDemo' (Id 596596), and # 2.b. Run name 'BacillusCereus' (Id 555555) @@ -40,6 +40,7 @@ 'file_large_md5': '9267236a2d870da1d4cb73868bb51b35', # for file id 9896135 # for upload tests 'file_small_upload': 'data/test.small.upload.txt', + 'file_small_upload_contents': open('data/test.small.upload.txt').read(), 'file_large_upload': 'data/BC-12_S12_L001_R2_001.fastq.gz', 'file_small_upload_size': 11, 'file_large_upload_size': 57995799, @@ -201,7 +202,7 @@ def test__uploadMultipartUnit__(self): Id = file.Id, partNumber = 1, md5 = md5, - data = tconst['file_small_upload']) + data = tconst['file_small_upload_contents']) self.assertNotEqual(response, None, 'Upload part failure will return None') self.assertTrue('ETag' in response['Response'], 'Upload part success will contain a Response dict with an ETag element') @@ -220,7 +221,7 @@ def test__finalizeMultipartFileUpload__(self): Id = file.Id, partNumber = 1, md5 = md5, - data = tconst['file_small_upload']) + data = tconst['file_small_upload_contents']) final_file = self.api.__finalizeMultipartFileUpload__(file.Id) self.assertEqual(final_file.UploadStatus, 'complete') @@ -319,7 +320,6 @@ def testMultiPartFileUpload(self): fileName=fileName, directory=testDir, contentType=tconst['file_large_upload_content_type'], - tempDir=None, processCount = 4, partSize= 10, # MB, chunk size #tempDir = args.temp_dir @@ -1495,8 +1495,7 @@ def testGetAppSessionPropertiesById(self): def testGetAppSessionPropertiesByIdWithQp(self): props = self.api.getAppSessionPropertiesById(self.ssn.Id, qp({'Limit':1})) - self.assertTrue(any((prop.Items[0].Id == self.ar.Id) for prop in props.Items if prop.Name == "Output.AppResults")) - self.assertEqual(len(props.Items), 1) + self.assertEqual(len(props.Items), 1) def testGetAppSessionPropertyByName(self): prop = self.api.getAppSessionPropertyByName(self.ssn.Id, 'Output.AppResults') @@ -1510,12 +1509,20 @@ def testGetAppSessionPropertyByNameWithQp(self): def testGetAppSessionInputsById(self): props = self.api.getAppSessionInputsById(self.ssn.Id) - self.assertEqual(len(props), 0) + self.assertEqual(len(props), 1) + self.assertEqual("Samples" in props, True) + self.assertEqual(len(props["Samples"].Items), 0) + # NB: these have changed from previous versions of the unit tests + # because it looks like appsessions created through the API now have an (empty) input samples list by default # TODO can't test this easily since self-created ssn don't have inputs. Add POST properties for ssns, and manually add an 'Input.Test' property, then test for it? def testGetAppSessionInputsByIdWithQp(self): props = self.api.getAppSessionInputsById(self.ssn.Id, qp({'Limit':1})) - self.assertEqual(len(props), 0) + self.assertEqual(len(props), 1) + self.assertEqual("Samples" in props, True) + self.assertEqual(len(props["Samples"].Items), 0) + # NB: these have changed from previous versions of the unit tests + # because it looks like appsessions created through the API now have an (empty) input samples list by default # TODO same as test above def testSetAppSessionState_UpdatedStatus(self): @@ -1679,14 +1686,14 @@ def setUp(self): def test_setCredentials_AllFromProfile(self): creds = self.api._setCredentials(clientKey=None, clientSecret=None, - apiServer=None, apiVersion=None, appSessionId='', accessToken='', + apiServer=None, appSessionId='', apiVersion=self.api.version, accessToken='', profile=self.profile) - self.assertEqual(creds['clientKey'], self.api.key) - self.assertEqual('profile' in creds, True) - self.assertEqual(creds['clientSecret'], self.api.secret) - self.assertEqual(urljoin(creds['apiServer'], creds['apiVersion']), self.api.apiClient.apiServerAndVersion) - self.assertEqual(creds['apiVersion'], self.api.version) - self.assertEqual(creds['appSessionId'], self.api.appSessionId) + # self.assertEqual(creds['clientKey'], self.api.key) + # self.assertEqual('profile' in creds, True) + # self.assertEqual(creds['clientSecret'], self.api.secret) + self.assertEqual(urljoin(creds['apiServer'], self.api.version), self.api.apiClient.apiServerAndVersion) + # self.assertEqual(creds['apiVersion'], self.api.version) + # self.assertEqual(creds['appSessionId'], self.api.appSessionId) self.assertEqual(creds['accessToken'], self.api.getAccessToken()) def test_setCredentials_AllFromConstructor(self): @@ -1705,7 +1712,7 @@ def test_setCredentials_MissingConfigCredsException(self): # Danger: if this test fails unexpectedly, the config file may not be renamed back to the original name # 1) mv current .basespacepy.cfg, 2) create new with new content, # 3) run test, 4) erase new, 5) mv current back - cfg = os.path.expanduser('~/.basespacepy.cfg') + cfg = os.path.expanduser('~/.basespace/unit_tests.cfg') tmp_cfg = cfg + '.unittesting.donotdelete' shutil.move(cfg, tmp_cfg) new_cfg_content = ("[" + self.profile + "]\n" @@ -1724,42 +1731,43 @@ def test__setCredentials_DefaultsForOptionalArgs(self): # Danger: if this test fails unexpectedly, the config file may not be renamed back to the original name # 1) mv current .basespacepy.cfg, 2) create new with new content, # 3) run test, 4) erase new, 5) mv current back - cfg = os.path.expanduser('~/.basespacepy.cfg') + cfg = os.path.expanduser('~/.basespace/unit_tests.cfg') tmp_cfg = cfg + '.unittesting.donotdelete' shutil.move(cfg, tmp_cfg) - new_cfg_content = ("[" + self.profile + "]\n" + new_cfg_content = ("[DEFAULT]\n" "clientKey=test\n" "clientSecret=test\n" "apiServer=test\n" - "apiVersion=test\n") + "apiVersion=test\n" + "accessToken=test\n") with open(cfg, "w") as f: f.write(new_cfg_content) creds = self.api._setCredentials(clientKey=None, clientSecret=None, - apiServer=None, apiVersion=None, appSessionId='', accessToken='', + apiServer=None, apiVersion=self.api.version, appSessionId='', accessToken='', profile=self.profile) self.assertEqual(creds['appSessionId'], '') - self.assertEqual(creds['accessToken'], '') + self.assertEqual(creds['accessToken'], 'test') os.remove(cfg) shutil.move(tmp_cfg, cfg) def test__getLocalCredentials(self): creds = self.api._getLocalCredentials(profile='unit_tests') self.assertEqual('name' in creds, True) - self.assertEqual('clientKey' in creds, True) - self.assertEqual('clientSecret' in creds, True) + # self.assertEqual('clientKey' in creds, True) + # self.assertEqual('clientSecret' in creds, True) self.assertEqual('apiServer' in creds, True) - self.assertEqual('apiVersion' in creds, True) - self.assertEqual('appSessionId' in creds, True) + # self.assertEqual('apiVersion' in creds, True) + # self.assertEqual('appSessionId' in creds, True) self.assertEqual('accessToken' in creds, True) def test__getLocalCredentials_DefaultProfile(self): creds = self.api._getLocalCredentials(profile=self.profile) self.assertEqual('name' in creds, True) - self.assertEqual('clientKey' in creds, True) - self.assertEqual('clientSecret' in creds, True) +# self.assertEqual('clientKey' in creds, True) +# self.assertEqual('clientSecret' in creds, True) self.assertEqual('apiServer' in creds, True) - self.assertEqual('apiVersion' in creds, True) - self.assertEqual('appSessionId' in creds, True) + # self.assertEqual('apiVersion' in creds, True) +# self.assertEqual('appSessionId' in creds, True) self.assertEqual('accessToken' in creds, True) def test__getLocalCredentials_MissingProfile(self): @@ -1922,12 +1930,12 @@ def setUp(self): def test__init__(self): creds = self.api._getLocalCredentials(profile='unit_tests') - self.assertEqual(creds['appSessionId'], self.api.appSessionId) - self.assertEqual(creds['clientKey'], self.api.key) - self.assertEqual(creds['clientSecret'], self.api.secret) + # self.assertEqual(creds['appSessionId'], self.api.appSessionId) + # self.assertEqual(creds['clientKey'], self.api.key) + # self.assertEqual(creds['clientSecret'], self.api.secret) self.assertEqual(creds['apiServer'], self.api.apiServer) - self.assertEqual(creds['apiVersion'], self.api.version) - self.assertEqual(creds['name'], self.api.profile) + # self.assertEqual(creds['apiVersion'], self.api.version) + # self.assertEqual(creds['name'], self.api.profile) self.assertEqual(creds['apiServer'].replace('api.',''), self.api.weburl) class TestBaseAPIMethods(TestCase): @@ -1942,7 +1950,7 @@ def test__init__(self): accessToken = "123" apiServerAndVersion = "http://api.tv" timeout = 50 - bapi = BaseAPI(accessToken, apiServerAndVersion, timeout) + bapi = BaseAPI(accessToken, apiServerAndVersion, timeout=timeout) self.assertEqual(bapi.apiClient.apiKey, accessToken) self.assertEqual(bapi.apiClient.apiServerAndVersion, apiServerAndVersion) self.assertEqual(bapi.apiClient.timeout, timeout) @@ -1986,15 +1994,6 @@ def test__singleRequest__WithForcePost(self): queryParams, headerParams, postData=postData, forcePost=1) self.assertTrue(hasattr(file, 'Id'), 'Successful force post should return file object with Id attribute here') - def test__singleRequest__Verbose(self): - # get current user - resourcePath = '/users/current' - method = 'GET' - queryParams = {} - headerParams = {} - user = self.bapi.__singleRequest__(UserResponse.UserResponse, resourcePath, method, queryParams, headerParams, verbose=True) - self.assertTrue(hasattr(user, 'Id')) - @skip("Not sure how to test this, requires no response from api server") def test__singleRequest__NoneResponseException(self): pass @@ -2027,16 +2026,6 @@ def test__listRequest__(self): self.assertTrue(isinstance(runs, list)) self.assertTrue(hasattr(runs[0], "Id")) - def test__listRequest__Verbose(self): - # get current user - resourcePath = '/users/current/runs' - method = 'GET' - queryParams = {} - headerParams = {} - runs = self.bapi.__listRequest__(Run.Run, resourcePath, method, queryParams, headerParams, verbose=True) - self.assertTrue(isinstance(runs, list)) - self.assertTrue(hasattr(runs[0], "Id")) - @skip("Not sure how to test this, requires no response from api server") def test__listRequest__NoneResponseException(self): pass @@ -2157,8 +2146,8 @@ def test__putCall__(self): resourcePath = resourcePath.replace('{Id}', file.Id) resourcePath = resourcePath.replace('{partNumber}', str(1)) headerParams = {'Content-MD5': md5} - transFile = tconst['file_small_upload'] - putResp = self.apiClient.__putCall__(resourcePath=self.apiClient.apiServerAndVersion + resourcePath, headers=headerParams, transFile=transFile) + data = tconst['file_small_upload_contents'] + putResp = self.apiClient.__putCall__(resourcePath=self.apiClient.apiServerAndVersion + resourcePath, headers=headerParams, data=data) #print "RESPONSE is: " + putResp jsonResp = putResp.split()[-1] # normally done in callAPI() dictResp = json.loads(jsonResp) @@ -2254,8 +2243,8 @@ def testCallAPI_PUT(self): resourcePath = resourcePath.replace('{partNumber}', str(1)) headerParams = {'Content-MD5': md5} queryParams = {} # not used for PUT calls - transFile = tconst['file_small_upload'] - dictResp = self.apiClient.callAPI(resourcePath, method, queryParams, postData=transFile, headerParams=headerParams) + data = tconst['file_small_upload_contents'] + dictResp = self.apiClient.callAPI(resourcePath, method, queryParams, postData=data, headerParams=headerParams) self.assertTrue('Response' in dictResp, 'Successful force post should return json with Response attribute: ' + str(dictResp)) self.assertTrue('ETag' in dictResp['Response'], 'Successful force post should return json with Response with Id attribute: ' + str(dictResp)) @@ -2283,7 +2272,7 @@ def testCallAPI_HandleHttpError_ForGET(self): self.assertTrue('ResponseStatus' in dictResp, 'response is: ' + str(dictResp)) self.assertTrue('ErrorCode' in dictResp['ResponseStatus']) self.assertTrue('Message' in dictResp['ResponseStatus']) - self.assertEqual(dictResp['ResponseStatus']['Message'], 'Unauthorized') + self.assertTrue('Unrecognized access token' in dictResp['ResponseStatus']['Message']) def testCallAPI_HandleHttpError_ForPOST(self): # bad access token throws 401 Error and HTTPError exception by urllib2; create a project uses POST @@ -2297,7 +2286,7 @@ def testCallAPI_HandleHttpError_ForPOST(self): self.assertTrue('ResponseStatus' in dictResp, 'response is: ' + str(dictResp)) self.assertTrue('ErrorCode' in dictResp['ResponseStatus']) self.assertTrue('Message' in dictResp['ResponseStatus']) - self.assertEqual(dictResp['ResponseStatus']['Message'], 'Unauthorized') + self.assertTrue('Unrecognized access token' in dictResp['ResponseStatus']['Message']) @skip('Not sure how to cause json returned from server to be malformed, in order to cause an exception in json parsing') def testCallAPI_JsonParsingException(self): From 9811675dd3a32656f930f60299159c253d0e7911 Mon Sep 17 00:00:00 2001 From: psaffrey Date: Tue, 5 Jan 2016 10:43:11 +0000 Subject: [PATCH 49/73] Fixed bug where projects with no samples cause a crash --- src/BaseSpacePy/api/BaseAPI.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BaseSpacePy/api/BaseAPI.py b/src/BaseSpacePy/api/BaseAPI.py index d0180dd..f19d96f 100644 --- a/src/BaseSpacePy/api/BaseAPI.py +++ b/src/BaseSpacePy/api/BaseAPI.py @@ -140,7 +140,7 @@ def __listRequest__(self, myModel, resourcePath, method, queryParams, headerPara # allow the total number to change on each call # to catch the race condition where a new entity appears while we're calling total_number = respObj.Response.TotalCount - if respObj.Response.DisplayedCount == 0: + if total_number > 0 and respObj.Response.DisplayedCount == 0: raise ServerResponseException("Paged query returned no results") number_received += respObj.Response.DisplayedCount From 6ff2a962aa8bcc39f324ff0baf043665f836af53 Mon Sep 17 00:00:00 2001 From: psaffrey Date: Thu, 7 Jan 2016 12:39:17 +0000 Subject: [PATCH 50/73] removed DEFAULT_SCOPE - the developer must supply this. Also cleaned up error handling. --- src/BaseSpacePy/api/AuthenticationAPI.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/BaseSpacePy/api/AuthenticationAPI.py b/src/BaseSpacePy/api/AuthenticationAPI.py index e72412c..e274226 100644 --- a/src/BaseSpacePy/api/AuthenticationAPI.py +++ b/src/BaseSpacePy/api/AuthenticationAPI.py @@ -27,11 +27,12 @@ Also partly available here is obtaining session tokens (cookies), although these are not currently used. """ -DEFAULT_SCOPE = "CREATE GLOBAL,BROWSE GLOBAL,CREATE PROJECTS,READ GLOBAL" - class AuthenticationException(Exception): pass +class AuthenticationScopeException(AuthenticationException): + pass + class AuthenticationAPI(object): DEFAULT_CONFIG_NAME = "DEFAULT" @@ -99,7 +100,8 @@ def __init__(self, config_path, api_server, api_version): super(OAuthAuthentication, self).__init__(config_path, api_server) self.api_version = api_version - def set_oauth_details(self, client_id, client_secret, scope=DEFAULT_SCOPE): + def set_oauth_details(self, client_id, client_secret, scopes): + scope_str = ",".join(scopes) OAUTH_URI = "%s%s/oauthv2/deviceauthorization" % (self.api_server, self.api_version) TOKEN_URI = "%s%s/oauthv2/token" % (self.api_server, self.api_version) s = requests.session() @@ -107,7 +109,7 @@ def set_oauth_details(self, client_id, client_secret, scope=DEFAULT_SCOPE): auth_payload = { "response_type": "device_code", "client_id": client_id, - "scope": scope, + "scope": scope_str, } try: r = s.post(url=OAUTH_URI, @@ -119,6 +121,11 @@ def set_oauth_details(self, client_id, client_secret, scope=DEFAULT_SCOPE): payload = r.json() except ValueError: raise AuthenticationException("bad payload from server - perhaps you should use https instead of http?") + if 'error' in payload: + if payload['error'] == 'invalid_scope': + raise AuthenticationScopeException("Authentication requested with invalid scope: %s" % scope_str) + else: + raise AuthenticationException("Authentication failed") auth_url = payload["verification_with_code_uri"] auth_code = payload["device_code"] print "please authenticate here: %s" % auth_url From 50604f740f7a45433a4f72b28fc428fab67e90dc Mon Sep 17 00:00:00 2001 From: psaffrey Date: Tue, 12 Jan 2016 12:00:50 +0000 Subject: [PATCH 51/73] Improved error reporting in raised exceptions --- src/BaseSpacePy/api/AuthenticationAPI.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/BaseSpacePy/api/AuthenticationAPI.py b/src/BaseSpacePy/api/AuthenticationAPI.py index e274226..33698e4 100644 --- a/src/BaseSpacePy/api/AuthenticationAPI.py +++ b/src/BaseSpacePy/api/AuthenticationAPI.py @@ -125,7 +125,8 @@ def set_oauth_details(self, client_id, client_secret, scopes): if payload['error'] == 'invalid_scope': raise AuthenticationScopeException("Authentication requested with invalid scope: %s" % scope_str) else: - raise AuthenticationException("Authentication failed") + msg = payload['error_description'] if 'error_description' in payload else payload['error'] + raise AuthenticationException(msg) auth_url = payload["verification_with_code_uri"] auth_code = payload["device_code"] print "please authenticate here: %s" % auth_url From 431b4628e482fa0743a2cc5164ab25176ccc2bec Mon Sep 17 00:00:00 2001 From: psaffrey Date: Tue, 2 Feb 2016 11:45:18 +0000 Subject: [PATCH 52/73] Removed pycurl --- README.md | 2 +- doc/_update_doc/Getting Started.txt | 2 +- doc/html/Getting Started.html | 2 +- .../BaseSpacePy/api/BaseSpaceAPI.html | 7 +++-- doc/html/_sources/Getting Started.txt | 2 +- doc/html/searchindex.js | 2 +- doc/latex/BaseSpacePy.tex | 2 +- src/BaseSpacePy/api/APIClient.py | 20 +++----------- src/BaseSpacePy/api/BaseAPI.py | 26 ++++++------------- src/BaseSpacePy/api/BaseSpaceAPI.py | 9 ------- src/setup.cfg | 1 - src/setup.py | 10 +------ 12 files changed, 21 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index 07f38c6..351a126 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Mauricio Varea REQUIREMENTS ========================================= -Python 2.6 with the packages 'pycurl', and 'python-dateutil' installed. You can install these on Ubuntu with 'apt-get install python-pycurl' and 'apt-get install python-dateutil'. +Python 2.6 with the package 'python-dateutil' installed. You can install these on Ubuntu with 'apt-get install python-dateutil'. INSTALL diff --git a/doc/_update_doc/Getting Started.txt b/doc/_update_doc/Getting Started.txt index d71eb83..654e28f 100644 --- a/doc/_update_doc/Getting Started.txt +++ b/doc/_update_doc/Getting Started.txt @@ -25,7 +25,7 @@ Version 0.1 of ``BaseSpacePy`` can be checked out here: Setup ################### -*Requirements:* Python 2.6 with the packages 'urllib2', 'pycurl', 'multiprocessing' and 'shutil' available. +*Requirements:* Python 2.6 with the packages 'urllib2', 'multiprocessing' and 'shutil' available. The multi-part file upload will currently only run on a unix setup. diff --git a/doc/html/Getting Started.html b/doc/html/Getting Started.html index 522ba9a..69388ca 100644 --- a/doc/html/Getting Started.html +++ b/doc/html/Getting Started.html @@ -67,7 +67,7 @@

Availability

Setup¶

-

Requirements: Python 2.6 with the packages ‘urllib2’, ‘pycurl’, ‘multiprocessing’ and ‘shutil’ available.

+

Requirements: Python 2.6 with the packages ‘urllib2’, ‘multiprocessing’ and ‘shutil’ available.

The multi-part file upload will currently only run on a unix setup.

To install ‘BaseSpacePy’ run the ‘setup.py’ script in the src directory (for a global install you will need to run this command with root privileges):

cd basespace-python-sdk/src
diff --git a/doc/html/_modules/BaseSpacePy/api/BaseSpaceAPI.html b/doc/html/_modules/BaseSpacePy/api/BaseSpaceAPI.html
index e2006c6..cbf5f83 100644
--- a/doc/html/_modules/BaseSpacePy/api/BaseSpaceAPI.html
+++ b/doc/html/_modules/BaseSpacePy/api/BaseSpaceAPI.html
@@ -50,7 +50,6 @@ 
Source code for BaseSpacePy.api.BaseSpaceAPI
import urllib2
 import shutil
 import urllib
-import pycurl
 import httplib
 import cStringIO
 import json
@@ -252,9 +251,9 @@ 
Source code for BaseSpacePy.api.BaseSpaceAPI
resourcePath = self.apiClient.apiServer + '/appsessions/{AppSessionId}'        
         resourcePath = resourcePath.replace('{AppSessionId}', Id)        
         response = cStringIO.StringIO()
-        c = pycurl.Curl()
-        c.setopt(pycurl.URL, resourcePath)
-        c.setopt(pycurl.USERPWD, self.key + ":" + self.secret)
+        c = .Curl()
+        c.setopt(.URL, resourcePath)
+        c.setopt(.USERPWD, self.key + ":" + self.secret)
         c.setopt(c.WRITEFUNCTION, response.write)
         c.perform()
         c.close()
diff --git a/doc/html/_sources/Getting Started.txt b/doc/html/_sources/Getting Started.txt
index d71eb83..654e28f 100644
--- a/doc/html/_sources/Getting Started.txt	
+++ b/doc/html/_sources/Getting Started.txt	
@@ -25,7 +25,7 @@ Version 0.1 of ``BaseSpacePy`` can be checked out here:
 Setup
 ###################
 
-*Requirements:* Python 2.6 with the packages 'urllib2', 'pycurl', 'multiprocessing' and 'shutil' available.
+*Requirements:* Python 2.6 with the packages 'urllib2', 'multiprocessing' and 'shutil' available.
 
 The multi-part file upload will currently only run on a unix setup.
 
diff --git a/doc/html/searchindex.js b/doc/html/searchindex.js
index d240da6..b6533b2 100644
--- a/doc/html/searchindex.js
+++ b/doc/html/searchindex.js
@@ -1 +1 @@
-Search.setIndex({objects:{"BaseSpacePy.model.File.File":{getIntervalCoverage:[2,0,1,""],isValidFileOption:[2,0,1,""],isInit:[2,0,1,""],downloadFile:[2,0,1,""],getVariantMeta:[2,0,1,""],getCoverageMeta:[2,0,1,""],getFileUrl:[2,0,1,""],filterVariant:[2,0,1,""],getFileS3metadata:[2,0,1,""]},"BaseSpacePy.model.QueryParameters":{QueryParameters:[2,1,1,""]},"BaseSpacePy.model.Run.Run":{isInit:[2,0,1,""],getFiles:[2,0,1,""],getSamples:[2,0,1,""],getAccessStr:[2,0,1,""]},"BaseSpacePy.api.BaseSpaceAPI.BaseSpaceAPI":{getFileById:[2,0,1,""],getAppResultFilesById:[2,0,1,""],getRunFilesById:[2,0,1,""],getUserById:[2,0,1,""],getProjectById:[2,0,1,""],filterVariantSet:[2,0,1,""],getIntervalCoverage:[2,0,1,""],getAppSession:[2,0,1,""],getAppSessionById:[2,0,1,""],getAccessibleRunsByUser:[2,0,1,""],getGenomeById:[2,0,1,""],setAppSessionState:[2,0,1,""],getWebVerificationCode:[2,0,1,""],createAppResult:[2,0,1,""],getSamplePropertiesById:[2,0,1,""],getAppSessionPropertiesById:[2,0,1,""],obtainAccessToken:[2,0,1,""],getAppSessionPropertyByName:[2,0,1,""],multipartFileDownload:[2,0,1,""],multipartFileUpload:[2,0,1,""],updatePrivileges:[2,0,1,""],getAppResultsByProject:[2,0,1,""],getAppResultFiles:[2,0,1,""],getProjectPropertiesById:[2,0,1,""],getVariantMetadata:[2,0,1,""],getRunById:[2,0,1,""],createProject:[2,0,1,""],getFilePropertiesById:[2,0,1,""],getAccess:[2,0,1,""],getFilesBySample:[2,0,1,""],getAppSessionInputsById:[2,0,1,""],getVerificationCode:[2,0,1,""],getAvailableGenomes:[2,0,1,""],getSampleById:[2,0,1,""],appResultFileUpload:[2,0,1,""],fileS3metadata:[2,0,1,""],fileDownload:[2,0,1,""],getSamplesByProject:[2,0,1,""],getCoverageMetaInfo:[2,0,1,""],getProjectByUser:[2,0,1,""],getSampleFilesById:[2,0,1,""],getAppResultById:[2,0,1,""],getRunSamplesById:[2,0,1,""],fileUrl:[2,0,1,""],getAppResultPropertiesById:[2,0,1,""],getRunPropertiesById:[2,0,1,""]},"BaseSpacePy.model.AppSession":{AppSession:[2,1,1,""]},"BaseSpacePy.model.QueryParameters.QueryParameters":{validate:[2,0,1,""]},"BaseSpacePy.model.Project":{Project:[2,1,1,""]},"BaseSpacePy.model.Sample":{Sample:[2,1,1,""]},"BaseSpacePy.model.AppResult.AppResult":{getFiles:[2,0,1,""],uploadFile:[2,0,1,""],isInit:[2,0,1,""],getReferencedSamples:[2,0,1,""],getAccessStr:[2,0,1,""],getReferencedSamplesIds:[2,0,1,""]},"BaseSpacePy.model.Run":{Run:[2,1,1,""]},"BaseSpacePy.api.BaseSpaceAPI":{BaseSpaceAPI:[2,1,1,""]},"BaseSpacePy.model.AppResult":{AppResult:[2,1,1,""]},"BaseSpacePy.model.Sample.Sample":{isInit:[2,0,1,""],getFiles:[2,0,1,""],getAccessStr:[2,0,1,""],getReferencedAppResults:[2,0,1,""]},"BaseSpacePy.model.Project.Project":{isInit:[2,0,1,""],getAppResults:[2,0,1,""],getAccessStr:[2,0,1,""],createAppResult:[2,0,1,""],getSamples:[2,0,1,""]},"BaseSpacePy.model.File":{File:[2,1,1,""]}},terms:{all:[0,2],code:[0,2],queri:[0,3,2],global:[0,3],getfilesbysampl:2,nwe:0,prefix:0,sleep:0,follow:0,getproject:0,depend:2,basespacepy_vx:0,getrunsamplesbyid:2,texliv:1,send:2,hrefcoverag:2,granular:2,present:2,sourc:2,string:[0,2],accesstoken:[0,2],fals:2,account:0,getfilepropertiesbyid:2,myprojects2:0,veri:[0,2],testfile2:0,brows:[0,3,2],getprojectbyid:[0,2],getappresultfilesbyid:2,contenttyp:2,level:2,list:[0,2],upload:[0,3,2],"try":0,item:0,getgenomebyid:[0,2],verif:[0,2],small:2,refer:2,round:2,dir:0,pleas:0,work:0,second:0,pass:2,download:[0,2],further:[0,2],cat:2,append:0,even:0,index:3,filedownload:2,compar:0,neg:2,section:0,abl:0,hrefvari:2,current:[0,2],delet:1,version:[0,1,2],"new":[0,1,2],method:[0,1,2],metadata:2,abov:0,gener:[0,1,2],here:0,shouldn:2,varmeta:0,let:0,ubuntu:0,basespaceauth:0,path:[0,2],getappsessioninputsbyid:2,sinc:1,valu:[0,2],search:3,queur:0,larger:2,prior:0,isinit:2,tauru:0,action:0,implement:2,chrchr2:0,getaccesstoken:0,privilig:0,extra:1,app:[0,2],apt:1,deprec:2,unix:[0,2],api:[0,3,2],getappsessionbyid:2,instal:[0,1],txt:[0,1],"2x26":0,cloud:0,from:[0,2],rattu:0,commun:2,visit:0,two:0,coverag:[0,2],next:0,websit:0,multipartfiledownload:2,call:[0,2],recommend:1,scope:[0,2],type:[0,2],nrun:0,more:[0,2],sort:[0,2],oauthexcept:2,desir:2,relat:[0,2],notic:0,granttyp:2,known:2,actual:2,hiseq:0,partsiz:2,must:[0,2],none:2,retriev:[0,2],pycurl:0,local:2,setup:[0,3],launch:0,getlaunchtyp:0,getfil:[0,2],getsamplepropertiesbyid:2,apporpri:0,endor:0,purpos:0,root:[0,2],norvegicu:0,nearest:2,prompt:0,stream:2,give:0,process:[0,2],accept:2,abort:2,want:0,sought:0,unknownparameterexcept:2,getrun:0,end:[0,2],applaunch:0,anoth:0,write:[0,2],how:0,regist:0,updat:[0,1,2],files3metadata:2,referenc:2,max:[0,2],clone:0,timedout:2,variant:[0,2],befor:0,mai:[0,1,2],associ:[0,2],parallel:2,demonstr:0,getfilebyid:[0,2],"short":0,attempt:2,chr2:[0,2],correspond:2,element:2,inform:[0,2],environ:0,thaliana:0,morten:0,varianthead:[0,2],mygenom:0,authorization_cod:2,help:[0,1],over:2,appsessionid:2,through:[0,2],coli:0,paramet:[0,2],alten:0,binari:2,getappresultfilebyid:2,getwebverificationcod:2,window:2,singleproject:0,pythonpath:0,local_dir:2,sapien:0,main:2,non:2,"return":2,thei:2,s_g1_l001_r2_001:0,python:[0,1],auto:1,cover:0,initi:[0,2],mybam:0,mybasespaceapi:0,now:0,introduct:[0,3],fontx:1,multiprocess:0,name:[0,2],edit:1,authent:[0,2],easili:0,token:[0,3,2],mode:2,timeout:2,genom:[0,2],debug:2,fulli:2,mean:[0,2],status:2,getreferencedappresult:2,chunk:2,hard:0,procedur:0,meta:0,expect:0,our:0,special:0,out:0,variabl:2,vcf:[0,3,2],newli:[0,1,2],getreferencedsamplesid:2,querypar:2,content:[1,3,2],uploadfil:[0,2],getprojectbyus:[0,2],etag:2,print:0,model:[3,2],after:[0,2],insid:2,situat:2,plex:0,triggerobj:0,base:[0,1,2],dictionari:2,needsattent:2,org:1,"byte":2,md5:2,thread:2,doctre:1,musculu:0,filter:[0,2],place:[0,2],isn:2,nthese:0,first:0,rang:2,directli:0,onc:[0,2],propertylist:2,number:[0,2],yourself:0,restrict:2,alreadi:[0,2],done:0,triggertyp:0,miss:2,primari:0,getfiles3metadata:2,size:2,createbspath:2,given:2,script:0,data:[0,3,2],top:0,system:2,store:2,option:[0,2],urllib2:0,specifi:[0,2],getappsesss:2,github:0,accompani:0,staphylococcu:0,than:2,endpo:2,downloadfil:[0,2],instanc:[0,2],provid:[0,2],remov:1,tree:[0,3],project:[0,3,2],str:0,posit:[0,2],multipart:2,comput:0,ana:0,fastq:0,filtervariantset:2,argument:2,client_kei:0,myproject:0,packag:0,bacillu:0,manner:0,have:[0,1,2],tabl:3,need:[0,1,2],getfileurl:2,illegalparameterexcept:2,amplicon:0,startpo:2,getrunbyid:2,getintervalcoverag:[0,2],client:[0,2],note:[0,2],also:0,chromosom:[0,2],exampl:[0,2],take:0,indic:3,singl:2,sure:[0,1],modelnotinitializedexcept:2,test:[0,2],object:[0,2],getverificationcod:[0,2],fileurl:2,"class":[0,1,2],latex:1,getsampl:[0,2],url:2,doc:1,request:[0,3,2],obtainaccesstoken:2,v1pre2:0,part:[0,2],getappresultbyid:2,getrunpropertiesbyid:2,phix:0,getrunfilesbyid:2,show:0,text:[0,2],filetyp:2,session:0,saccharomyc:0,permiss:0,redirecturl:2,cereu:0,redirect:2,access:[0,3,2],onli:[0,2],locat:0,createappresult:2,illumina:0,multivaluepropertyappresultslist:2,getappsess:2,transact:0,solut:0,state:[0,2],haven:0,dict:2,analyz:0,folder:[0,2],analys:0,get:[0,1,2,3],familiar:0,stop:2,getsamplebyid:2,chrom:2,gen:0,requir:[0,2],accesstyp:2,enabl:0,undefinedparameterexcept:2,acut:2,remot:2,allgenom:0,gran:0,privileg:0,grab:0,bam:[0,3,2],basespaceapi:[0,2],summari:[0,2],set:2,see:[0,1],result:[0,3,2],respons:2,fail:2,subject:0,statu:[0,2],getappsessionpropertybynam:2,getaccessiblerunsbyus:2,appresult:[3,2],favor:2,written:1,between:0,"import":0,attribut:2,altern:0,kei:[0,2],buckets:0,filtervari:[0,2],popul:2,verification_with_code_uri:0,last:0,cov:[0,2],samplecount:0,region:2,equal:2,createproject:2,etc:2,redirect_uri:2,pdf:1,com:0,getcoveragemeta:[0,2],nsome:0,simpli:0,coveragemetadata:2,can:[0,2],instanti:0,clientsecret:2,applicationact:0,header:2,empti:2,suppli:0,getappresultpropertiesbyid:2,assum:0,devic:[0,2],due:2,been:[0,2],getaccessstr:[0,2],secret:[0,2],trigger:[0,3],rhodobact:0,interest:2,basic:0,addit:0,clientkei:2,getprojectpropertiesbyid:2,getsamplesbyproject:2,getcoveragemetainfo:2,coordin:2,cerevisia:0,repres:2,those:2,"case":2,multi:[0,2],look:0,access_token:0,plain:[0,2],align:2,properti:2,basespac:[0,2],coveragemeta:0,"while":2,na18507:0,myvcf:0,error:2,getappresultfil:2,setappsessionst:2,howev:2,loop:0,getavailablegenom:[0,2],file:[0,3,2],site:0,getappresult:2,myapi:0,basespacetestfil:0,s_g1_l001_r2_002:0,descript:[0,2],shutil:0,multipartfileupload:2,updateprivileg:2,par:2,disabl:2,develop:[0,1],sphaeroid:0,grant:[0,2],getapptrigg:0,make:[0,1,2],belong:0,createbsdir:2,same:[0,2],setstatu:0,handl:0,mkallberg:0,html:1,"2x151":0,document:[0,1],latexpdf:1,complet:[0,2],byterang:2,http:[0,1],resequenc:0,rais:2,temporari:2,user:[0,2],client_secret:0,appsess:[3,2],chang:[0,2],expand:0,bsauth:0,well:0,without:2,command:0,thi:[0,1,2],everyth:0,identifi:0,paus:0,sample_3:0,getvariantmetadata:2,obtain:0,rest:[0,2],sample_2:0,sample_1:0,human:0,outlin:0,yet:2,web:[0,2],cut:0,easi:0,getanalys:0,point:0,except:2,device_cod:0,add:[0,1],other:2,input:2,modul:[1,3,2],match:2,applic:[0,3,2],appresultfileupload:2,which:2,format:2,read:[0,2],piec:0,getvariantmeta:[0,2],isvalidfileopt:2,getappsessionpropertiesbyid:2,desc:2,applicationactionid:0,specif:[0,2],success:2,filenam:2,should:2,server:[0,2],href:0,necessari:0,either:[0,2],localdir:2,output:0,page:3,interv:[0,2],some:0,localpath:2,intern:2,homo:0,sampl:[0,3,2],analysisfil:0,aureu:0,octet:2,achiev:0,per:2,ucsc:0,larg:2,basespacepi:[0,1,2,3],approv:[0,2],getsamplefilesbyid:2,who:0,run:[0,1,2,3],ntype:0,nmy:0,step:0,apiserv:[0,2],bolt:0,src:0,about:[0,2],obj:2,getuserbyid:[0,2],genomev1:2,hg19:0,getaccess:2,produc:0,own:0,analysis2:0,within:[0,2],automat:0,mydir:0,s_g1_l001_r1_001:0,s_g1_l001_r1_002:0,your:0,getbasespaceapi:0,git:0,byterangeexcept:2,transfer:2,support:2,json:2,custom:2,avail:[0,1,2,3],start:[0,3,2],covmeta:0,includ:[0,2],"var":0,modelnotsupportedexcept:2,individu:2,analysi:[0,3],properli:[1,2],form:2,escherichia:0,projectlist:2,yourproject:0,link:1,oauth:0,"true":[0,2],sdk:[0,1],info:0,made:[0,2],arabidopsi:0,temp:2,possibl:2,"default":2,wish:[0,1,2],maximum:2,record:2,below:0,statussummari:2,processcount:2,"export":0,getappresultsbyproject:2,basespaceurl:0,displaynam:0,creat:[0,1,2,3],flow:0,uri:[0,2],exist:[1,2],kallberg:0,ing:2,check:0,fill:0,again:0,titl:1,when:[0,2],detail:0,invalid:2,field:2,valid:[0,2],tempdir:2,you:[0,1,2],nthe:0,nproject:0,ecoli:0,clean:1,sequenc:[0,2],nafter:0,docstr:1,ngenom:0,log:0,getreferencedsampl:2,sphinx:1,faster:0,directori:[0,1,2],deviceinfo:0,ignor:2,createanalysi:0,time:0,queryparamet:[3,2],profil:2},objtypes:{"0":"py:method","1":"py:class"},titles:["Getting Started","<no title>","Available modules","BaseSpacePy"],objnames:{"0":["py","method","Python method"],"1":["py","class","Python class"]},filenames:["Getting Started","README","Available modules","index"]})
\ No newline at end of file
+Search.setIndex({objects:{"BaseSpacePy.model.File.File":{getIntervalCoverage:[2,0,1,""],isValidFileOption:[2,0,1,""],isInit:[2,0,1,""],downloadFile:[2,0,1,""],getVariantMeta:[2,0,1,""],getCoverageMeta:[2,0,1,""],getFileUrl:[2,0,1,""],filterVariant:[2,0,1,""],getFileS3metadata:[2,0,1,""]},"BaseSpacePy.model.QueryParameters":{QueryParameters:[2,1,1,""]},"BaseSpacePy.model.Run.Run":{isInit:[2,0,1,""],getFiles:[2,0,1,""],getSamples:[2,0,1,""],getAccessStr:[2,0,1,""]},"BaseSpacePy.api.BaseSpaceAPI.BaseSpaceAPI":{getFileById:[2,0,1,""],getAppResultFilesById:[2,0,1,""],getRunFilesById:[2,0,1,""],getUserById:[2,0,1,""],getProjectById:[2,0,1,""],filterVariantSet:[2,0,1,""],getIntervalCoverage:[2,0,1,""],getAppSession:[2,0,1,""],getAppSessionById:[2,0,1,""],getAccessibleRunsByUser:[2,0,1,""],getGenomeById:[2,0,1,""],setAppSessionState:[2,0,1,""],getWebVerificationCode:[2,0,1,""],createAppResult:[2,0,1,""],getSamplePropertiesById:[2,0,1,""],getAppSessionPropertiesById:[2,0,1,""],obtainAccessToken:[2,0,1,""],getAppSessionPropertyByName:[2,0,1,""],multipartFileDownload:[2,0,1,""],multipartFileUpload:[2,0,1,""],updatePrivileges:[2,0,1,""],getAppResultsByProject:[2,0,1,""],getAppResultFiles:[2,0,1,""],getProjectPropertiesById:[2,0,1,""],getVariantMetadata:[2,0,1,""],getRunById:[2,0,1,""],createProject:[2,0,1,""],getFilePropertiesById:[2,0,1,""],getAccess:[2,0,1,""],getFilesBySample:[2,0,1,""],getAppSessionInputsById:[2,0,1,""],getVerificationCode:[2,0,1,""],getAvailableGenomes:[2,0,1,""],getSampleById:[2,0,1,""],appResultFileUpload:[2,0,1,""],fileS3metadata:[2,0,1,""],fileDownload:[2,0,1,""],getSamplesByProject:[2,0,1,""],getCoverageMetaInfo:[2,0,1,""],getProjectByUser:[2,0,1,""],getSampleFilesById:[2,0,1,""],getAppResultById:[2,0,1,""],getRunSamplesById:[2,0,1,""],fileUrl:[2,0,1,""],getAppResultPropertiesById:[2,0,1,""],getRunPropertiesById:[2,0,1,""]},"BaseSpacePy.model.AppSession":{AppSession:[2,1,1,""]},"BaseSpacePy.model.QueryParameters.QueryParameters":{validate:[2,0,1,""]},"BaseSpacePy.model.Project":{Project:[2,1,1,""]},"BaseSpacePy.model.Sample":{Sample:[2,1,1,""]},"BaseSpacePy.model.AppResult.AppResult":{getFiles:[2,0,1,""],uploadFile:[2,0,1,""],isInit:[2,0,1,""],getReferencedSamples:[2,0,1,""],getAccessStr:[2,0,1,""],getReferencedSamplesIds:[2,0,1,""]},"BaseSpacePy.model.Run":{Run:[2,1,1,""]},"BaseSpacePy.api.BaseSpaceAPI":{BaseSpaceAPI:[2,1,1,""]},"BaseSpacePy.model.AppResult":{AppResult:[2,1,1,""]},"BaseSpacePy.model.Sample.Sample":{isInit:[2,0,1,""],getFiles:[2,0,1,""],getAccessStr:[2,0,1,""],getReferencedAppResults:[2,0,1,""]},"BaseSpacePy.model.Project.Project":{isInit:[2,0,1,""],getAppResults:[2,0,1,""],getAccessStr:[2,0,1,""],createAppResult:[2,0,1,""],getSamples:[2,0,1,""]},"BaseSpacePy.model.File":{File:[2,1,1,""]}},terms:{all:[0,2],code:[0,2],queri:[0,3,2],global:[0,3],getfilesbysampl:2,nwe:0,prefix:0,sleep:0,follow:0,getproject:0,depend:2,basespacepy_vx:0,getrunsamplesbyid:2,texliv:1,send:2,hrefcoverag:2,granular:2,present:2,sourc:2,string:[0,2],accesstoken:[0,2],fals:2,account:0,getfilepropertiesbyid:2,myprojects2:0,veri:[0,2],testfile2:0,brows:[0,3,2],getprojectbyid:[0,2],getappresultfilesbyid:2,contenttyp:2,level:2,list:[0,2],upload:[0,3,2],"try":0,item:0,getgenomebyid:[0,2],verif:[0,2],small:2,refer:2,round:2,dir:0,pleas:0,work:0,second:0,pass:2,download:[0,2],further:[0,2],cat:2,append:0,even:0,index:3,filedownload:2,compar:0,neg:2,section:0,abl:0,hrefvari:2,current:[0,2],delet:1,version:[0,1,2],"new":[0,1,2],method:[0,1,2],metadata:2,abov:0,gener:[0,1,2],here:0,shouldn:2,varmeta:0,let:0,ubuntu:0,basespaceauth:0,path:[0,2],getappsessioninputsbyid:2,sinc:1,valu:[0,2],search:3,queur:0,larger:2,prior:0,isinit:2,tauru:0,action:0,implement:2,chrchr2:0,getaccesstoken:0,privilig:0,extra:1,app:[0,2],apt:1,deprec:2,unix:[0,2],api:[0,3,2],getappsessionbyid:2,instal:[0,1],txt:[0,1],"2x26":0,cloud:0,from:[0,2],rattu:0,commun:2,visit:0,two:0,coverag:[0,2],next:0,websit:0,multipartfiledownload:2,call:[0,2],recommend:1,scope:[0,2],type:[0,2],nrun:0,more:[0,2],sort:[0,2],oauthexcept:2,desir:2,relat:[0,2],notic:0,granttyp:2,known:2,actual:2,hiseq:0,partsiz:2,must:[0,2],none:2,retriev:[0,2],local:2,setup:[0,3],launch:0,getlaunchtyp:0,getfil:[0,2],getsamplepropertiesbyid:2,apporpri:0,endor:0,purpos:0,root:[0,2],norvegicu:0,nearest:2,prompt:0,stream:2,give:0,process:[0,2],accept:2,abort:2,want:0,sought:0,unknownparameterexcept:2,getrun:0,end:[0,2],applaunch:0,anoth:0,write:[0,2],how:0,regist:0,updat:[0,1,2],files3metadata:2,referenc:2,max:[0,2],clone:0,timedout:2,variant:[0,2],befor:0,mai:[0,1,2],associ:[0,2],parallel:2,demonstr:0,getfilebyid:[0,2],"short":0,attempt:2,chr2:[0,2],correspond:2,element:2,inform:[0,2],environ:0,thaliana:0,morten:0,varianthead:[0,2],mygenom:0,authorization_cod:2,help:[0,1],over:2,appsessionid:2,through:[0,2],coli:0,paramet:[0,2],alten:0,binari:2,getappresultfilebyid:2,getwebverificationcod:2,window:2,singleproject:0,pythonpath:0,local_dir:2,sapien:0,main:2,non:2,"return":2,thei:2,s_g1_l001_r2_001:0,python:[0,1],auto:1,cover:0,initi:[0,2],mybam:0,mybasespaceapi:0,now:0,introduct:[0,3],fontx:1,multiprocess:0,name:[0,2],edit:1,authent:[0,2],easili:0,token:[0,3,2],mode:2,timeout:2,genom:[0,2],debug:2,fulli:2,mean:[0,2],status:2,getreferencedappresult:2,chunk:2,hard:0,procedur:0,meta:0,expect:0,our:0,special:0,out:0,variabl:2,vcf:[0,3,2],newli:[0,1,2],getreferencedsamplesid:2,querypar:2,content:[1,3,2],uploadfil:[0,2],getprojectbyus:[0,2],etag:2,print:0,model:[3,2],after:[0,2],insid:2,situat:2,plex:0,triggerobj:0,base:[0,1,2],dictionari:2,needsattent:2,org:1,"byte":2,md5:2,thread:2,doctre:1,musculu:0,filter:[0,2],place:[0,2],isn:2,nthese:0,first:0,rang:2,directli:0,onc:[0,2],propertylist:2,number:[0,2],yourself:0,restrict:2,alreadi:[0,2],done:0,triggertyp:0,miss:2,primari:0,getfiles3metadata:2,size:2,createbspath:2,given:2,script:0,data:[0,3,2],top:0,system:2,store:2,option:[0,2],urllib2:0,specifi:[0,2],getappsesss:2,github:0,accompani:0,staphylococcu:0,than:2,endpo:2,downloadfil:[0,2],instanc:[0,2],provid:[0,2],remov:1,tree:[0,3],project:[0,3,2],str:0,posit:[0,2],multipart:2,comput:0,ana:0,fastq:0,filtervariantset:2,argument:2,client_kei:0,myproject:0,packag:0,bacillu:0,manner:0,have:[0,1,2],tabl:3,need:[0,1,2],getfileurl:2,illegalparameterexcept:2,amplicon:0,startpo:2,getrunbyid:2,getintervalcoverag:[0,2],client:[0,2],note:[0,2],also:0,chromosom:[0,2],exampl:[0,2],take:0,indic:3,singl:2,sure:[0,1],modelnotinitializedexcept:2,test:[0,2],object:[0,2],getverificationcod:[0,2],fileurl:2,"class":[0,1,2],latex:1,getsampl:[0,2],url:2,doc:1,request:[0,3,2],obtainaccesstoken:2,v1pre2:0,part:[0,2],getappresultbyid:2,getrunpropertiesbyid:2,phix:0,getrunfilesbyid:2,show:0,text:[0,2],filetyp:2,session:0,saccharomyc:0,permiss:0,redirecturl:2,cereu:0,redirect:2,access:[0,3,2],onli:[0,2],locat:0,createappresult:2,illumina:0,multivaluepropertyappresultslist:2,getappsess:2,transact:0,solut:0,state:[0,2],haven:0,dict:2,analyz:0,folder:[0,2],analys:0,get:[0,1,2,3],familiar:0,stop:2,getsamplebyid:2,chrom:2,gen:0,requir:[0,2],accesstyp:2,enabl:0,undefinedparameterexcept:2,acut:2,remot:2,allgenom:0,gran:0,privileg:0,grab:0,bam:[0,3,2],basespaceapi:[0,2],summari:[0,2],set:2,see:[0,1],result:[0,3,2],respons:2,fail:2,subject:0,statu:[0,2],getappsessionpropertybynam:2,getaccessiblerunsbyus:2,appresult:[3,2],favor:2,written:1,between:0,"import":0,attribut:2,altern:0,kei:[0,2],buckets:0,filtervari:[0,2],popul:2,verification_with_code_uri:0,last:0,cov:[0,2],samplecount:0,region:2,equal:2,createproject:2,etc:2,redirect_uri:2,pdf:1,com:0,getcoveragemeta:[0,2],nsome:0,simpli:0,coveragemetadata:2,can:[0,2],instanti:0,clientsecret:2,applicationact:0,header:2,empti:2,suppli:0,getappresultpropertiesbyid:2,assum:0,devic:[0,2],due:2,been:[0,2],getaccessstr:[0,2],secret:[0,2],trigger:[0,3],rhodobact:0,interest:2,basic:0,addit:0,clientkei:2,getprojectpropertiesbyid:2,getsamplesbyproject:2,getcoveragemetainfo:2,coordin:2,cerevisia:0,repres:2,those:2,"case":2,multi:[0,2],look:0,access_token:0,plain:[0,2],align:2,properti:2,basespac:[0,2],coveragemeta:0,"while":2,na18507:0,myvcf:0,error:2,getappresultfil:2,setappsessionst:2,howev:2,loop:0,getavailablegenom:[0,2],file:[0,3,2],site:0,getappresult:2,myapi:0,basespacetestfil:0,s_g1_l001_r2_002:0,descript:[0,2],shutil:0,multipartfileupload:2,updateprivileg:2,par:2,disabl:2,develop:[0,1],sphaeroid:0,grant:[0,2],getapptrigg:0,make:[0,1,2],belong:0,createbsdir:2,same:[0,2],setstatu:0,handl:0,mkallberg:0,html:1,"2x151":0,document:[0,1],latexpdf:1,complet:[0,2],byterang:2,http:[0,1],resequenc:0,rais:2,temporari:2,user:[0,2],client_secret:0,appsess:[3,2],chang:[0,2],expand:0,bsauth:0,well:0,without:2,command:0,thi:[0,1,2],everyth:0,identifi:0,paus:0,sample_3:0,getvariantmetadata:2,obtain:0,rest:[0,2],sample_2:0,sample_1:0,human:0,outlin:0,yet:2,web:[0,2],cut:0,easi:0,getanalys:0,point:0,except:2,device_cod:0,add:[0,1],other:2,input:2,modul:[1,3,2],match:2,applic:[0,3,2],appresultfileupload:2,which:2,format:2,read:[0,2],piec:0,getvariantmeta:[0,2],isvalidfileopt:2,getappsessionpropertiesbyid:2,desc:2,applicationactionid:0,specif:[0,2],success:2,filenam:2,should:2,server:[0,2],href:0,necessari:0,either:[0,2],localdir:2,output:0,page:3,interv:[0,2],some:0,localpath:2,intern:2,homo:0,sampl:[0,3,2],analysisfil:0,aureu:0,octet:2,achiev:0,per:2,ucsc:0,larg:2,basespacepi:[0,1,2,3],approv:[0,2],getsamplefilesbyid:2,who:0,run:[0,1,2,3],ntype:0,nmy:0,step:0,apiserv:[0,2],bolt:0,src:0,about:[0,2],obj:2,getuserbyid:[0,2],genomev1:2,hg19:0,getaccess:2,produc:0,own:0,analysis2:0,within:[0,2],automat:0,mydir:0,s_g1_l001_r1_001:0,s_g1_l001_r1_002:0,your:0,getbasespaceapi:0,git:0,byterangeexcept:2,transfer:2,support:2,json:2,custom:2,avail:[0,1,2,3],start:[0,3,2],covmeta:0,includ:[0,2],"var":0,modelnotsupportedexcept:2,individu:2,analysi:[0,3],properli:[1,2],form:2,escherichia:0,projectlist:2,yourproject:0,link:1,oauth:0,"true":[0,2],sdk:[0,1],info:0,made:[0,2],arabidopsi:0,temp:2,possibl:2,"default":2,wish:[0,1,2],maximum:2,record:2,below:0,statussummari:2,processcount:2,"export":0,getappresultsbyproject:2,basespaceurl:0,displaynam:0,creat:[0,1,2,3],flow:0,uri:[0,2],exist:[1,2],kallberg:0,ing:2,check:0,fill:0,again:0,titl:1,when:[0,2],detail:0,invalid:2,field:2,valid:[0,2],tempdir:2,you:[0,1,2],nthe:0,nproject:0,ecoli:0,clean:1,sequenc:[0,2],nafter:0,docstr:1,ngenom:0,log:0,getreferencedsampl:2,sphinx:1,faster:0,directori:[0,1,2],deviceinfo:0,ignor:2,createanalysi:0,time:0,queryparamet:[3,2],profil:2},objtypes:{"0":"py:method","1":"py:class"},titles:["Getting Started","<no title>","Available modules","BaseSpacePy"],objnames:{"0":["py","method","Python method"],"1":["py","class","Python class"]},filenames:["Getting Started","README","Available modules","index"]})
\ No newline at end of file
diff --git a/doc/latex/BaseSpacePy.tex b/doc/latex/BaseSpacePy.tex
index 05ae4ff..b230124 100644
--- a/doc/latex/BaseSpacePy.tex
+++ b/doc/latex/BaseSpacePy.tex
@@ -1894,7 +1894,7 @@ \subsection{Availability}
 
 \subsection{Setup}
 \label{Getting Started:setup}
-\emph{Requirements:} Python 2.6 with the packages `urllib2', `pycurl', `multiprocessing' and `shutil' available.
+\emph{Requirements:} Python 2.6 with the packages `urllib2', `multiprocessing' and `shutil' available.
 
 The multi-part file upload will currently only run on a unix setup.
 
diff --git a/src/BaseSpacePy/api/APIClient.py b/src/BaseSpacePy/api/APIClient.py
index 9c43ff4..d25eea6 100644
--- a/src/BaseSpacePy/api/APIClient.py
+++ b/src/BaseSpacePy/api/APIClient.py
@@ -31,7 +31,7 @@ def __init__(self, AccessToken, apiServerAndVersion, userAgent=None, timeout=10)
 
     def __forcePostCall__(self, resourcePath, postData, headers):
         '''
-        For forcing a REST POST request using pycurl (seems to be used when POSTing with no post data)
+        For forcing a REST POST request (seems to be used when POSTing with no post data)
                 
         :param resourcePath: the url to call, including server address and api version
         :param postData: a dictionary of data to post
@@ -46,20 +46,6 @@ def __forcePostCall__(self, resourcePath, postData, headers):
             pass
         import logging
         logging.getLogger("requests").setLevel(logging.WARNING)
-        # pycurl is hard to get working, so best to cauterise it into only the functions where it is needed
-        # import pycurl
-        # postData = [(p,postData[p]) for p in postData]
-        # headerPrep  = [k + ':' + headers[k] for k in headers.keys()]
-        # response = cStringIO.StringIO()
-        # c = pycurl.Curl()
-        # c.setopt(pycurl.URL,resourcePath + '?' + post)
-        # c.setopt(pycurl.HTTPHEADER, headerPrep)
-        # c.setopt(pycurl.POST, 1)
-        # c.setopt(pycurl.POSTFIELDS, post)
-        # c.setopt(c.WRITEFUNCTION, response.write)
-        # c.perform()
-        # c.close()
-        # return response.getvalue()
         encodedPost =  urllib.urlencode(postData)
         resourcePath = "%s?%s" % (resourcePath, encodedPost)
         response = requests.post(resourcePath, data=json.dumps(postData), headers=headers)
@@ -150,9 +136,9 @@ def callAPI(self, resourcePath, method, queryParams, postData, headerParams=None
                 if data and not len(data): 
                     data='\n' # temp fix, in case is no data in the file, to prevent post request from failing
                 request = urllib2.Request(url=url, headers=headers, data=data)#,timeout=self.timeout)
-            else:                                    # use pycurl to force a post call, even w/o data
+            else:
                 response = self.__forcePostCall__(forcePostUrl, sentQueryParams, headers)
-            if method in ['PUT', 'DELETE']: #urllib doesnt do put and delete, default to pycurl here
+            if method in ['PUT', 'DELETE']:
                 if method == 'DELETE':
                     raise NotImplementedError("DELETE REST API calls aren't currently supported")
                 response = self.__putCall__(url, headers, data)
diff --git a/src/BaseSpacePy/api/BaseAPI.py b/src/BaseSpacePy/api/BaseAPI.py
index f19d96f..dab6b2f 100644
--- a/src/BaseSpacePy/api/BaseAPI.py
+++ b/src/BaseSpacePy/api/BaseAPI.py
@@ -51,7 +51,7 @@ def __singleRequest__(self, myModel, resourcePath, method, queryParams, headerPa
         :param headerParams: a dictionary of header parameters
         :param postData: (optional) data to POST, default None
         :param version: (optional) print detailed output, default False
-        :param forcePost: (optional) use a POST call with pycurl instead of urllib, default False (used only when POSTing with no post data?)
+        :param forcePost: (optional) use a POST call, default False (used only when POSTing with no post data?)
 
         :raises ServerResponseException: if server returns an error or has no response
         :returns: an instance of the Response model from the provided myModel
@@ -149,31 +149,21 @@ def __listRequest__(self, myModel, resourcePath, method, queryParams, headerPara
     def __makeCurlRequest__(self, data, url):
         '''
         Make a curl POST request
-        
+
         :param data: data to post (eg. list of tuples of form (key, value))
         :param url: url to post data to
-        
+
         :raises ServerResponseException: if server returns an error or has no response
         :returns: dictionary of api server response
         '''
-        # pycurl is hard to get working, so best to cauterise it into only the functions where it is needed
-        import pycurl
-        post = urllib.urlencode(data)
-        response = cStringIO.StringIO()
-        c = pycurl.Curl()
-        c.setopt(pycurl.URL,url)
-        c.setopt(pycurl.POST, 1)
-        c.setopt(pycurl.POSTFIELDS, post)
-        c.setopt(c.WRITEFUNCTION, response.write)
-        c.perform()
-        c.close()
-        respVal = response.getvalue()
-        if not respVal:
+        import requests
+        r = requests.post(url, data)
+        if not r:
             raise ServerResponseException("No response from server")
-        obj = json.loads(respVal)
+        obj = json.loads(r.text)
         if obj.has_key('error'):
             raise ServerResponseException(str(obj['error'] + ": " + obj['error_description']))
-        return obj      
+        return obj
 
     def getTimeout(self):
         '''
diff --git a/src/BaseSpacePy/api/BaseSpaceAPI.py b/src/BaseSpacePy/api/BaseSpaceAPI.py
index bb00457..8f9b508 100755
--- a/src/BaseSpacePy/api/BaseSpaceAPI.py
+++ b/src/BaseSpacePy/api/BaseSpaceAPI.py
@@ -180,7 +180,6 @@ def getAppSessionOld(self, Id=None):
         :param Id: an AppSession Id; if not provided, the AppSession Id of the BaseSpaceAPI instance will be used 
         :returns: An AppSession instance                
         '''
-        # pycurl is hard to get working, so best to cauterise it into only the functions where it is needed
         if Id is None:
             Id = self.appSessionId
         if not Id:
@@ -188,14 +187,6 @@ def getAppSessionOld(self, Id=None):
         resourcePath = self.apiClient.apiServerAndVersion + '/appsessions/{AppSessionId}'        
         resourcePath = resourcePath.replace('{AppSessionId}', Id)        
         response = cStringIO.StringIO()
-        # import pycurl
-        # c = pycurl.Curl()
-        # c.setopt(pycurl.URL, resourcePath)
-        # c.setopt(pycurl.USERPWD, self.key + ":" + self.secret)
-        # c.setopt(c.WRITEFUNCTION, response.write)
-        # c.perform()
-        # c.close()
-        # resp_dict = json.loads(response.getvalue())        
         import requests
         response = requests.get(resourcePath, auth=(self.key, self.secret))
         resp_dict = json.loads(response.text)
diff --git a/src/setup.cfg b/src/setup.cfg
index 70be50d..0ac0f47 100644
--- a/src/setup.cfg
+++ b/src/setup.cfg
@@ -1,7 +1,6 @@
 [bdist_rpm]
 
 requires = python >= 2.6
-           python-pycurl
            python-dateutil
            python-requests
 no-autoreq = yes
diff --git a/src/setup.py b/src/setup.py
index bf27bb3..0458d32 100755
--- a/src/setup.py
+++ b/src/setup.py
@@ -33,21 +33,13 @@
       author_email='techsupport@illumina.com',
       packages=['BaseSpacePy.api','BaseSpacePy.model','BaseSpacePy'],
       package_dir={'BaseSpacePy' : os.path.join(os.path.dirname(__file__),'BaseSpacePy')},
-      # this line moves closer to a Python configuration that does not issue the SSLContext warning
-      # it fails because of missing headers when building a dependency
-      #install_requires=['pycurl','python-dateutil','pyOpenSSL>=0.13','requests','requests[security]'],
-      install_requires=['pycurl','python-dateutil','requests'],
-      #setup_requires=['rpm-build','stdeb'],
+      install_requires=['python-dateutil','requests'],
       zip_safe=False,
 )
 
 
 # Warn use if dependent packages aren't installed
 #try:
-#    import pycurl
-#except:
-#    print "WARNING - please install required package 'pycurl'"
-#try:
 #    import dateutil
 #except:
 #    print "WARNING - please install required package 'python-dateutil'"

From 9732e576234bcfb01d33477eff3751b5f23907da Mon Sep 17 00:00:00 2001
From: psaffrey 
Date: Thu, 4 Feb 2016 15:23:20 +0000
Subject: [PATCH 53/73] Derive names for launches based on ID-specified
 appresults

---
 src/BaseSpacePy/api/AppLaunchHelpers.py | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/src/BaseSpacePy/api/AppLaunchHelpers.py b/src/BaseSpacePy/api/AppLaunchHelpers.py
index f5c8c28..76a7327 100644
--- a/src/BaseSpacePy/api/AppLaunchHelpers.py
+++ b/src/BaseSpacePy/api/AppLaunchHelpers.py
@@ -438,6 +438,11 @@ class LaunchPayload(object):
     """
 
     LAUNCH_NAME = "LaunchName"
+    ENTITY_TYPE_TO_METHOD_NAME = {
+        "sample" : "getSampleById",
+        "appresult" : "getAppResultById",
+        "project" : "getProjectById"
+    }
 
     def __init__(self, launch_spec, args, configoptions, api, disable_consistency_checking=True):
         """
@@ -478,8 +483,7 @@ def _find_all_entity_names(self, entity_type):
                     # note we're relying on the regular naming of the API to provide the right method name
                     entry = entry.strip('"')
                     try:
-                        method_name = "get%sById" % entity_type.title()
-                        method = getattr(self._api, method_name)
+                        method = getattr(self._api, self.ENTITY_TYPE_TO_METHOD_NAME[entity_type])
                         entity_names.append(method(entry).Name)
                     except (AttributeError, ServerResponseException):
                         pass
@@ -509,17 +513,12 @@ def is_valid_basespace_id(self, varname, basespace_id):
         """
         This is not really needed if users are specifying inputs as BaseMount paths,
         because in these cases validation happens elsewhere
-
-        To validate other kinds of ID, we should (TODO!) resolve the type based on the varname
-        and use the SDK to look it up.
         """
         vartype = self._launch_spec.get_property_bald_type(varname)
-        if vartype == "Sample":
-            self._api.getSampleById(basespace_id)
-        elif vartype == "Project":
-            self._api.getProjectById(basespace_id)
-        elif vartype == "AppResult":
-            self._api.getAppResultById(basespace_id)
+        flat_vartype = vartype.lower()
+        if flat_vartype in self.ENTITY_TYPE_TO_METHOD_NAME:
+            method = getattr(self._api, self.ENTITY_TYPE_TO_METHOD_NAME[flat_vartype])
+            method(basespace_id)
         else:
             return True
 

From 0c46a6dc9c74b7b6d86c5d2a400cb1679f975f68 Mon Sep 17 00:00:00 2001
From: psaffrey 
Date: Wed, 10 Feb 2016 13:09:28 +0000
Subject: [PATCH 54/73] Added support in applaunchhelpers for properties with
 more nested names to provide support for apps like AmpliconDS and
 tumor/normal

---
 src/BaseSpacePy/api/AppLaunchHelpers.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/BaseSpacePy/api/AppLaunchHelpers.py b/src/BaseSpacePy/api/AppLaunchHelpers.py
index 76a7327..71e9a72 100644
--- a/src/BaseSpacePy/api/AppLaunchHelpers.py
+++ b/src/BaseSpacePy/api/AppLaunchHelpers.py
@@ -38,7 +38,7 @@ class AppSessionMetaData(object):
 
     __metaclass__ = abc.ABCMeta
 
-    SKIP_PROPERTIES = ["app-session-name"]
+    SKIP_PROPERTIES = ["app-session-name", "attributes", "columns", "num-columns", "rowcount", "IsMultiNode"]
 
 
     def __init__(self, appsession_metadata):
@@ -63,8 +63,6 @@ def get_refined_appsession_properties(self):
             property_type = str(self.unpack_bs_property(as_property, "Type"))
             if not property_name.startswith("Input"):
                 continue
-            if property_name.count(".") != 1:
-                continue
             property_basename = property_name.split(".")[-1]
             if property_basename in self.SKIP_PROPERTIES:
                 continue
@@ -184,7 +182,9 @@ def clean_name(parameter_name):
         :param parameter_name: parameter name to clean
         :return: cleaned name
         """
-        prefix, cleaned_name = parameter_name.split(".")
+        split_name = parameter_name.split(".")
+        prefix = split_name[0]
+        cleaned_name = split_name[-1]
         assert prefix == "Input"
         return cleaned_name
 

From cd22f54dc744a81f56f83e7a59ab1c3b2ee5fccb Mon Sep 17 00:00:00 2001
From: psaffrey 
Date: Mon, 15 Feb 2016 14:42:20 +0000
Subject: [PATCH 55/73] Cleaned up handling of BaseMount paths for app launch

---
 src/BaseSpacePy/api/AppLaunchHelpers.py   | 10 ++++++----
 src/BaseSpacePy/api/BaseMountInterface.py |  2 +-
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/BaseSpacePy/api/AppLaunchHelpers.py b/src/BaseSpacePy/api/AppLaunchHelpers.py
index 71e9a72..4078032 100644
--- a/src/BaseSpacePy/api/AppLaunchHelpers.py
+++ b/src/BaseSpacePy/api/AppLaunchHelpers.py
@@ -476,7 +476,9 @@ def _find_all_entity_names(self, entity_type):
                 if not self._launch_spec.is_list_property(varname):
                     arg = [arg]
                 for entry in arg:
-                    if os.path.exists(entry):
+                    # if the argument contains a path separator, it must be a valid BaseMount path
+                    # otherwise, an exception will be raised by BaseMountInterface
+                    if os.path.sep in entry:
                         bmi = BaseMountInterface(entry)
                         entity_names.append(bmi.name)
                     # if this is not a BaseMount path, try to resolve an entity name using the API
@@ -531,10 +533,10 @@ def to_basespace_id(self, param_name, varval):
 
         :return basespaceid
         """
-        if varval.startswith("/") and not os.path.exists(varval):
-            raise LaunchSpecificationException("Parameter looks like a path, but does not exist: %s" % varval)
         spec_type = self._launch_spec.get_property_bald_type(param_name)
-        if os.path.exists(varval):
+        if os.path.sep in varval:
+            # if the argument contains a path separator, it must be a valid BaseMount path
+            # otherwise, an exception will be raised by BaseMountInterface
             bmi = BaseMountInterface(varval)
             # make sure we have a BaseMount access token to compare - old versions won't have one
             # also make sure we've been passed an access token -
diff --git a/src/BaseSpacePy/api/BaseMountInterface.py b/src/BaseSpacePy/api/BaseMountInterface.py
index 35f3fab..199f240 100644
--- a/src/BaseSpacePy/api/BaseMountInterface.py
+++ b/src/BaseSpacePy/api/BaseMountInterface.py
@@ -22,7 +22,7 @@ class BaseMountInterface(object):
     def __init__(self, path):
         if path.endswith(os.sep):
             path = path[:-1]
-        self.path = path
+        self.path = os.path.expanduser(path)
         self.id = None
         self.type = None
         self.access_token = None

From 1a6375ab94ec0a69a21e170281343e625b2ecb59 Mon Sep 17 00:00:00 2001
From: psaffrey 
Date: Mon, 29 Feb 2016 14:15:11 +0000
Subject: [PATCH 56/73] broadened checking of error conditions in
 authentication

---
 src/BaseSpacePy/api/AuthenticationAPI.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/BaseSpacePy/api/AuthenticationAPI.py b/src/BaseSpacePy/api/AuthenticationAPI.py
index 33698e4..f0cd2ab 100644
--- a/src/BaseSpacePy/api/AuthenticationAPI.py
+++ b/src/BaseSpacePy/api/AuthenticationAPI.py
@@ -143,7 +143,7 @@ def set_oauth_details(self, client_id, client_secret, scopes):
             r = s.post(url=TOKEN_URI,
                        data=token_payload)
             if r.status_code == 400:
-                if r.json()["error"] == "access_denied":
+                if r.json()["error"] == "access_denied" or r.json()["error"] == "AccessDenied":
                     sys.stdout.write("\n")
                     break
                 sys.stdout.write(".")

From 6f64ae6abdd4a91ac2c6da3fcd792bfcbbb60c11 Mon Sep 17 00:00:00 2001
From: psaffrey 
Date: Mon, 23 May 2016 17:05:32 +0100
Subject: [PATCH 57/73] Added token information endpoint

---
 src/BaseSpacePy/api/BaseSpaceAPI.py | 31 +++++++++++++++++++++++++++++
 src/BaseSpacePy/model/Token.py      | 10 ++++++++++
 src/BaseSpacePy/model/__init__.py   |  1 +
 3 files changed, 42 insertions(+)
 create mode 100644 src/BaseSpacePy/model/Token.py

diff --git a/src/BaseSpacePy/api/BaseSpaceAPI.py b/src/BaseSpacePy/api/BaseSpaceAPI.py
index 8f9b508..fe03c90 100755
--- a/src/BaseSpacePy/api/BaseSpaceAPI.py
+++ b/src/BaseSpacePy/api/BaseSpaceAPI.py
@@ -14,6 +14,7 @@
 import urlparse
 import logging
 import getpass
+import requests
 
 from BaseSpacePy.api.APIClient import APIClient
 from BaseSpacePy.api.BaseAPI import BaseAPI
@@ -377,6 +378,36 @@ def obtainAccessToken(self, code, grantType='device', redirect_uri=None):
         resp_dict = self.__makeCurlRequest__(data, self.apiClient.apiServerAndVersion + tokenURL)
         return str(resp_dict['access_token'])
 
+    def getAccessTokenDetails(self):
+        '''
+        Because this does not use the standard API prefix, this has to be done as a special case
+        :return:
+        '''
+        endpoint = self.apiClient.apiServerAndVersion + tokenURL + "/current"
+
+        print endpoint
+        args = {
+            "access_token": self.apiClient.apiKey
+        }
+        try:
+            response_raw = requests.get(endpoint, args)
+            response = response_raw.json()
+        except Exception as e:
+            raise ServerResponseException('Could not query access token endpoint: %s' % str(e))
+        if not response:
+            raise ServerResponseException('No response returned')
+        if response.has_key('ResponseStatus'):
+            if response['ResponseStatus'].has_key('ErrorCode'):
+                raise ServerResponseException(str(response['ResponseStatus']['ErrorCode'] + ": " + response['ResponseStatus']['Message']))
+            elif response['ResponseStatus'].has_key('Message'):
+                raise ServerResponseException(str(response['ResponseStatus']['Message']))
+        elif response.has_key('ErrorCode'):
+            raise ServerResponseException(response["MessageFormatted"])
+
+        responseObject = self.apiClient.deserialize(response["Response"], Token.Token)
+        return responseObject
+
+
     def updatePrivileges(self, code, grantType='device', redirect_uri=None):
         '''
         Retrieves a user-specific access token, and sets the token on the current object.
diff --git a/src/BaseSpacePy/model/Token.py b/src/BaseSpacePy/model/Token.py
new file mode 100644
index 0000000..55ea5f9
--- /dev/null
+++ b/src/BaseSpacePy/model/Token.py
@@ -0,0 +1,10 @@
+class Token(object):
+    def __init__(self):
+        self.swaggerTypes = {
+            'Scopes': 'list',
+            'DateCreated': 'datetime',
+            'UserResourceOwner': 'User',
+            'Application': 'Application',
+            'AccessToken': 'str'
+        }
+    
diff --git a/src/BaseSpacePy/model/__init__.py b/src/BaseSpacePy/model/__init__.py
index f78aadf..0f8e44f 100644
--- a/src/BaseSpacePy/model/__init__.py
+++ b/src/BaseSpacePy/model/__init__.py
@@ -1,5 +1,6 @@
 
 __all__= [
+ 'Token',
  'ListResponse',
  'ResponseStatus',
  'File',

From 8960ccea3dfb16864dee832419c0448456616d4b Mon Sep 17 00:00:00 2001
From: psaffrey 
Date: Tue, 24 May 2016 12:11:39 +0100
Subject: [PATCH 58/73] Removed spurious print

---
 src/BaseSpacePy/api/BaseSpaceAPI.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/BaseSpacePy/api/BaseSpaceAPI.py b/src/BaseSpacePy/api/BaseSpaceAPI.py
index fe03c90..099195b 100755
--- a/src/BaseSpacePy/api/BaseSpaceAPI.py
+++ b/src/BaseSpacePy/api/BaseSpaceAPI.py
@@ -385,7 +385,6 @@ def getAccessTokenDetails(self):
         '''
         endpoint = self.apiClient.apiServerAndVersion + tokenURL + "/current"
 
-        print endpoint
         args = {
             "access_token": self.apiClient.apiKey
         }

From df570225020e64e157ec417d6ebd390938febd18 Mon Sep 17 00:00:00 2001
From: psaffrey 
Date: Wed, 6 Jul 2016 14:55:11 +0100
Subject: [PATCH 59/73] Altered model for PropertyMap type

---
 src/BaseSpacePy/model/PropertyMap.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/BaseSpacePy/model/PropertyMap.py b/src/BaseSpacePy/model/PropertyMap.py
index 279bba7..61e55d4 100644
--- a/src/BaseSpacePy/model/PropertyMap.py
+++ b/src/BaseSpacePy/model/PropertyMap.py
@@ -7,7 +7,7 @@ def __init__(self):
             'Href': 'str',
             'Name': 'str',
             'Description': 'str',
-            'Items': 'list',           
+            'Content': 'list',
         }
 
     def __str__(self):

From abdd609ccbad930d015a1cd5c803702fc88e21f5 Mon Sep 17 00:00:00 2001
From: psaffrey 
Date: Thu, 7 Jul 2016 15:03:18 +0100
Subject: [PATCH 60/73] Add app version methods

---
 src/BaseSpacePy/api/AppLaunchHelpers.py | 54 +++++++++++++++++++++----
 1 file changed, 47 insertions(+), 7 deletions(-)

diff --git a/src/BaseSpacePy/api/AppLaunchHelpers.py b/src/BaseSpacePy/api/AppLaunchHelpers.py
index 4078032..38c8008 100644
--- a/src/BaseSpacePy/api/AppLaunchHelpers.py
+++ b/src/BaseSpacePy/api/AppLaunchHelpers.py
@@ -48,6 +48,25 @@ def __init__(self, appsession_metadata):
         """
         self.asm = appsession_metadata
 
+    def _get_all_duplicate_names(self):
+        appsession_properties = self.get_properties()
+        all_names = set()
+        duplicate_names = set()
+        for as_property in appsession_properties:
+            property_name = str(self.unpack_bs_property(as_property, "Name"))
+            if not property_name.startswith("Input"):
+                continue
+            property_basename = property_name.split(".")[-1]
+            if property_basename in self.SKIP_PROPERTIES:
+                continue
+            if property_basename in BS_ENTITY_LIST_NAMES:
+                continue
+            if property_basename in all_names:
+                duplicate_names.add(property_basename)
+            else:
+                all_names.add(property_basename)
+        return duplicate_names
+
     def get_refined_appsession_properties(self):
         """
         Unpacks the properties from an appsession and refines them ready to make a launch specification
@@ -55,6 +74,7 @@ def get_refined_appsession_properties(self):
         :return: properties (list of dict of "Name" and "Type")
                  defaults (dict from property name to default value)
         """
+        all_names = self._get_all_duplicate_names()
         appsession_properties = self.get_properties()
         properties = []
         defaults = {}
@@ -64,6 +84,9 @@ def get_refined_appsession_properties(self):
             if not property_name.startswith("Input"):
                 continue
             property_basename = property_name.split(".")[-1]
+            if property_basename in all_names:
+                property_basename = ".".join(property_name.split(".")[-2:])
+                import pdb; pdb.set_trace()
             if property_basename in self.SKIP_PROPERTIES:
                 continue
             if property_basename in BS_ENTITY_LIST_NAMES:
@@ -172,21 +195,38 @@ class LaunchSpecification(object):
 
     def __init__(self, properties, defaults):
         self.properties = properties
+        self.cleaned_names = {}
         self.property_lookup = dict((self.clean_name(property_["Name"]), property_) for property_ in self.properties)
         self.defaults = defaults
 
-    @staticmethod
-    def clean_name(parameter_name):
+    def clean_name(self, parameter_name):
         """
         strip off the Input. prefix, which is needed by the launch payload but gets in the way otherwise
         :param parameter_name: parameter name to clean
         :return: cleaned name
         """
-        split_name = parameter_name.split(".")
-        prefix = split_name[0]
-        cleaned_name = split_name[-1]
-        assert prefix == "Input"
-        return cleaned_name
+        if not self.cleaned_names:
+            dup_names = set()
+            all_names = set()
+            for property_ in self.properties:
+                split_name = property_["Name"].split(".")
+                name_prefix = split_name[0]
+                assert name_prefix == "Input"
+                name_suffix = split_name[-1]
+                if name_suffix in all_names:
+                    dup_names.add(name_suffix)
+                else:
+                    all_names.add(name_suffix)
+            for property_ in self.properties:
+                full_name = property_["Name"]
+                split_name = full_name.split(".")
+                name_suffix = split_name[-1]
+                if name_suffix in dup_names:
+                    clean_name = ".".join(split_name[-2:])
+                else:
+                    clean_name = split_name[-1]
+                self.cleaned_names[full_name] = clean_name
+        return self.cleaned_names[parameter_name]
 
     def process_parameter(self, param, varname):
         # if option is prefixed with an @, it's a file (or process substitution with <() )

From 1b784fb9c33361dc67e2a7e3035f9238528d0ef4 Mon Sep 17 00:00:00 2001
From: psaffrey 
Date: Fri, 8 Jul 2016 16:36:52 +0100
Subject: [PATCH 61/73] Removed stray debugging line

---
 src/BaseSpacePy/api/AppLaunchHelpers.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/BaseSpacePy/api/AppLaunchHelpers.py b/src/BaseSpacePy/api/AppLaunchHelpers.py
index 38c8008..e985328 100644
--- a/src/BaseSpacePy/api/AppLaunchHelpers.py
+++ b/src/BaseSpacePy/api/AppLaunchHelpers.py
@@ -86,7 +86,6 @@ def get_refined_appsession_properties(self):
             property_basename = property_name.split(".")[-1]
             if property_basename in all_names:
                 property_basename = ".".join(property_name.split(".")[-2:])
-                import pdb; pdb.set_trace()
             if property_basename in self.SKIP_PROPERTIES:
                 continue
             if property_basename in BS_ENTITY_LIST_NAMES:

From df3a4ca596be2b252484ab716b3fbb7df4422b13 Mon Sep 17 00:00:00 2001
From: psaffrey 
Date: Fri, 23 Sep 2016 16:59:45 +0100
Subject: [PATCH 62/73] Fixed bug where LaunchName was not a valid
 configuration variable

---
 src/BaseSpacePy/api/AppLaunchHelpers.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/BaseSpacePy/api/AppLaunchHelpers.py b/src/BaseSpacePy/api/AppLaunchHelpers.py
index e985328..9bdc0eb 100644
--- a/src/BaseSpacePy/api/AppLaunchHelpers.py
+++ b/src/BaseSpacePy/api/AppLaunchHelpers.py
@@ -26,6 +26,7 @@
 BS_ENTITIES = ["sample", "project", "appresult", "file"]
 BS_ENTITY_LIST_NAMES = ["Samples", "Projects", "AppResults", "Files"]
 
+LAUNCH_NAME = "LaunchName"
 
 class AppSessionMetaData(object):
     """
@@ -250,6 +251,8 @@ def resolve_list_variables(self, var_dict):
         """
         for varname in var_dict:
             varval = var_dict[varname]
+            if varname == LAUNCH_NAME:
+                continue
             if self.is_list_property(varname) and not isinstance(varval, list):
                 var_dict[varname] = [varval]
                 # raise AppLaunchException("non-list property specified for list parameter")
@@ -476,7 +479,6 @@ class LaunchPayload(object):
     and mapping BaseMount paths to the API reference strings the launch needs
     """
 
-    LAUNCH_NAME = "LaunchName"
     ENTITY_TYPE_TO_METHOD_NAME = {
         "sample" : "getSampleById",
         "appresult" : "getAppResultById",
@@ -537,8 +539,8 @@ def derive_launch_name(self, app_name):
         :param app_name: name of app
         :return: useful name for app launch
         """
-        if self.LAUNCH_NAME in self._configoptions:
-            return self._configoptions[self.LAUNCH_NAME]
+        if LAUNCH_NAME in self._configoptions:
+            return self._configoptions[LAUNCH_NAME]
         else:
             launch_names = self._find_all_entity_names("sample")
             if not launch_names:

From 9e26ff98957b22e3829108a9500560758c24702c Mon Sep 17 00:00:00 2001
From: psaffrey 
Date: Mon, 26 Sep 2016 10:18:21 +0100
Subject: [PATCH 63/73] Added capability to augment launchname with a batch
 number

---
 src/BaseSpacePy/api/AppLaunchHelpers.py | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/src/BaseSpacePy/api/AppLaunchHelpers.py b/src/BaseSpacePy/api/AppLaunchHelpers.py
index 9bdc0eb..04df2c2 100644
--- a/src/BaseSpacePy/api/AppLaunchHelpers.py
+++ b/src/BaseSpacePy/api/AppLaunchHelpers.py
@@ -27,6 +27,7 @@
 BS_ENTITY_LIST_NAMES = ["Samples", "Projects", "AppResults", "Files"]
 
 LAUNCH_NAME = "LaunchName"
+BATCH_NUMBER = "BatchNumber"
 
 class AppSessionMetaData(object):
     """
@@ -251,7 +252,7 @@ def resolve_list_variables(self, var_dict):
         """
         for varname in var_dict:
             varval = var_dict[varname]
-            if varname == LAUNCH_NAME:
+            if varname == LAUNCH_NAME or varname == BATCH_NUMBER:
                 continue
             if self.is_list_property(varname) and not isinstance(varval, list):
                 var_dict[varname] = [varval]
@@ -431,9 +432,9 @@ def make_launch_json(self, user_supplied_vars, launch_name, api_version, sample_
         if required_vars - supplied_var_names:
             raise LaunchSpecificationException(
                 "Compulsory variable(s) missing! (%s)" % str(required_vars - supplied_var_names))
-        if supplied_var_names - (self.get_variable_requirements() | set(["LaunchName"])):
+        if supplied_var_names - (self.get_variable_requirements() | set([LAUNCH_NAME, BATCH_NUMBER])):
             print "warning! unused variable(s) specified: (%s)" % str(
-                supplied_var_names - self.get_variable_requirements())
+                supplied_var_names - (self.get_variable_requirements() | set([LAUNCH_NAME, BATCH_NUMBER])))
         all_vars = copy.copy(self.defaults)
         all_vars.update(user_supplied_vars)
         self.resolve_list_variables(all_vars)
@@ -490,7 +491,10 @@ def __init__(self, launch_spec, args, configoptions, api, disable_consistency_ch
         :param launch_spec (LaunchSpecification)
         :param args (list) list or arguments to the app launch. These could be BaseSpace IDs or BaseMount paths
         :param configoptions (dict) key->value mapping for additional option values, such as genome-id
-        the ordering of args has to match the ordering of the sorted minimum requirements
+        :param api (BaseSpaceAPI) BaseSpace API object
+        :param disable_consistency_checking (bool) default behaviour is to ensure all entities and the launch itself
+            is done with the same access token. This parameter allows inconsistency (at user's risk!)
+        the ordering of args has to match the ordering of the sorted minimum requirements from the launch_spec
         It might be better to use a dict, but ultimately call order has to match at some point
         """
         self._launch_spec = launch_spec
@@ -540,6 +544,13 @@ def derive_launch_name(self, app_name):
         :return: useful name for app launch
         """
         if LAUNCH_NAME in self._configoptions:
+            # if there is a batch number, the user might have provided a format string (eg. "launch%d") in LAUNCH_NAME
+            # to create batch-specific launch names. Try to use this, but otherwise just return the LAUNCH_NAME
+            if BATCH_NUMBER in self._configoptions:
+                try:
+                    return self._configoptions[LAUNCH_NAME] % self._configoptions[BATCH_NUMBER]
+                except TypeError:
+                    pass
             return self._configoptions[LAUNCH_NAME]
         else:
             launch_names = self._find_all_entity_names("sample")

From 7aa866024f52e95696abd127df86b11b72642547 Mon Sep 17 00:00:00 2001
From: psaffrey 
Date: Mon, 3 Oct 2016 13:45:49 +0100
Subject: [PATCH 64/73] Added method to stop appsessions

---
 src/BaseSpacePy/api/BaseSpaceAPI.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/src/BaseSpacePy/api/BaseSpaceAPI.py b/src/BaseSpacePy/api/BaseSpaceAPI.py
index 099195b..aaa7dec 100755
--- a/src/BaseSpacePy/api/BaseSpaceAPI.py
+++ b/src/BaseSpacePy/api/BaseSpaceAPI.py
@@ -297,6 +297,25 @@ def setAppSessionState(self, Id, Status, Summary):
         postData['statussummary'] = Summary
         return self.__singleRequest__(AppSessionResponse.AppSessionResponse, resourcePath, method, queryParams, headerParams, postData=postData)
 
+    def stopAppSession(self, Id):
+        """
+        Unfortunately, the v1pre3 appsession stop endpoint does not support tokens,
+        so this method has to create a special API object to call a v2 endpoint :(
+
+        :param Id:
+        :return: An AppSessionResponse that contains the appsession we just stopped
+        """
+        resourcePath = '/appsessions/{Id}/stop'
+        method = 'POST'
+        resourcePath = resourcePath.replace('{Id}', Id)
+        queryParams = {}
+        headerParams = {}
+        postData = {}
+        apiServerAndVersion = urlparse.urljoin(self.apiServer, "v2")
+        v2api = BaseAPI(self.getAccessToken(), apiServerAndVersion)
+        return v2api.__singleRequest__(AppSessionResponse.AppSessionResponse, resourcePath, method, queryParams,
+                                  headerParams, postData=postData)
+
     def __deserializeObject__(self, dct, type):
         '''
         Converts API response into object instances for Projects, Samples, and AppResults.

From be6843da5441f878db699bf5d28148ebcc1411ba Mon Sep 17 00:00:00 2001
From: psaffrey 
Date: Thu, 13 Oct 2016 14:40:14 +0100
Subject: [PATCH 65/73] Refactoring of launch name setting (again)

---
 src/BaseSpacePy/api/AppLaunchHelpers.py | 36 +++++++------------------
 1 file changed, 10 insertions(+), 26 deletions(-)

diff --git a/src/BaseSpacePy/api/AppLaunchHelpers.py b/src/BaseSpacePy/api/AppLaunchHelpers.py
index 04df2c2..076a1bc 100644
--- a/src/BaseSpacePy/api/AppLaunchHelpers.py
+++ b/src/BaseSpacePy/api/AppLaunchHelpers.py
@@ -26,9 +26,6 @@
 BS_ENTITIES = ["sample", "project", "appresult", "file"]
 BS_ENTITY_LIST_NAMES = ["Samples", "Projects", "AppResults", "Files"]
 
-LAUNCH_NAME = "LaunchName"
-BATCH_NUMBER = "BatchNumber"
-
 class AppSessionMetaData(object):
     """
     Class to help extract information from an appsession.
@@ -252,8 +249,6 @@ def resolve_list_variables(self, var_dict):
         """
         for varname in var_dict:
             varval = var_dict[varname]
-            if varname == LAUNCH_NAME or varname == BATCH_NUMBER:
-                continue
             if self.is_list_property(varname) and not isinstance(varval, list):
                 var_dict[varname] = [varval]
                 # raise AppLaunchException("non-list property specified for list parameter")
@@ -432,9 +427,8 @@ def make_launch_json(self, user_supplied_vars, launch_name, api_version, sample_
         if required_vars - supplied_var_names:
             raise LaunchSpecificationException(
                 "Compulsory variable(s) missing! (%s)" % str(required_vars - supplied_var_names))
-        if supplied_var_names - (self.get_variable_requirements() | set([LAUNCH_NAME, BATCH_NUMBER])):
-            print "warning! unused variable(s) specified: (%s)" % str(
-                supplied_var_names - (self.get_variable_requirements() | set([LAUNCH_NAME, BATCH_NUMBER])))
+        if supplied_var_names - self.get_variable_requirements():
+            print "warning! unused variable(s) specified: (%s)" % str(supplied_var_names - self.get_variable_requirements())
         all_vars = copy.copy(self.defaults)
         all_vars.update(user_supplied_vars)
         self.resolve_list_variables(all_vars)
@@ -543,25 +537,15 @@ def derive_launch_name(self, app_name):
         :param app_name: name of app
         :return: useful name for app launch
         """
-        if LAUNCH_NAME in self._configoptions:
-            # if there is a batch number, the user might have provided a format string (eg. "launch%d") in LAUNCH_NAME
-            # to create batch-specific launch names. Try to use this, but otherwise just return the LAUNCH_NAME
-            if BATCH_NUMBER in self._configoptions:
-                try:
-                    return self._configoptions[LAUNCH_NAME] % self._configoptions[BATCH_NUMBER]
-                except TypeError:
-                    pass
-            return self._configoptions[LAUNCH_NAME]
+        launch_names = self._find_all_entity_names("sample")
+        if not launch_names:
+            launch_names = self._find_all_entity_names("appresult")
+        if len(launch_names) > 3:
+            contracted_names = launch_names[:3] + ["%dmore" % (len(launch_names) - 3)]
+            launch_instance_name = "+".join(contracted_names)
         else:
-            launch_names = self._find_all_entity_names("sample")
-            if not launch_names:
-                launch_names = self._find_all_entity_names("appresult")
-            if len(launch_names) > 3:
-                contracted_names = launch_names[:3] + ["%dmore" % (len(launch_names) - 3)]
-                launch_instance_name = "+".join(contracted_names)
-            else:
-                launch_instance_name = "+".join(launch_names)
-            return "%s : %s" % (app_name, launch_instance_name)
+            launch_instance_name = "+".join(launch_names)
+        return "%s : %s" % (app_name, launch_instance_name)
 
     def is_valid_basespace_id(self, varname, basespace_id):
         """

From 88c388d0fda3df7110b2178196a1667907d1df07 Mon Sep 17 00:00:00 2001
From: psaffrey 
Date: Fri, 14 Oct 2016 16:39:44 +0100
Subject: [PATCH 66/73] Refactoring to allow string args that contain slashes

---
 src/BaseSpacePy/api/AppLaunchHelpers.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/BaseSpacePy/api/AppLaunchHelpers.py b/src/BaseSpacePy/api/AppLaunchHelpers.py
index 076a1bc..7b856a3 100644
--- a/src/BaseSpacePy/api/AppLaunchHelpers.py
+++ b/src/BaseSpacePy/api/AppLaunchHelpers.py
@@ -560,7 +560,7 @@ def is_valid_basespace_id(self, varname, basespace_id):
         else:
             return True
 
-    def to_basespace_id(self, param_name, varval):
+    def preprocess_arg(self, param_name, varval):
         """
         Checks if a value for a parameter looks like a BaseMount path and tries to convert it into a BaseSpace ID
 
@@ -570,6 +570,8 @@ def to_basespace_id(self, param_name, varval):
         :return basespaceid
         """
         spec_type = self._launch_spec.get_property_bald_type(param_name)
+        if spec_type == "string":
+            return varval
         if os.path.sep in varval:
             # if the argument contains a path separator, it must be a valid BaseMount path
             # otherwise, an exception will be raised by BaseMountInterface
@@ -605,9 +607,9 @@ def get_args(self):
         for i, param_name in enumerate(params):
             arg = self._args[i]
             if isinstance(arg, list):
-                arg_map[param_name] = [self.to_basespace_id(param_name, arg_part) for arg_part in arg]
+                arg_map[param_name] = [self.preprocess_arg(param_name, arg_part) for arg_part in arg]
             else:
-                arg_map[param_name] = self.to_basespace_id(param_name, arg)
+                arg_map[param_name] = self.preprocess_arg(param_name, arg)
         return arg_map
 
     def get_all_variables(self):

From eda57be6213a0681c345c09911c12bd81b54975a Mon Sep 17 00:00:00 2001
From: psaffrey 
Date: Tue, 18 Oct 2016 15:43:04 +0100
Subject: [PATCH 67/73] Add app version methods

---
 src/BaseSpacePy/api/AppLaunchHelpers.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/BaseSpacePy/api/AppLaunchHelpers.py b/src/BaseSpacePy/api/AppLaunchHelpers.py
index 7b856a3..8cd1df9 100644
--- a/src/BaseSpacePy/api/AppLaunchHelpers.py
+++ b/src/BaseSpacePy/api/AppLaunchHelpers.py
@@ -229,7 +229,8 @@ def clean_name(self, parameter_name):
     def process_parameter(self, param, varname):
         # if option is prefixed with an @, it's a file (or process substitution with <() )
         # so we should read inputs from there
-        if param.startswith("@"):
+        property_type = self.get_property_bald_type(varname)
+        if param.startswith("@") and property_type != "string":
             assert self.is_list_property(varname), "cannot specify non-list parameter with file"
             with open(param[1:]) as fh:
                 processed_param = [line.strip() for line in fh]

From 3cf21d70a9c27164703b748d0bc8006382e76fcc Mon Sep 17 00:00:00 2001
From: Lilian Janin 
Date: Mon, 24 Oct 2016 13:38:27 +0100
Subject: [PATCH 68/73] Property map items bug, as fixed in
 https://git.illumina.com/BaseSpace/basespace-python-sdk/pull/1

---
 src/BaseSpacePy/api/AppLaunchHelpers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/BaseSpacePy/api/AppLaunchHelpers.py b/src/BaseSpacePy/api/AppLaunchHelpers.py
index 8cd1df9..e4ed736 100644
--- a/src/BaseSpacePy/api/AppLaunchHelpers.py
+++ b/src/BaseSpacePy/api/AppLaunchHelpers.py
@@ -322,7 +322,7 @@ def populate_properties(self, var_dict, api_version, sample_attributes={}):
             all_sample_attributes = {
                 "Type": "map[]",
                 "Name": "Input.sample-id.attributes",
-                "items": []
+                "itemsmap": []
             }
         for property_ in populated_properties:
             property_name = self.clean_name(property_["Name"])

From e97ebec19882ad585bade29b7a702ef847ba97ab Mon Sep 17 00:00:00 2001
From: Lilian Janin 
Date: Mon, 24 Oct 2016 14:35:46 +0100
Subject: [PATCH 69/73] Property map items bug fix, part 2

---
 src/BaseSpacePy/api/AppLaunchHelpers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/BaseSpacePy/api/AppLaunchHelpers.py b/src/BaseSpacePy/api/AppLaunchHelpers.py
index e4ed736..9f0711d 100644
--- a/src/BaseSpacePy/api/AppLaunchHelpers.py
+++ b/src/BaseSpacePy/api/AppLaunchHelpers.py
@@ -339,7 +339,7 @@ def populate_properties(self, var_dict, api_version, sample_attributes={}):
                         if sample_attributes and bald_type == "sample":
                             one_sample_attributes = self.make_sample_attribute_entry(one_val, wrapped_value,
                                                                                      sample_attributes)
-                            all_sample_attributes["items"].append(one_sample_attributes)
+                            all_sample_attributes["itemsmap"].append(one_sample_attributes)
                 else:
                     processed_value = "%s/%ss/%s" % (api_version, bald_type, property_value)
                     if sample_attributes and bald_type == "sample":

From 0ddf50752612fb6f27a1c9dc80893bcef8221bc2 Mon Sep 17 00:00:00 2001
From: psaffrey 
Date: Tue, 6 Dec 2016 20:40:59 +0000
Subject: [PATCH 70/73] Improved deserialisation of map types; support for map
 types in AppLaunchHelpers

---
 src/BaseSpacePy/api/AppLaunchHelpers.py | 235 +++++++++++++++++++++---
 src/BaseSpacePy/model/KeyValues.py      |  13 ++
 src/BaseSpacePy/model/PropertyMap.py    |   2 +-
 src/BaseSpacePy/model/__init__.py       |   1 +
 4 files changed, 221 insertions(+), 30 deletions(-)
 create mode 100644 src/BaseSpacePy/model/KeyValues.py

diff --git a/src/BaseSpacePy/api/AppLaunchHelpers.py b/src/BaseSpacePy/api/AppLaunchHelpers.py
index 8cd1df9..03b1fb5 100644
--- a/src/BaseSpacePy/api/AppLaunchHelpers.py
+++ b/src/BaseSpacePy/api/AppLaunchHelpers.py
@@ -26,6 +26,7 @@
 BS_ENTITIES = ["sample", "project", "appresult", "file"]
 BS_ENTITY_LIST_NAMES = ["Samples", "Projects", "AppResults", "Files"]
 
+
 class AppSessionMetaData(object):
     """
     Class to help extract information from an appsession.
@@ -39,7 +40,6 @@ class AppSessionMetaData(object):
 
     SKIP_PROPERTIES = ["app-session-name", "attributes", "columns", "num-columns", "rowcount", "IsMultiNode"]
 
-
     def __init__(self, appsession_metadata):
         """
 
@@ -66,6 +66,31 @@ def _get_all_duplicate_names(self):
                 all_names.add(property_basename)
         return duplicate_names
 
+    @staticmethod
+    def _get_map_underlying_types(content):
+        """
+        Takes the content present in an existing appsession map type
+        and converts this into the underlying columns with their name and type
+        :param content: the raw content of the appsession we are deriving from
+        :return: a list of generic columns with name and type
+        """
+        # this should return a list of strings for a single row of the raw table
+        # this only gets us the names - we need to look up types later :(
+        first_row = content[0]
+        columns = [".".join(column.split(".")[:-1]) for column in first_row.Values]
+        return columns
+
+    @staticmethod
+    def _get_owning_map(all_properties, property_name):
+        # look for the property name without its suffix, since for an underlying property this will be eg. "row1"
+        property_name = ".".join(property_name.split(".")[:-1])
+        for property_ in all_properties:
+            if property_["Type"] == "map":
+                if property_name in property_["ColumnTypes"]:
+                    return property_
+        # if we didn't find a map with this property, return None
+        return None
+
     def get_refined_appsession_properties(self):
         """
         Unpacks the properties from an appsession and refines them ready to make a launch specification
@@ -93,6 +118,28 @@ def get_refined_appsession_properties(self):
                 "Name": property_name,
                 "Type": property_type,
             }
+            # this sets up the map, but we need to see examples of the columns in other properties to get the types
+            if property_type == "map":
+                content = self.unpack_bs_property(as_property, "Content")
+                columns = self._get_map_underlying_types(content)
+                this_property["ColumnNames"] = columns
+                # set a dict with an empty target that will be filled in later
+                this_property["ColumnTypes"] = dict((column, None) for column in columns)
+                properties.append(this_property)
+                continue
+            # check to see whether this property is part of an existing map property
+            owning_map = self._get_owning_map(properties, property_name)
+            if owning_map:
+                # if this property is part of a map, the last part of the name will be its row
+                # eg. Input.sample-experiments.happy-id.row1
+                # we should remove that suffix
+                property_name = ".".join(property_name.split(".")[:-1])
+                if owning_map["ColumnTypes"][property_name]:
+                    assert owning_map["ColumnTypes"][property_name] == property_type
+                else:
+                    owning_map["ColumnTypes"][property_name] = property_type
+                # if we are just using the property to grab types for a map, we shouldn't record it anywhere else
+                continue
             properties.append(this_property)
             bald_type = property_type.translate(None, "[]")
             if bald_type in BS_ENTITIES:
@@ -197,6 +244,24 @@ def __init__(self, properties, defaults):
         self.property_lookup = dict((self.clean_name(property_["Name"]), property_) for property_ in self.properties)
         self.defaults = defaults
 
+    def get_map_underlying_names_by_type(self, map_name, underlying_type):
+        map_property = self.property_lookup[map_name]
+        return [varname for varname, vartype in map_property["ColumnTypes"].iteritems() if vartype == underlying_type]
+
+    def get_map_position_by_underlying_name(self, map_name, underlying_name):
+        map_property = self.property_lookup[map_name]
+        return map_property["ColumnNames"].index(underlying_name)
+
+    def get_map_underlying_name_and_type_by_position(self, map_name, position):
+        map_property = self.property_lookup[map_name]
+        underlying_name = map_property["ColumnNames"][position]
+        underlying_type = map_property["ColumnTypes"][underlying_name]
+        return underlying_name, underlying_type
+
+    def get_map_underlying_types(self, map_name):
+        map_property = self.property_lookup[map_name]
+        return map_property["ColumnTypes"].values()
+
     def clean_name(self, parameter_name):
         """
         strip off the Input. prefix, which is needed by the launch payload but gets in the way otherwise
@@ -229,14 +294,20 @@ def clean_name(self, parameter_name):
     def process_parameter(self, param, varname):
         # if option is prefixed with an @, it's a file (or process substitution with <() )
         # so we should read inputs from there
+        # for properties with type map this is probably going to be prone to error :(
         property_type = self.get_property_bald_type(varname)
         if param.startswith("@") and property_type != "string":
             assert self.is_list_property(varname), "cannot specify non-list parameter with file"
             with open(param[1:]) as fh:
-                processed_param = [line.strip() for line in fh]
+                if property_type == "map":
+                    processed_param = [line.strip().split(",") for line in fh]
+                else:
+                    processed_param = [line.strip() for line in fh]
         else:
             if self.is_list_property(varname):
                 processed_param = param.split(",")
+            elif property_type == "map":
+                processed_param = [row.split(",") for row in param.split("::")]
             else:
                 processed_param = param
         return processed_param
@@ -253,7 +324,8 @@ def resolve_list_variables(self, var_dict):
             if self.is_list_property(varname) and not isinstance(varval, list):
                 var_dict[varname] = [varval]
                 # raise AppLaunchException("non-list property specified for list parameter")
-            if not self.is_list_property(varname) and isinstance(varval, list):
+            # if they've supplied a list, it must be for a list or map property
+            if (not self.is_list_property(varname) and self.get_property_type(varname) != "map") and isinstance(varval, list):
                 raise LaunchSpecificationException("list property specified for non-list parameter")
 
     @staticmethod
@@ -330,6 +402,7 @@ def populate_properties(self, var_dict, api_version, sample_attributes={}):
             bald_type = str(property_type).translate(None, "[]")
             property_value = var_dict[property_name]
             processed_value = ""
+            map_properties = []
             if bald_type in BS_ENTITIES:
                 if "[]" in property_type:
                     processed_value = []
@@ -346,14 +419,39 @@ def populate_properties(self, var_dict, api_version, sample_attributes={}):
                         one_sample_attributes = self.make_sample_attribute_entry(property_value, processed_value,
                                                                                  sample_attributes)
                         sample_attributes["Items"].append(one_sample_attributes)
+            if bald_type == "map":
+                # for each argument, create one entry in the property list for each column
+                for rownum, row in enumerate(property_value):
+                    for colnum, value in enumerate(row):
+                        underlying_name, underlying_type = self.get_map_underlying_name_and_type_by_position(property_name, colnum)
+                        if underlying_type in BS_ENTITIES:
+                            wrapped_value = "%s/%ss/%s" % (api_version, underlying_type, value)
+                        else:
+                            wrapped_value = value
+                        assembled_args = {
+                            "Type" : underlying_type,
+                            "Name" : "%s.row%d" % (underlying_name, rownum+1),
+                            "Content" : wrapped_value
+                        }
+                        map_properties.append(assembled_args)
+                rowcount_entry = {
+                    "Type" : "string",
+                    "Name" : "%s.rowcount" % (property_name),
+                    "Content" : len(property_value)
+                }
+                map_properties.append(rowcount_entry)
+                # also create an entry for the number of columns
             if not processed_value:
                 processed_value = property_value
             if "[]" in property_type:
                 property_["items"] = processed_value
             else:
                 property_["Content"] = processed_value
+        populated_properties.extend(map_properties)
         if sample_attributes:
             populated_properties.append(all_sample_attributes)
+        # remove map properties, which aren't needed for launch
+        populated_properties = [property_ for property_ in populated_properties if property_["Type"] != "map"]
         return populated_properties
 
     def get_variable_requirements(self):
@@ -392,6 +490,11 @@ def get_property_bald_type(self, property_name):
         """
         return str(self.get_property_type(property_name)).translate(None, "[]")
 
+    def get_underlying_map_type(self, map_property_name, position):
+        map_property = self.property_lookup[map_property_name]
+        underlying_property_name = map_property["ColumnNames"][position]
+        return map_property["ColumnTypes"][underlying_property_name]
+
     def is_list_property(self, property_name):
         """
         is a given property a list property
@@ -429,7 +532,8 @@ def make_launch_json(self, user_supplied_vars, launch_name, api_version, sample_
             raise LaunchSpecificationException(
                 "Compulsory variable(s) missing! (%s)" % str(required_vars - supplied_var_names))
         if supplied_var_names - self.get_variable_requirements():
-            print "warning! unused variable(s) specified: (%s)" % str(supplied_var_names - self.get_variable_requirements())
+            print "warning! unused variable(s) specified: (%s)" % str(
+                supplied_var_names - self.get_variable_requirements())
         all_vars = copy.copy(self.defaults)
         all_vars.update(user_supplied_vars)
         self.resolve_list_variables(all_vars)
@@ -451,7 +555,10 @@ def property_information_generator(self):
 
     def format_property_information(self):
         header = ["\t".join(["Name", "Type", "Default"])]
-        return "\n".join(header + [ "\t".join(line) for line in self.property_information_generator() ])
+        return "\n".join(header + ["\t".join(line) for line in self.property_information_generator()])
+
+    def format_map_types(self, property_name):
+        return ",".join(self.get_map_underlying_types(property_name))
 
     def dump_property_information(self):
         """
@@ -462,8 +569,15 @@ def dump_property_information(self):
 
     def format_minimum_requirements(self):
         minimum_requirements = self.get_minimum_requirements()
-        description = ["%s (%s)" % (varname, self.get_property_type(varname)) for varname in minimum_requirements]
-        return " ".join(description)
+        descriptions = []
+        for varname in minimum_requirements:
+            property_type = self.get_property_type(varname)
+            if property_type == "map":
+                description = "%s (%s[%s])" % (varname, property_type, self.format_map_types(varname))
+            else:
+                description = "%s (%s)" % (varname, property_type)
+            descriptions.append(description)
+        return " ".join(descriptions)
 
 
 class LaunchPayload(object):
@@ -476,9 +590,9 @@ class LaunchPayload(object):
     """
 
     ENTITY_TYPE_TO_METHOD_NAME = {
-        "sample" : "getSampleById",
-        "appresult" : "getAppResultById",
-        "project" : "getProjectById"
+        "sample": "getSampleById",
+        "appresult": "getAppResultById",
+        "project": "getProjectById"
     }
 
     def __init__(self, launch_spec, args, configoptions, api, disable_consistency_checking=True):
@@ -501,10 +615,47 @@ def __init__(self, launch_spec, args, configoptions, api, disable_consistency_ch
         if len(varnames) != len(self._args):
             raise LaunchSpecificationException("Number of arguments does not match specification")
 
+    def _arg_entry_to_name(self, entry, entity_type):
+        # if the argument contains a path separator, it must be a valid BaseMount path
+        # otherwise, an exception will be raised by BaseMountInterface
+        if os.path.sep in entry:
+            bmi = BaseMountInterface(entry)
+            return bmi.name
+        # if this is not a BaseMount path, try to resolve an entity name using the API
+        # note we're relying on the regular naming of the API to provide the right method name
+        # if this throws an exception, it's up to the caller to catch and handle
+        entry = entry.strip('"')
+        method = getattr(self._api, self.ENTITY_TYPE_TO_METHOD_NAME[entity_type])
+        return method(entry).Name
+
     def _find_all_entity_names(self, entity_type):
         """
         get all the entity names for a particular entity type
         used to make useful launch names
+
+        doing this for map types is pretty complicated. Imagine a map type defined like this:
+
+        {
+        "Name": "Input.sample-experiments",
+        "Type": "map",
+        "ColumnTypes": {
+          "sample-experiments.happy-id": "appresult",
+          "sample-experiments.result-label": "string"
+        },
+        "ColumnNames": [
+          "sample-experiments.happy-id",
+          "sample-experiments.result-label"
+        ]
+        }
+
+        and a map call like this:
+
+        [ [ "124124", "first" ] ,  [ "124127", "second" ] ]
+
+        then we want to look up the entity names of "124124" and "124127" based on their types
+        we get the type by looking up the variable in the position where the map call has been made
+        and then using that type definition to see where we can find the appropriate IDs in the underlying map
+
         :param entity_type: the entity type to look for
         :return: list of entity names
         """
@@ -512,23 +663,33 @@ def _find_all_entity_names(self, entity_type):
         varnames = self._launch_spec.get_minimum_requirements()
         for i, varname in enumerate(varnames):
             arg = self._args[i]
-            if self._launch_spec.get_property_bald_type(varname) == entity_type:
+            this_type = self._launch_spec.get_property_bald_type(varname)
+            if this_type == entity_type:
                 if not self._launch_spec.is_list_property(varname):
                     arg = [arg]
                 for entry in arg:
-                    # if the argument contains a path separator, it must be a valid BaseMount path
-                    # otherwise, an exception will be raised by BaseMountInterface
-                    if os.path.sep in entry:
-                        bmi = BaseMountInterface(entry)
-                        entity_names.append(bmi.name)
-                    # if this is not a BaseMount path, try to resolve an entity name using the API
-                    # note we're relying on the regular naming of the API to provide the right method name
-                    entry = entry.strip('"')
                     try:
-                        method = getattr(self._api, self.ENTITY_TYPE_TO_METHOD_NAME[entity_type])
-                        entity_names.append(method(entry).Name)
+                        name = self._arg_entry_to_name(entry, this_type)
+                        entity_names.append(name)
+                    # if we were unable to find a name, just press on
                     except (AttributeError, ServerResponseException):
                         pass
+            if this_type == "map":
+                # from the type we're after, get the variable name.
+                underlying_names = self._launch_spec.get_map_underlying_names_by_type(varname, entity_type)
+                # Use this to get the parameter positions.
+                varpositions = [self._launch_spec.get_map_position_by_underlying_name(varname, underlying_name)
+                                 for underlying_name in underlying_names]
+                # Use this to get the specifics for this call
+                for entry in arg:
+                    for position in varpositions:
+                        try:
+                            name = self._arg_entry_to_name(entry[position], entity_type)
+                            entity_names.append(name)
+                        # if we were unable to find a name, just press on
+                        except (AttributeError, ServerResponseException):
+                            pass
+
 
         return entity_names
 
@@ -561,17 +722,16 @@ def is_valid_basespace_id(self, varname, basespace_id):
         else:
             return True
 
-    def preprocess_arg(self, param_name, varval):
+    def preprocess_arg(self, param_type, varval):
         """
         Checks if a value for a parameter looks like a BaseMount path and tries to convert it into a BaseSpace ID
 
-        :param param_name: name of the parameter
+        :param param_type: type of the parameter
         :param varval: value of parameter
 
         :return basespaceid
         """
-        spec_type = self._launch_spec.get_property_bald_type(param_name)
-        if spec_type == "string":
+        if param_type == "string":
             return varval
         if os.path.sep in varval:
             # if the argument contains a path separator, it must be a valid BaseMount path
@@ -584,9 +744,9 @@ def preprocess_arg(self, param_name, varval):
                 raise LaunchSpecificationException(
                     "Access tokens between launch configuration and referenced BaseMount path do not match: %s" % varval)
             basemount_type = bmi.type
-            if spec_type != basemount_type:
+            if param_type != basemount_type:
                 raise LaunchSpecificationException(
-                    "wrong type of BaseMount path selected: %s needs to be of type %s" % (varval, spec_type))
+                    "wrong type of BaseMount path selected: %s needs to be of type %s" % (varval, param_type))
             bid = bmi.id
         else:
             # strip off quotes, which will be what comes in from bs list samples -f csv
@@ -608,9 +768,26 @@ def get_args(self):
         for i, param_name in enumerate(params):
             arg = self._args[i]
             if isinstance(arg, list):
-                arg_map[param_name] = [self.preprocess_arg(param_name, arg_part) for arg_part in arg]
+                if isinstance(arg[0], list):
+                    preprocessed_rows = []
+                    # for each row
+                    for row in arg:
+                    # for each column
+                        preprocessed_row = []
+                        for position, column_value in enumerate(row):
+                            # look up type
+                            underlying_type = self._launch_spec.get_underlying_map_type(param_name, position)
+                            # convert
+                            preprocessed_arg = self.preprocess_arg(underlying_type, column_value)
+                            preprocessed_row.append(preprocessed_arg)
+                        preprocessed_rows.append(preprocessed_row)
+                    arg_map[param_name] = preprocessed_rows
+                else:
+                    param_type = self._launch_spec.get_property_bald_type(param_name)
+                    arg_map[param_name] = [self.preprocess_arg(param_type, arg_part) for arg_part in arg]
             else:
-                arg_map[param_name] = self.preprocess_arg(param_name, arg)
+                param_type = self._launch_spec.get_property_bald_type(param_name)
+                arg_map[param_name] = self.preprocess_arg(param_type, arg)
         return arg_map
 
     def get_all_variables(self):
diff --git a/src/BaseSpacePy/model/KeyValues.py b/src/BaseSpacePy/model/KeyValues.py
new file mode 100644
index 0000000..c211e5b
--- /dev/null
+++ b/src/BaseSpacePy/model/KeyValues.py
@@ -0,0 +1,13 @@
+class KeyValues(object):
+
+    def __init__(self):
+        self.swaggerTypes = {
+            'Key': 'str',
+            'Values': 'list',
+        }
+
+    def __str__(self):
+        return str(self.Key)
+
+    def __repr__(self):
+        return str(self)
diff --git a/src/BaseSpacePy/model/PropertyMap.py b/src/BaseSpacePy/model/PropertyMap.py
index 61e55d4..a1344ab 100644
--- a/src/BaseSpacePy/model/PropertyMap.py
+++ b/src/BaseSpacePy/model/PropertyMap.py
@@ -7,7 +7,7 @@ def __init__(self):
             'Href': 'str',
             'Name': 'str',
             'Description': 'str',
-            'Content': 'list',
+            'Content': 'list',
         }
 
     def __str__(self):
diff --git a/src/BaseSpacePy/model/__init__.py b/src/BaseSpacePy/model/__init__.py
index 0f8e44f..7259b23 100644
--- a/src/BaseSpacePy/model/__init__.py
+++ b/src/BaseSpacePy/model/__init__.py
@@ -73,4 +73,5 @@
  'RunResponse',
  'Run',
  'MultipartFileTransfer',
+ 'KeyValues',
  ]

From 406f53e92248d61a22b2718c4fcbc80d8e2ef0f9 Mon Sep 17 00:00:00 2001
From: Lilian Janin 
Date: Mon, 8 May 2017 11:04:14 +0100
Subject: [PATCH 71/73] Removed getApplications restriction

---
 src/BaseSpacePy/api/BaseSpaceAPI.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/BaseSpacePy/api/BaseSpaceAPI.py b/src/BaseSpacePy/api/BaseSpaceAPI.py
index aaa7dec..de63c4c 100755
--- a/src/BaseSpacePy/api/BaseSpaceAPI.py
+++ b/src/BaseSpacePy/api/BaseSpaceAPI.py
@@ -1535,14 +1535,12 @@ def getApplications(self, queryPars=None):
         '''
         Get details about all apps.
         Note that each app will only have a single entry, even if it has many versions
-        :param queryPars: query parameters. Will default to a limit of 1000, so all are gained
+        :param queryPars: query parameters
         :return: list of model.Application.Application objects
         '''
         resourcePath = '/applications'
         method = 'GET'
         headerParams = {}
-        if not queryPars:
-            queryPars = qp({"Limit": 1000})
         queryParams = self._validateQueryParameters(queryPars)
         return self.__listRequest__(Application.Application, resourcePath, method, queryParams, headerParams)
 
@@ -1556,4 +1554,4 @@ def getApplicationById(self, Id):
         method = 'GET'
         headerParams = {}
         queryParams = {}
-        return self.__singleRequest__(Application.Application, resourcePath, method, queryParams, headerParams)
\ No newline at end of file
+        return self.__singleRequest__(Application.Application, resourcePath, method, queryParams, headerParams)

From fa44f90abfc8529bbee86020843631edb746163c Mon Sep 17 00:00:00 2001
From: Lilian Janin 
Date: Mon, 8 May 2017 11:22:15 +0100
Subject: [PATCH 72/73] Updated 0.4 => 0.4.1

---
 src/setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/setup.py b/src/setup.py
index 0458d32..25601fd 100755
--- a/src/setup.py
+++ b/src/setup.py
@@ -24,7 +24,7 @@
 setup(name='basespace-python-sdk',
       description='A Python SDK for connecting to Illumina BaseSpace data',
       author='Illumina',
-      version='0.4',
+      version='0.4.1',
       long_description="""
 BaseSpacePy is a Python based SDK to be used in the development of Apps and scripts for working with
 Illumina's BaseSpace cloud-computing solution for next-gen sequencing data analysis.

From 8562ac9aa5f964dfd945cbd60fb4c8101f3db2d2 Mon Sep 17 00:00:00 2001
From: psaffrey 
Date: Wed, 30 May 2018 12:23:49 +0100
Subject: [PATCH 73/73] Change to paging code to fix `bs list apps` problem

---
 src/BaseSpacePy/api/BaseAPI.py | 5 ++++-
 src/setup.py                   | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/BaseSpacePy/api/BaseAPI.py b/src/BaseSpacePy/api/BaseAPI.py
index dab6b2f..69de48c 100644
--- a/src/BaseSpacePy/api/BaseAPI.py
+++ b/src/BaseSpacePy/api/BaseAPI.py
@@ -141,7 +141,10 @@ def __listRequest__(self, myModel, resourcePath, method, queryParams, headerPara
             # to catch the race condition where a new entity appears while we're calling
             total_number = respObj.Response.TotalCount
             if total_number > 0 and respObj.Response.DisplayedCount == 0:
-                raise ServerResponseException("Paged query returned no results")
+                # sometimes the API DisplayedCount and TotalCount don't match :(
+                # if there are none left, just return what we've found already
+                break
+                #raise ServerResponseException("Paged query returned no results")
             number_received += respObj.Response.DisplayedCount
 
         return [self.apiClient.deserialize(c, myModel) for c in chain(*[ ro._convertToObjectList() for ro in responses ])]
diff --git a/src/setup.py b/src/setup.py
index 25601fd..fb41500 100755
--- a/src/setup.py
+++ b/src/setup.py
@@ -24,7 +24,7 @@
 setup(name='basespace-python-sdk',
       description='A Python SDK for connecting to Illumina BaseSpace data',
       author='Illumina',
-      version='0.4.1',
+      version='0.4.2',
       long_description="""
 BaseSpacePy is a Python based SDK to be used in the development of Apps and scripts for working with
 Illumina's BaseSpace cloud-computing solution for next-gen sequencing data analysis.