From c35ed4e148936f963fc29852c29b13a01282ebc1 Mon Sep 17 00:00:00 2001 From: Matt Robenolt Date: Sat, 25 Jul 2015 13:44:42 -0700 Subject: [PATCH 001/292] Provide proper compatability support for str/unicode in py3 Fixes bug introduced with ae6763af1f91f2d35aa3ff238c77500dfb6caab0 Also cleans up a few lint issues and unused imports --- marathon/_compat.py | 11 +++++++++++ marathon/client.py | 12 +----------- marathon/util.py | 11 ++++++----- 3 files changed, 18 insertions(+), 16 deletions(-) create mode 100644 marathon/_compat.py diff --git a/marathon/_compat.py b/marathon/_compat.py new file mode 100644 index 0000000..b2c19ff --- /dev/null +++ b/marathon/_compat.py @@ -0,0 +1,11 @@ +""" +Support for python 2 & 3, ripped pieces from six.py +""" +import sys + +PY3 = sys.version_info[0] == 3 + +if PY3: + string_types = str, +else: + string_types = basestring, diff --git a/marathon/client.py b/marathon/client.py index 96c7959..61668c3 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -1,21 +1,11 @@ import itertools import time -import sys try: import json except ImportError: import simplejson as json -# Support Python 2 & 3 - -if sys.version_info[0] == 3: - import urllib.parse as urlparse - from urllib.error import HTTPError -else: - import urlparse - from urllib2 import HTTPError - import requests import requests.exceptions @@ -552,4 +542,4 @@ def get_metrics(self): :rtype: dict """ response = self._do_request('GET', '/metrics') - return response.json() \ No newline at end of file + return response.json() diff --git a/marathon/util.py b/marathon/util.py index 944c723..22a7451 100644 --- a/marathon/util.py +++ b/marathon/util.py @@ -1,6 +1,5 @@ import collections import datetime -import types try: import json @@ -8,9 +7,11 @@ import simplejson as json import re +from ._compat import string_types + def is_stringy(obj): - return isinstance(obj, str) or isinstance(obj, unicode) + return isinstance(obj, string_types) class MarathonJsonEncoder(json.JSONEncoder): @@ -25,7 +26,7 @@ def default(self, obj): if isinstance(obj, collections.Iterable) and not is_stringy(obj): try: - return {k: self.default(v) for k,v in obj.items()} + return {k: self.default(v) for k, v in obj.items()} except AttributeError: return [self.default(e) for e in obj] @@ -44,9 +45,9 @@ def default(self, obj): if isinstance(obj, collections.Iterable) and not is_stringy(obj): try: - return {k: self.default(v) for k,v in obj.items() if (v or v == False)} + return {k: self.default(v) for k, v in obj.items() if (v or v is False)} except AttributeError: - return [self.default(e) for e in obj if (e or e == False)] + return [self.default(e) for e in obj if (e or e is False)] return obj From 3d49f803f266c8f1b7b43ca3ad63d47018726c2f Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Wed, 28 Oct 2015 15:30:55 +0200 Subject: [PATCH 002/292] Remove call to logging.basicConfig It's rude to interfere with the using application logging configuration... --- marathon/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/marathon/__init__.py b/marathon/__init__.py index ea21d0a..8d2975c 100644 --- a/marathon/__init__.py +++ b/marathon/__init__.py @@ -5,4 +5,3 @@ from .exceptions import MarathonError, MarathonHttpError, NotFoundError, InvalidChoiceError log = logging.getLogger(__name__) -logging.basicConfig() \ No newline at end of file From 69013b302b86064bc8cf17d45dfe633cbf1f1afd Mon Sep 17 00:00:00 2001 From: Robert Johnson Date: Tue, 20 Oct 2015 09:51:07 -0700 Subject: [PATCH 003/292] use the /v2/tasks endpoint for list_tasks this ensures that the correct endpoint is used irrespective of whether an app_id is passed to list_tasks or not --- marathon/client.py | 9 +++-- tests/test_api.py | 87 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 5 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index 96c7959..7f9cad7 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -349,12 +349,11 @@ def list_tasks(self, app_id=None, **kwargs): :returns: list of tasks :rtype: list[:class:`marathon.models.task.MarathonTask`] """ + response = self._do_request('GET', '/v2/tasks') + tasks = self._parse_response(response, MarathonTask, is_list=True, resource_name='tasks') if app_id: - response = self._do_request('GET', '/v2/apps/{app_id}/tasks'.format(app_id=app_id)) - else: - response = self._do_request('GET', '/v2/tasks') + tasks = [task for task in tasks if task.app_id == app_id] - tasks = self._parse_response(response, MarathonTask, is_list=True, resource_name='tasks') [setattr(t, 'app_id', app_id) for t in tasks if app_id and t.app_id is None] for k, v in kwargs.items(): tasks = [o for o in tasks if getattr(o, k) == v] @@ -552,4 +551,4 @@ def get_metrics(self): :rtype: dict """ response = self._do_request('GET', '/metrics') - return response.json() \ No newline at end of file + return response.json() diff --git a/tests/test_api.py b/tests/test_api.py index 05bbd17..23e8259 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -20,3 +20,90 @@ def test_get_deployments(m): version=u"fakeversion" )] assert expected_deployments == actual_deployments + + +@requests_mock.mock() +def test_list_tasks_with_app_id(m): + fake_response = '{ "tasks": [ { "appId": "/anapp", "healthCheckResults": [ { "alive": true, "consecutiveFailures": 0, "firstSuccess": "2014-10-03T22:57:02.246Z", "lastFailure": null, "lastSuccess": "2014-10-03T22:57:41.643Z", "taskId": "bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799" } ], "host": "10.141.141.10", "id": "bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799", "ports": [ 31000 ], "servicePorts": [ 9000 ], "stagedAt": "2014-10-03T22:16:27.811Z", "startedAt": "2014-10-03T22:57:41.587Z", "version": "2014-10-03T22:16:23.634Z" }, { "appId": "/anotherapp", "healthCheckResults": [ { "alive": true, "consecutiveFailures": 0, "firstSuccess": "2014-10-03T22:57:02.246Z", "lastFailure": null, "lastSuccess": "2014-10-03T22:57:41.649Z", "taskId": "bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799" } ], "host": "10.141.141.10", "id": "bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799", "ports": [ 31001 ], "servicePorts": [ 9000 ], "stagedAt": "2014-10-03T22:16:33.814Z", "startedAt": "2014-10-03T22:57:41.593Z", "version": "2014-10-03T22:16:23.634Z" } ] }' + m.get('http://fake_server/v2/tasks', text=fake_response) + mock_client = MarathonClient(servers='http://fake_server') + actual_deployments = mock_client.list_tasks(app_id='/anapp') + expected_deployments = [ models.task.MarathonTask( + app_id="/anapp", + health_check_results= [ + models.task.MarathonHealthCheckResult( + alive= True, + consecutive_failures= 0, + first_success= "2014-10-03T22:57:02.246Z", + last_failure= None, + last_success= "2014-10-03T22:57:41.643Z", + task_id= "bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799" + ) + ], + host= "10.141.141.10", + id= "bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799", + ports= [ + 31000 + ], + service_ports= [ + 9000 + ], + staged_at= "2014-10-03T22:16:27.811Z", + started_at= "2014-10-03T22:57:41.587Z", + version= "2014-10-03T22:16:23.634Z" + )] + assert actual_deployments == expected_deployments + + +@requests_mock.mock() +def test_list_tasks_without_app_id(m): + fake_response = '{ "tasks": [ { "appId": "/anapp", "healthCheckResults": [ { "alive": true, "consecutiveFailures": 0, "firstSuccess": "2014-10-03T22:57:02.246Z", "lastFailure": null, "lastSuccess": "2014-10-03T22:57:41.643Z", "taskId": "bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799" } ], "host": "10.141.141.10", "id": "bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799", "ports": [ 31000 ], "servicePorts": [ 9000 ], "stagedAt": "2014-10-03T22:16:27.811Z", "startedAt": "2014-10-03T22:57:41.587Z", "version": "2014-10-03T22:16:23.634Z" }, { "appId": "/anotherapp", "healthCheckResults": [ { "alive": true, "consecutiveFailures": 0, "firstSuccess": "2014-10-03T22:57:02.246Z", "lastFailure": null, "lastSuccess": "2014-10-03T22:57:41.649Z", "taskId": "bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799" } ], "host": "10.141.141.10", "id": "bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799", "ports": [ 31001 ], "servicePorts": [ 9000 ], "stagedAt": "2014-10-03T22:16:33.814Z", "startedAt": "2014-10-03T22:57:41.593Z", "version": "2014-10-03T22:16:23.634Z" } ] }' + m.get('http://fake_server/v2/tasks', text=fake_response) + mock_client = MarathonClient(servers='http://fake_server') + actual_deployments = mock_client.list_tasks() + expected_deployments = [ + models.task.MarathonTask( + app_id="/anapp", + health_check_results= [ + models.task.MarathonHealthCheckResult( + alive= True, + consecutive_failures= 0, + first_success= "2014-10-03T22:57:02.246Z", + last_failure= None, + last_success= "2014-10-03T22:57:41.643Z", + task_id= "bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799" + ) + ], + host= "10.141.141.10", + id="bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799", + ports= [ + 31000 + ], + service_ports= [ + 9000 + ], + staged_at= "2014-10-03T22:16:27.811Z", + started_at= "2014-10-03T22:57:41.587Z", + version= "2014-10-03T22:16:23.634Z" + ), + models.task.MarathonTask( + app_id= "/anotherapp", + health_check_results= [ + models.task.MarathonHealthCheckResult( + alive= True, + consecutive_failures= 0, + first_success = "2014-10-03T22:57:02.246Z", + last_failure= None, + last_success= "2014-10-03T22:57:41.649Z", + task_id= "bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799" + ) + ], + host= "10.141.141.10", + id= "bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799", + ports= [ 31001 ], + service_ports= [ 9000 ], + staged_at = "2014-10-03T22:16:33.814Z", + started_at= "2014-10-03T22:57:41.593Z", + version= "2014-10-03T22:16:23.634Z" + )] + assert actual_deployments == expected_deployments From d514eeceb78326cc7d09f7dcdbafe9d1b01d52c2 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Wed, 11 Nov 2015 17:45:32 -0800 Subject: [PATCH 004/292] Release 0.7.3 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0960391..fea05df 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='marathon', - version='0.7.2', + version='0.7.3', description='Marathon Client Library', long_description="""Python interface to the Mesos Marathon REST API.""", author='Mike Babineau', From af995fb81de272bf3f40a2dd4369b0d29610e043 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Sat, 14 Nov 2015 10:09:00 -0800 Subject: [PATCH 005/292] Fix dockerfile to have java 8 and travis --- itests/Dockerfile | 8 ++++++-- itests/install-marathon.sh | 10 ++++++++++ itests/start-marathon.sh | 3 ++- tox.ini | 2 ++ 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/itests/Dockerfile b/itests/Dockerfile index 90f513d..8cf69f3 100644 --- a/itests/Dockerfile +++ b/itests/Dockerfile @@ -1,5 +1,9 @@ -FROM ubuntu-debootstrap:14.04 -RUN apt-get update && apt-get -y install sudo lsb-release +FROM ubuntu:14.04 +RUN apt-get install -y software-properties-common +RUN add-apt-repository ppa:webupd8team/java +RUN echo "debconf shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections +RUN echo "debconf shared/accepted-oracle-license-v1-1 seen true" | debconf-set-selections +RUN apt-get update && apt-get -y install lsb-release oracle-java8-installer java8-runtime-headless # Setup ADD ./marathon-version /root/marathon-version diff --git a/itests/install-marathon.sh b/itests/install-marathon.sh index 8dd00ec..208fb36 100755 --- a/itests/install-marathon.sh +++ b/itests/install-marathon.sh @@ -18,4 +18,14 @@ echo "deb http://repos.mesosphere.com/${DISTRO} ${CODENAME} main" | sudo apt-get -y update # Install packages +sudo apt-get -y install oracle-java8-installer +sudo apt-get -y purge oracle-java7-installer +sudo update-java-alternatives -s java-8-oracle +sudo apt-get install oracle-java8-set-default + sudo apt-get -y --force-yes install mesos marathon=$MARATHONVERSION* + +# WTF MARATHON? +# Why does the precise version have java7 hardcoded if it requires java8? +sudo mkdir -p /usr/lib/jvm/java-7-oracle/bin/ +sudo ln -s /usr/lib/jvm/java-8-oracle/bin/java /usr/lib/jvm/java-7-oracle/bin/java diff --git a/itests/start-marathon.sh b/itests/start-marathon.sh index 5caa960..6c21fa2 100755 --- a/itests/start-marathon.sh +++ b/itests/start-marathon.sh @@ -6,4 +6,5 @@ else LOGGER="" fi -exec marathon --master local $LOGGER --hostname localhost +java -version +exec /usr/bin/marathon --master local $LOGGER --hostname localhost diff --git a/tox.ini b/tox.ini index cb7d104..0c164c4 100644 --- a/tox.ini +++ b/tox.ini @@ -11,6 +11,8 @@ whitelist_externals=/bin/bash skipsdist=True changedir=itests/ deps = + requests<2.7 + {[testenv]deps} docker-compose==1.3.1 behave mock From c0f728085de560fc803e6540e5ae6eeae8604508 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Sat, 14 Nov 2015 11:36:02 -0800 Subject: [PATCH 006/292] Added mesos_leader_ui_url to the marathon info model --- marathon/models/info.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/marathon/models/info.py b/marathon/models/info.py index f2a2a65..140088e 100644 --- a/marathon/models/info.py +++ b/marathon/models/info.py @@ -4,7 +4,7 @@ class MarathonInfo(MarathonResource): """Marathon Info. - See: https://mesosphere.github.io/marathon/docs/rest-api.html#get-/v2/info + See: https://mesosphere.github.io/marathon/docs/rest-api.html#get-v2-info :param str framework_id: :param str leader: @@ -58,6 +58,7 @@ class MarathonConfig(MarathonObject): :param int local_port_min: :param int local_port_max: :param str master: + :param str mesos_leader_ui_url: :param str mesos_role: :param str mesos_user: :param str webui_url: @@ -69,7 +70,7 @@ class MarathonConfig(MarathonObject): def __init__(self, checkpoint=None, executor=None, failover_timeout=None, framework_name=None, ha=None, hostname=None, leader_proxy_connection_timeout_ms=None, leader_proxy_read_timeout_ms=None, - local_port_min=None, local_port_max=None, master=None, mesos_role=None, mesos_user=None, + local_port_min=None, local_port_max=None, master=None, mesos_leader_ui_url=None, mesos_role=None, mesos_user=None, webui_url=None, reconciliation_initial_delay=None, reconciliation_interval=None, task_launch_timeout=None, marathon_store_timeout=None): self.checkpoint = checkpoint @@ -80,6 +81,7 @@ def __init__(self, checkpoint=None, executor=None, failover_timeout=None, framew self.local_port_min = local_port_min self.local_port_max = local_port_max self.master = master + self.mesos_leader_ui_url = mesos_leader_ui_url self.mesos_role = mesos_role self.mesos_user = mesos_user self.webui_url = webui_url From 4e8849064b853cbfe65cdc2520ee1acc98048089 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Fri, 20 Nov 2015 07:51:58 -0800 Subject: [PATCH 007/292] Use automatic changelog generation --- CHANGELOG.md | 273 +++++++++++++++++++++++++++++++++------------------ Makefile | 1 + 2 files changed, 176 insertions(+), 98 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1dc3eb7..98cffb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,149 +1,226 @@ -## 0.7.2 (2015-09-17) +# Change Log -Support for Marathon 0.9.1 +## [Unreleased](https://github.com/thefactory/marathon-python/tree/HEAD) -Changes: -* Add `accepted_resource_role` field to `MarathonApp` +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.7.3...HEAD) -## 0.7.1 (2015-07-14) +**Merged pull requests:** -Critical fixes and @solarkennedy has been addded as a contributor! +- Marathon 11 Support [\#68](https://github.com/thefactory/marathon-python/pull/68) ([solarkennedy](https://github.com/solarkennedy)) -Changes (all @solarkennedy - huge thanks): -* Fixed `MarathonApp` regex issue -* Hooked up Travis CI -* Added integration tests +## [0.7.3](https://github.com/thefactory/marathon-python/tree/0.7.3) (2015-11-12) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.7.2...0.7.3) -## 0.7.0 (2015-07-05) +**Closed issues:** -Support for Marathon 0.8.2 +- When will you guys release 0.7.2 [\#62](https://github.com/thefactory/marathon-python/issues/62) +- 0.7.1 tag missing [\#61](https://github.com/thefactory/marathon-python/issues/61) -Changes: -* Tests (huge thanks @kevinschoon) -* Added support for Marathon EventBus (thanks @kevinschoon) -* Marathon app_id and group_id validation (thanks @mattrobenolt) -* Fixed bug in creation of deployment.steps list (thanks @AFriemann) -* Add `ignore_http1xx` on `MarathonHealthCheck` (thanks @mrtheb) +**Merged pull requests:** -## 0.6.15 (2015-06-05) +- use the /v2/tasks endpoint for list\_tasks [\#65](https://github.com/thefactory/marathon-python/pull/65) ([Rob-Johnson](https://github.com/Rob-Johnson)) +- Remove call to logging.basicConfig [\#64](https://github.com/thefactory/marathon-python/pull/64) ([itamaro](https://github.com/itamaro)) -Changes: -* Fix `force_pull_image` on `MarathonDockerContainer` (thanks @mattrobenolt) +## [0.7.2](https://github.com/thefactory/marathon-python/tree/0.7.2) (2015-09-18) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.7.0...0.7.2) -## 0.6.14 (2015-05-28) +**Closed issues:** -Changes: -* Add `force_pull_image` field to `MarathonDockerContainer` (thanks @solarkennedy) -* Add `kwargs` to `MarathonDockerContainer` for better forward compatibility (thanks @g----) -* Fix issue with `use_2to3` (thanks @vitan) +- Marathon Json encoder can't handle unicode strings [\#50](https://github.com/thefactory/marathon-python/issues/50) +- Marathon app name validation is broken [\#45](https://github.com/thefactory/marathon-python/issues/45) +- New release for 8.2 compatibility [\#38](https://github.com/thefactory/marathon-python/issues/38) +- Task.app\_id is None when using c.get\_app\("xxx"\).tasks [\#9](https://github.com/thefactory/marathon-python/issues/9) -## 0.6.13 (2015-03-24) +**Merged pull requests:** -Support for Marathon 0.8.1 +- Updated to support Marathon 0.9.1 with get\_info\(\) calls [\#59](https://github.com/thefactory/marathon-python/pull/59) ([pyronicide](https://github.com/pyronicide)) +- Add support for building with a wheel and cleanup setup.py [\#58](https://github.com/thefactory/marathon-python/pull/58) ([mattrobenolt](https://github.com/mattrobenolt)) +- travis should run unit tests [\#55](https://github.com/thefactory/marathon-python/pull/55) ([Rob-Johnson](https://github.com/Rob-Johnson)) +- implement \_\_eq\_\_ on base models + fix tests to be useful [\#54](https://github.com/thefactory/marathon-python/pull/54) ([Rob-Johnson](https://github.com/Rob-Johnson)) +- Fix deployments parsing [\#53](https://github.com/thefactory/marathon-python/pull/53) ([Rob-Johnson](https://github.com/Rob-Johnson)) +- Added failing test and fix for unicode handling [\#52](https://github.com/thefactory/marathon-python/pull/52) ([solarkennedy](https://github.com/solarkennedy)) +- Removed previously unused test framework in favor of tox + docker-compose version [\#49](https://github.com/thefactory/marathon-python/pull/49) ([solarkennedy](https://github.com/solarkennedy)) +- Add accepted\_resource\_role kwarg for Marathon 0.9.0 support [\#48](https://github.com/thefactory/marathon-python/pull/48) ([keshavdv](https://github.com/keshavdv)) +- Upstream merge - Add itest framework and fix regex [\#46](https://github.com/thefactory/marathon-python/pull/46) ([solarkennedy](https://github.com/solarkennedy)) +- First pass at adding an itest framework [\#42](https://github.com/thefactory/marathon-python/pull/42) ([solarkennedy](https://github.com/solarkennedy)) -Changes: -* Better handling of nulls and empty collections (thanks @wndhydrnt) -* Updated object signatures to match 0.8.1 (thanks @pradeepchhetri) +## [0.7.0](https://github.com/thefactory/marathon-python/tree/0.7.0) (2015-07-06) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.6.15...0.7.0) -## 0.6.12 (2015-03-06) +**Closed issues:** -Changes: -* Replace defunct `MarathonEndpoint` resource with a working helper object +- MarathonHealthCheck class doesn't support 0.8.2 [\#34](https://github.com/thefactory/marathon-python/issues/34) -## 0.6.11 (2015-03-06) +**Merged pull requests:** -Support for Marathon 0.8.0 +- Update endpoint docstring. [\#41](https://github.com/thefactory/marathon-python/pull/41) ([Poogles](https://github.com/Poogles)) +- fixed variable name in list comprehension [\#39](https://github.com/thefactory/marathon-python/pull/39) ([AFriemann](https://github.com/AFriemann)) +- Add validation to marathon app/group ids [\#37](https://github.com/thefactory/marathon-python/pull/37) ([mattrobenolt](https://github.com/mattrobenolt)) +- adds ignore\_http1xx and forward compat kwargs to MarathonHealthCheck [\#36](https://github.com/thefactory/marathon-python/pull/36) ([mrtheb](https://github.com/mrtheb)) +- Feature/event factory [\#32](https://github.com/thefactory/marathon-python/pull/32) ([kevinschoon](https://github.com/kevinschoon)) -## 0.6.10 (2014-12-17) +## [0.6.15](https://github.com/thefactory/marathon-python/tree/0.6.15) (2015-06-05) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.6.14...0.6.15) -Changes: -* Added `force` option to `delete_deployment()` +**Merged pull requests:** -## 0.6.9 (2014-12-03) +- Make `force\_pull\_image` actually work [\#33](https://github.com/thefactory/marathon-python/pull/33) ([mattrobenolt](https://github.com/mattrobenolt)) -Changes: -* Added lastFailureCause field to app.lastTaskFailure -* Added `parameters` and `privileged` fields to `MarathonContainer` +## [0.6.14](https://github.com/thefactory/marathon-python/tree/0.6.14) (2015-05-28) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.6.13...0.6.14) -## 0.6.8 (2014-11-18) +**Closed issues:** -Changes: -* (Temporarily) Added `apps` field to `DeploymentAction` (https://github.com/mesosphere/marathon/pull/802) +- forcePullImage not honored by marathon-python \(marathon 0.8.2 RC2\) [\#29](https://github.com/thefactory/marathon-python/issues/29) +- create\_app\(\) not working for docker container [\#28](https://github.com/thefactory/marathon-python/issues/28) +- Urgent BUG: to\_json\(\) is returning unexpected result under python3 version [\#26](https://github.com/thefactory/marathon-python/issues/26) +- portMapping isn't iterable [\#25](https://github.com/thefactory/marathon-python/issues/25) -## 0.6.7 (2014-11-18) +**Merged pull requests:** -Changes: -* Updated `list_tasks()` and `get_info()` to match latest Marathon response signature -* Fixed `__repr__` for `MarathonInfo()` and other `MarathonResources` without `id +- Added forcePullImage parameter for the container model [\#31](https://github.com/thefactory/marathon-python/pull/31) ([solarkennedy](https://github.com/solarkennedy)) +- Quick fix \#29 - add kwargs to MarathonDockerContainer.\_\_init\_\_ [\#30](https://github.com/thefactory/marathon-python/pull/30) ([g----](https://github.com/g----)) +- Fixed \#26:Using try/except to get rid of use\_2to3 failing [\#27](https://github.com/thefactory/marathon-python/pull/27) ([vitan](https://github.com/vitan)) -## 0.6.6 (2014-11-17) +## [0.6.13](https://github.com/thefactory/marathon-python/tree/0.6.13) (2015-03-24) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.6.12...0.6.13) -Changes: -* Improved behavior of `MarathonClient.update_app()` to strip `version` from the passed `app` +**Merged pull requests:** -## 0.6.5 (2014-11-14) +- Added get\_leader and delete\_leader functions [\#24](https://github.com/thefactory/marathon-python/pull/24) ([pradeepchhetri](https://github.com/pradeepchhetri)) +- Fixed get\_info for marathon-0.8.1-RC2 [\#23](https://github.com/thefactory/marathon-python/pull/23) ([pradeepchhetri](https://github.com/pradeepchhetri)) +- Added two app parameters - tasks\_healthy, tasks\_unhealthy \(marathon-0.8.1-RC2\) [\#21](https://github.com/thefactory/marathon-python/pull/21) ([pradeepchhetri](https://github.com/pradeepchhetri)) +- Possibility to send the full object to Marathon on update [\#20](https://github.com/thefactory/marathon-python/pull/20) ([wndhydrnt](https://github.com/wndhydrnt)) -Changes: -* Fixed bug with `MarathonClient.scale_app()` and add `force` support +## [0.6.12](https://github.com/thefactory/marathon-python/tree/0.6.12) (2015-03-07) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.6.11...0.6.12) -## 0.6.4 (2014-11-13) +## [0.6.11](https://github.com/thefactory/marathon-python/tree/0.6.11) (2015-03-06) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.6.10...0.6.11) -Support for Marathon 0.7.5 +**Merged pull requests:** -Changes: -* Added support for app.lastTaskFailure -* Added support for task.healthCheckResult -* Fixed support for app.upgradeStrategy +- Small changes to fix compatibility issues with Marathon 0.8.0 [\#19](https://github.com/thefactory/marathon-python/pull/19) ([cloudify](https://github.com/cloudify)) -## 0.6.3 (2014-10-10) +## [0.6.10](https://github.com/thefactory/marathon-python/tree/0.6.10) (2014-12-17) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.6.8...0.6.10) -Changes: -* Added support for embedding tasks in get/list app results -* Switched to patch-style updates for apps and groups (send partial object) +**Merged pull requests:** -## 0.6.2 (2014-10-09) +- Added optional ?force=true param to MarathonClient.delete\_deployment\(\) [\#18](https://github.com/thefactory/marathon-python/pull/18) ([mattcallanan](https://github.com/mattcallanan)) +- Add parameters and privileged fields to Container model [\#17](https://github.com/thefactory/marathon-python/pull/17) ([gabrtv](https://github.com/gabrtv)) +- apparently undocumented API in Marathon [\#16](https://github.com/thefactory/marathon-python/pull/16) ([elyast](https://github.com/elyast)) -Changes: -* Added support for service port in container port mappings (Marathon 0.7.3) +## [0.6.8](https://github.com/thefactory/marathon-python/tree/0.6.8) (2014-11-19) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.6.7...0.6.8) -## 0.6.2 (2014-10-09) +## [0.6.7](https://github.com/thefactory/marathon-python/tree/0.6.7) (2014-11-18) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.6.6...0.6.7) -Changes: -* Added support for LIKE and UNLIKE constraint operators -* Added support for patch-style app and group updates +**Closed issues:** -## 0.6.1 (2014-09-29) +- update\_app\(\) no-ops if Version is passed [\#14](https://github.com/thefactory/marathon-python/issues/14) -Changes: -* Fixed broken exception import +**Merged pull requests:** -## 0.6.0 (2014-09-29) +- fixing issues with resources /v2/tasks, v2/info [\#15](https://github.com/thefactory/marathon-python/pull/15) ([elyast](https://github.com/elyast)) -Mesos 0.20.1- and Marathon 0.7.1-compatible release. +## [0.6.6](https://github.com/thefactory/marathon-python/tree/0.6.6) (2014-11-17) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.6.5...0.6.6) -Changes: -* Added bridge networking support -* Use common exception for all pick-from-a-list options (`InvalidChoiceError`). `InvalidOperatorError` has been removed -* Updated `MarathonObject.__repr__` to be more useful -* Changed default HTTP request timeout from 5s to 10s +**Closed issues:** -## 0.5.1 (2014-09-18) +- scale\_app\(...\) calls update\_app\(...\) with only 1 argument [\#13](https://github.com/thefactory/marathon-python/issues/13) -Changes: -* Added support for multiple Marathon servers (if a request to one fails for network reasons, try the next) -* Fixed a bug with HTTP 4xx response handling +## [0.6.5](https://github.com/thefactory/marathon-python/tree/0.6.5) (2014-11-14) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.6.4...0.6.5) -## 0.5.0 (2014-09-15) +## [0.6.4](https://github.com/thefactory/marathon-python/tree/0.6.4) (2014-11-14) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.6.3...0.6.4) -Initial release compatible with Marathon 0.7.0 and Mesos 0.20.0. +**Merged pull requests:** -_Warning: this includes multiple breaking changes, both from Marathon and from this library_ +- Add MarathonHealthCheckResult Class to tasks File and Include it in MarathonTask [\#12](https://github.com/thefactory/marathon-python/pull/12) ([JTCunning](https://github.com/JTCunning)) -Changes: -* Added support for Deployments, Groups, native Docker containers, Queue, Server Info, Server Metrics, and Ping -* Updated object attributes to be at parity with Marathon 0.7.0 (RC2) -* Updated return types to be at parity with Marathon 0.7.0 (RC2) -* Updated method signatures to be more consistent -* HTTP 4xx errors other than 404 now throw `MarathonHttpError` instead of `NotFoundError` -* `json_encode()` and `json_decode()` on MarathonResource have been renamed to `to_json()` and `from_json()`, respectively -* Refactored serialization/deserialization to reduce boilerplate +## [0.6.3](https://github.com/thefactory/marathon-python/tree/0.6.3) (2014-10-10) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.6.2...0.6.3) + +**Merged pull requests:** + +- add service\_port argument [\#11](https://github.com/thefactory/marathon-python/pull/11) ([danielfrg](https://github.com/danielfrg)) + +## [0.6.2](https://github.com/thefactory/marathon-python/tree/0.6.2) (2014-10-09) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.6.1...0.6.2) + +**Merged pull requests:** + +- Add `LIKE` and `UNLIKE` constraint [\#10](https://github.com/thefactory/marathon-python/pull/10) ([iven](https://github.com/iven)) + +## [0.6.1](https://github.com/thefactory/marathon-python/tree/0.6.1) (2014-09-29) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.6.0...0.6.1) + +## [0.6.0](https://github.com/thefactory/marathon-python/tree/0.6.0) (2014-09-29) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.5.1...0.6.0) + +**Closed issues:** + +- Support for HA nodes [\#8](https://github.com/thefactory/marathon-python/issues/8) + +## [0.5.1](https://github.com/thefactory/marathon-python/tree/0.5.1) (2014-09-18) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.5.0...0.5.1) + +## [0.5.0](https://github.com/thefactory/marathon-python/tree/0.5.0) (2014-09-18) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.4.0...0.5.0) + +**Merged pull requests:** + +- Bug Fix: Cannot define constraints with a tuple of strings [\#6](https://github.com/thefactory/marathon-python/pull/6) ([adgaudio](https://github.com/adgaudio)) + +## [0.4.0](https://github.com/thefactory/marathon-python/tree/0.4.0) (2014-08-19) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.3.1...0.4.0) + +**Merged pull requests:** + +- Throwing exceptions on 400s and 500s in \_do\_request [\#5](https://github.com/thefactory/marathon-python/pull/5) ([Codeacious](https://github.com/Codeacious)) +- Fix container options not being sent to marathon [\#4](https://github.com/thefactory/marathon-python/pull/4) ([boffbowsh](https://github.com/boffbowsh)) + +## [0.3.1](https://github.com/thefactory/marathon-python/tree/0.3.1) (2014-08-05) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.2.9...0.3.1) + +**Merged pull requests:** + +- Raise exceptions instead of swallowing them silently [\#3](https://github.com/thefactory/marathon-python/pull/3) ([StephanErb](https://github.com/StephanErb)) + +## [0.2.9](https://github.com/thefactory/marathon-python/tree/0.2.9) (2014-08-04) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.2.7...0.2.9) + +## [0.2.7](https://github.com/thefactory/marathon-python/tree/0.2.7) (2014-07-24) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.2.6...0.2.7) + +## [0.2.6](https://github.com/thefactory/marathon-python/tree/0.2.6) (2014-07-24) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.2.5...0.2.6) + +**Merged pull requests:** + +- Updated README.md with correction to create\_app args [\#2](https://github.com/thefactory/marathon-python/pull/2) ([rasathus](https://github.com/rasathus)) + +## [0.2.5](https://github.com/thefactory/marathon-python/tree/0.2.5) (2014-07-02) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.2.3...0.2.5) + +**Merged pull requests:** + +- allowing stagedAt and startedAt keys to be null [\#1](https://github.com/thefactory/marathon-python/pull/1) ([Codeacious](https://github.com/Codeacious)) + +## [0.2.3](https://github.com/thefactory/marathon-python/tree/0.2.3) (2014-06-02) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.2.0...0.2.3) + +## [0.2.0](https://github.com/thefactory/marathon-python/tree/0.2.0) (2014-04-28) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.1.1...0.2.0) + +## [0.1.1](https://github.com/thefactory/marathon-python/tree/0.1.1) (2014-04-23) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.1.0...0.1.1) + +## [0.1.0](https://github.com/thefactory/marathon-python/tree/0.1.0) (2014-04-23) + + +\* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)* \ No newline at end of file diff --git a/Makefile b/Makefile index 23f347e..4ad278f 100644 --- a/Makefile +++ b/Makefile @@ -14,5 +14,6 @@ package: clean publish: package pip install twine twine upload dist/* + github_changelog_generator .PHONY: itests test clean package publish From f0062b3b72b9dd6674c97b21b46685685a7c78d7 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Fri, 20 Nov 2015 07:55:26 -0800 Subject: [PATCH 008/292] Release 0.7.4 --- CHANGELOG.md | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1dc3eb7..98c71ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.7.4 (2015-10-20) + +- Marathon 11 Support [\#68](https://github.com/thefactory/marathon-python/pull/68) ([solarkennedy](https://github.com/solarkennedy)) + ## 0.7.2 (2015-09-17) Support for Marathon 0.9.1 diff --git a/setup.py b/setup.py index fea05df..26c1c88 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='marathon', - version='0.7.3', + version='0.7.4', description='Marathon Client Library', long_description="""Python interface to the Mesos Marathon REST API.""", author='Mike Babineau', From 77a8fa97f27f032af7ccbca12f9ec7d4a09e4141 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Sat, 21 Nov 2015 08:32:02 -0800 Subject: [PATCH 009/292] Update the changelog again to account for 0.7.4 --- CHANGELOG.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 863af96..20964bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,7 @@ # Change Log -## [Unreleased](https://github.com/thefactory/marathon-python/tree/HEAD) - -[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.7.3...HEAD) +## [0.7.4](https://github.com/thefactory/marathon-python/tree/0.7.4) (2015-11-20) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.7.3...0.7.4) **Merged pull requests:** @@ -223,4 +222,4 @@ ## [0.1.0](https://github.com/thefactory/marathon-python/tree/0.1.0) (2014-04-23) -\* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)* +\* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)* \ No newline at end of file From b49ea7645a4fe57f0cf4d2b97a6599f027d5829c Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Wed, 2 Dec 2015 16:11:38 -0800 Subject: [PATCH 010/292] Added test for killing tasks on an app. --- itests/Dockerfile | 2 +- itests/marathon_python.feature | 11 ++++++----- itests/steps/marathon_steps.py | 9 ++++++++- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/itests/Dockerfile b/itests/Dockerfile index 8cf69f3..5135abb 100644 --- a/itests/Dockerfile +++ b/itests/Dockerfile @@ -3,7 +3,7 @@ RUN apt-get install -y software-properties-common RUN add-apt-repository ppa:webupd8team/java RUN echo "debconf shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections RUN echo "debconf shared/accepted-oracle-license-v1-1 seen true" | debconf-set-selections -RUN apt-get update && apt-get -y install lsb-release oracle-java8-installer java8-runtime-headless +RUN apt-get update && apt-get -y install lsb-release oracle-java8-installer # Setup ADD ./marathon-version /root/marathon-version diff --git a/itests/marathon_python.feature b/itests/marathon_python.feature index 76583b6..df131a7 100644 --- a/itests/marathon_python.feature +++ b/itests/marathon_python.feature @@ -2,14 +2,15 @@ Feature: marathon-python can create and list marathon apps Scenario: Metadata can be fetched Given a working marathon instance - Then we get the marathon instance's info + Then we get the marathon instance's info Scenario: Trivial apps can be deployed Given a working marathon instance - When we create a trivial new app - Then we should see the trivial app running via the marathon api + When we create a trivial new app + Then we should see the trivial app running via the marathon api + And we should be able to kill the tasks Scenario: Complex apps can be deployed Given a working marathon instance - When we create a complex new app - Then we should see the complex app running via the marathon api + When we create a complex new app + Then we should see the complex app running via the marathon api diff --git a/itests/steps/marathon_steps.py b/itests/steps/marathon_steps.py index 19befe7..00e8676 100644 --- a/itests/steps/marathon_steps.py +++ b/itests/steps/marathon_steps.py @@ -3,7 +3,6 @@ import marathon from behave import given, when, then -import mock from itest_utils import get_marathon_connection_string sys.path.append('../') @@ -29,6 +28,14 @@ def create_trivial_new_app(context): context.client.create_app('test-trivial-app', marathon.MarathonApp(cmd='sleep 100', mem=16, cpus=1)) +@then(u'we should be able to kill the tasks') +def kill_a_task(context): + time.sleep(5) + app = context.client.get_app('test-trivial-app') + tasks = app.tasks + context.client.kill_task(app_id='test-trivial-app', task_id=tasks[0].id, scale=True) + + @when(u'we create a complex new app') def create_complex_new_app_with_unicode(context): app_config = { From b148cfaf557bb26a49eacb9322de3f0f39e348a9 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Thu, 3 Dec 2015 14:35:54 -0800 Subject: [PATCH 011/292] Catch a key-error when Marathon doesn't provide task information when killing tasks --- marathon/client.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index 0b3b4bb..7f7a835 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -373,7 +373,14 @@ def batch(iterable, size): params = {'scale': scale} if host: params['host'] = host response = self._do_request('DELETE', '/v2/apps/{app_id}/tasks'.format(app_id=app_id), params) - return self._parse_response(response, MarathonTask, is_list=True, resource_name='tasks') + try: + return self._parse_response(response, MarathonTask, is_list=True, resource_name='tasks') + except KeyError: + # Marathon is inconsistent about what type of object it returns on the multi + # task deletion endpoint, depending on the version of Marathon. See: + # https://github.com/mesosphere/marathon/blob/06a6f763a75fb6d652b4f1660685ae234bd15387/src/main/scala/mesosphere/marathon/api/v2/AppTasksResource.scala#L88-L95 + # TODO: Parse as a deployment if scale==True when only supporting Marathon >=0.11 + return None else: # Terminate in batches tasks = self.list_tasks(app_id, host=host) if host else self.list_tasks(app_id) @@ -412,7 +419,14 @@ def kill_task(self, app_id, task_id, scale=False): params = {'scale': scale} response = self._do_request('DELETE', '/v2/apps/{app_id}/tasks/{task_id}' .format(app_id=app_id, task_id=task_id), params) - return self._parse_response(response, MarathonTask, resource_name='task') + try: + return self._parse_response(response, MarathonTask, resource_name='task') + except KeyError: + # Marathon is inconsistent about what type of object it returns on the single + # task deletion endpoint, depending on the version of Marathon. See: + # https://github.com/mesosphere/marathon/blob/06a6f763a75fb6d652b4f1660685ae234bd15387/src/main/scala/mesosphere/marathon/api/v2/AppTasksResource.scala#L112-L119 + # TODO: Parse as a deployment if scale==True when only supporting Marathon >=0.11 + return None def list_versions(self, app_id): """List the versions of an app. From 318017e7b2d2603f6ade1479583ba8b1901e465e Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Fri, 4 Dec 2015 11:16:56 -0800 Subject: [PATCH 012/292] Make kill_task and kill_tasks return the right objects --- marathon/client.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index 7f7a835..525e45d 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -373,14 +373,13 @@ def batch(iterable, size): params = {'scale': scale} if host: params['host'] = host response = self._do_request('DELETE', '/v2/apps/{app_id}/tasks'.format(app_id=app_id), params) - try: + # Marathon is inconsistent about what type of object it returns on the multi + # task deletion endpoint, depending on the version of Marathon. See: + # https://github.com/mesosphere/marathon/blob/06a6f763a75fb6d652b4f1660685ae234bd15387/src/main/scala/mesosphere/marathon/api/v2/AppTasksResource.scala#L88-L95 + if response.json().has_key("tasks"): return self._parse_response(response, MarathonTask, is_list=True, resource_name='tasks') - except KeyError: - # Marathon is inconsistent about what type of object it returns on the multi - # task deletion endpoint, depending on the version of Marathon. See: - # https://github.com/mesosphere/marathon/blob/06a6f763a75fb6d652b4f1660685ae234bd15387/src/main/scala/mesosphere/marathon/api/v2/AppTasksResource.scala#L88-L95 - # TODO: Parse as a deployment if scale==True when only supporting Marathon >=0.11 - return None + else: + return response.json() else: # Terminate in batches tasks = self.list_tasks(app_id, host=host) if host else self.list_tasks(app_id) @@ -419,14 +418,13 @@ def kill_task(self, app_id, task_id, scale=False): params = {'scale': scale} response = self._do_request('DELETE', '/v2/apps/{app_id}/tasks/{task_id}' .format(app_id=app_id, task_id=task_id), params) - try: - return self._parse_response(response, MarathonTask, resource_name='task') - except KeyError: - # Marathon is inconsistent about what type of object it returns on the single - # task deletion endpoint, depending on the version of Marathon. See: - # https://github.com/mesosphere/marathon/blob/06a6f763a75fb6d652b4f1660685ae234bd15387/src/main/scala/mesosphere/marathon/api/v2/AppTasksResource.scala#L112-L119 - # TODO: Parse as a deployment if scale==True when only supporting Marathon >=0.11 - return None + # Marathon is inconsistent about what type of object it returns on the multi + # task deletion endpoint, depending on the version of Marathon. See: + # https://github.com/mesosphere/marathon/blob/06a6f763a75fb6d652b4f1660685ae234bd15387/src/main/scala/mesosphere/marathon/api/v2/AppTasksResource.scala#L88-L95 + if response.json().has_key("task"): + return self._parse_response(response, MarathonTask, is_list=False, resource_name='task') + else: + return response.json() def list_versions(self, app_id): """List the versions of an app. From 150c4d0836e482bed247c77ff59317626b11ed85 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Tue, 8 Dec 2015 18:02:18 -0800 Subject: [PATCH 013/292] Release 0.7.5 for official Marathon 11 support --- CHANGELOG.md | 10 ++++++++++ README.md | 6 ++++-- setup.py | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20964bd..fa39a37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,20 @@ # Change Log +## [Unreleased](https://github.com/thefactory/marathon-python/tree/HEAD) + +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.7.4...HEAD) + +**Merged pull requests:** + +- Added tests for killing tasks on an app [\#72](https://github.com/thefactory/marathon-python/pull/72) ([solarkennedy](https://github.com/solarkennedy)) +- Provide proper compatability support for str/unicode in py3 [\#57](https://github.com/thefactory/marathon-python/pull/57) ([mattrobenolt](https://github.com/mattrobenolt)) + ## [0.7.4](https://github.com/thefactory/marathon-python/tree/0.7.4) (2015-11-20) [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.7.3...0.7.4) **Merged pull requests:** +- Use automatic changelog generation [\#69](https://github.com/thefactory/marathon-python/pull/69) ([solarkennedy](https://github.com/solarkennedy)) - Marathon 11 Support [\#68](https://github.com/thefactory/marathon-python/pull/68) ([solarkennedy](https://github.com/solarkennedy)) ## [0.7.3](https://github.com/thefactory/marathon-python/tree/0.7.3) (2015-11-12) diff --git a/README.md b/README.md index 0036f2c..b5f4bd6 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,11 @@ This is a Python library for interfacing with [Marathon](https://github.com/meso marathon-python is primarily developed against Marathon 0.8.x (see [Marathon docs](https://mesosphere.github.io/marathon/)) -* For Marathon 0.8.x-0.9.x, use the latest release +* For Marathon greater than 0.11.x: Not supported yet. Please submit a patch! +* For Marathon 0.8.x-0.11.x, use marathon-python 0.7.5 +* For Marathon 0.8.x-0.9.x, use marathon-python 0.6.11 - 0.7.4 * For Marathon 0.7.x, use marathon-python 0.6.10 -* For older versions, please see `CHANGELOG.md` +* For all version changes, please see `CHANGELOG.md` ## Installation diff --git a/setup.py b/setup.py index 26c1c88..fb8d550 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='marathon', - version='0.7.4', + version='0.7.5', description='Marathon Client Library', long_description="""Python interface to the Mesos Marathon REST API.""", author='Mike Babineau', From add03f0be68de07d9c73209c61b7629bdad8b0bb Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Wed, 9 Dec 2015 10:12:36 -0800 Subject: [PATCH 014/292] Updated changelog for 0.7.5 release --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa39a37..aa4ff1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,11 @@ # Change Log -## [Unreleased](https://github.com/thefactory/marathon-python/tree/HEAD) - -[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.7.4...HEAD) +## [0.7.5](https://github.com/thefactory/marathon-python/tree/0.7.5) (2015-12-09) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.7.4...0.7.5) **Merged pull requests:** +- Release 0.7.5 for official Marathon 11 support [\#73](https://github.com/thefactory/marathon-python/pull/73) ([solarkennedy](https://github.com/solarkennedy)) - Added tests for killing tasks on an app [\#72](https://github.com/thefactory/marathon-python/pull/72) ([solarkennedy](https://github.com/solarkennedy)) - Provide proper compatability support for str/unicode in py3 [\#57](https://github.com/thefactory/marathon-python/pull/57) ([mattrobenolt](https://github.com/mattrobenolt)) From c1e1a7de8442a5548ae3cee2994b884c5abb8af4 Mon Sep 17 00:00:00 2001 From: Guanglu Guo Date: Thu, 10 Dec 2015 17:21:37 +0800 Subject: [PATCH 015/292] Use the /v2/tasks/delete endpoint for task kill --- ...n_python.feature => marathon_apps.feature} | 1 - itests/marathon_tasks.feature | 13 ++++++++++++ itests/steps/marathon_steps.py | 21 ++++++++++++++++++- marathon/client.py | 14 +++++++++++++ 4 files changed, 47 insertions(+), 2 deletions(-) rename itests/{marathon_python.feature => marathon_apps.feature} (92%) create mode 100644 itests/marathon_tasks.feature diff --git a/itests/marathon_python.feature b/itests/marathon_apps.feature similarity index 92% rename from itests/marathon_python.feature rename to itests/marathon_apps.feature index df131a7..7871be9 100644 --- a/itests/marathon_python.feature +++ b/itests/marathon_apps.feature @@ -8,7 +8,6 @@ Feature: marathon-python can create and list marathon apps Given a working marathon instance When we create a trivial new app Then we should see the trivial app running via the marathon api - And we should be able to kill the tasks Scenario: Complex apps can be deployed Given a working marathon instance diff --git a/itests/marathon_tasks.feature b/itests/marathon_tasks.feature new file mode 100644 index 0000000..c3e0cab --- /dev/null +++ b/itests/marathon_tasks.feature @@ -0,0 +1,13 @@ +Feature: marathon-python can operate marathon app tasks + + Scenario: App tasks can be killed + Given a working marathon instance + When we create a trivial new app + And we wait the trivial app deployment finish + Then we should be able to kill the tasks + + Scenario: A list of app tasks can be killed + Given a working marathon instance + When we create a trivial new app + And we wait the trivial app deployment finish + Then we should be able to kill the #0,1,2 tasks of the trivial app diff --git a/itests/steps/marathon_steps.py b/itests/steps/marathon_steps.py index 00e8676..1fa5be4 100644 --- a/itests/steps/marathon_steps.py +++ b/itests/steps/marathon_steps.py @@ -25,7 +25,7 @@ def get_marathon_info(context): @when(u'we create a trivial new app') def create_trivial_new_app(context): - context.client.create_app('test-trivial-app', marathon.MarathonApp(cmd='sleep 100', mem=16, cpus=1)) + context.client.create_app('test-trivial-app', marathon.MarathonApp(cmd='sleep 3600', mem=16, cpus=1, instances=5)) @then(u'we should be able to kill the tasks') @@ -76,3 +76,22 @@ def create_complex_new_app_with_unicode(context): def see_complext_app_running(context, which): print(context.client.list_apps()) assert context.client.get_app('test-%s-app' % which) + + +@when(u'we wait the {which} app deployment finish') +def wait_deployment_finish(context, which): + while True: + time.sleep(1) + app = context.client.get_app('test-%s-app' % which, embed_tasks=True) + if not app.deployments: + break + + +@then(u'we should be able to kill the #{to_kill} tasks of the {which} app') +def kill_tasks(context, to_kill, which): + app_tasks = context.client.get_app('test-%s-app' % which, embed_tasks=True).tasks + + index_to_kill = eval("[" + to_kill + "]") + task_to_kill = [app_tasks[index].id for index in index_to_kill] + + context.client.kill_given_tasks(task_to_kill) diff --git a/marathon/client.py b/marathon/client.py index 525e45d..0e4d6bf 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -350,6 +350,20 @@ def list_tasks(self, app_id=None, **kwargs): return tasks + def kill_given_tasks(self, task_ids, scale=False): + """Kill a list of given tasks. + + :param list[str] task_ids: tasks to kill + :param bool scale: if true, scale down the app by the number of tasks killed + + :return: True on success + :rtype: bool + """ + params = {'scale': scale} + data = json.dumps({"ids": task_ids}) + response = self._do_request('POST', '/v2/tasks/delete', params=params, data=data) + return response == 200 + def kill_tasks(self, app_id, scale=False, host=None, batch_size=0, batch_delay=0): """Kill all tasks belonging to app. From 1d0d45e95e519acaa5f0e87675afddcdec3a1260 Mon Sep 17 00:00:00 2001 From: Guanglu Guo Date: Fri, 27 Nov 2015 17:09:02 +0800 Subject: [PATCH 016/292] Modify list_apps so user can input app_id without the starting slash --- itests/marathon_tasks.feature | 6 ++++++ itests/steps/marathon_steps.py | 7 +++++++ marathon/client.py | 2 +- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/itests/marathon_tasks.feature b/itests/marathon_tasks.feature index c3e0cab..f9b68f4 100644 --- a/itests/marathon_tasks.feature +++ b/itests/marathon_tasks.feature @@ -1,5 +1,11 @@ Feature: marathon-python can operate marathon app tasks + Scenario: App tasks can be listed + Given a working marathon instance + When we create a trivial new app + And we wait the trivial app deployment finish + Then we should be able to list tasks of the trivial app + Scenario: App tasks can be killed Given a working marathon instance When we create a trivial new app diff --git a/itests/steps/marathon_steps.py b/itests/steps/marathon_steps.py index 1fa5be4..cf37d7b 100644 --- a/itests/steps/marathon_steps.py +++ b/itests/steps/marathon_steps.py @@ -95,3 +95,10 @@ def kill_tasks(context, to_kill, which): task_to_kill = [app_tasks[index].id for index in index_to_kill] context.client.kill_given_tasks(task_to_kill) + + +@then(u'we should be able to list tasks of the {which} app') +def list_tasks(context, which): + app = context.client.get_app('test-%s-app' % which) + tasks = context.client.list_tasks('test-%s-app' % which) + assert len(tasks) == app.instances diff --git a/marathon/client.py b/marathon/client.py index 0e4d6bf..d76cd04 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -342,7 +342,7 @@ def list_tasks(self, app_id=None, **kwargs): response = self._do_request('GET', '/v2/tasks') tasks = self._parse_response(response, MarathonTask, is_list=True, resource_name='tasks') if app_id: - tasks = [task for task in tasks if task.app_id == app_id] + tasks = [task for task in tasks if task.app_id.lstrip('/') == app_id.lstrip('/')] [setattr(t, 'app_id', app_id) for t in tasks if app_id and t.app_id is None] for k, v in kwargs.items(): From 8650eeccd1aa30d7d070d86bebbcc8624a9677fc Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Thu, 28 Jan 2016 19:14:23 -0800 Subject: [PATCH 017/292] Try to pin to mesos 0.23 and see what happens --- itests/install-marathon.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/itests/install-marathon.sh b/itests/install-marathon.sh index 208fb36..4c44f79 100755 --- a/itests/install-marathon.sh +++ b/itests/install-marathon.sh @@ -23,7 +23,7 @@ sudo apt-get -y purge oracle-java7-installer sudo update-java-alternatives -s java-8-oracle sudo apt-get install oracle-java8-set-default -sudo apt-get -y --force-yes install mesos marathon=$MARATHONVERSION* +sudo apt-get -y --force-yes install mesos=0.23.* marathon=$MARATHONVERSION* # WTF MARATHON? # Why does the precise version have java7 hardcoded if it requires java8? From 68d8a9fabaa0eb65b9695d2c2f9b78338294959b Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Thu, 28 Jan 2016 19:39:31 -0800 Subject: [PATCH 018/292] Try to shorten the hostname to work around travis-ci/travis-ci#5227 --- .travis.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.travis.yml b/.travis.yml index c7a23be..0b9ad3c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,3 +20,10 @@ script: - /etc/init.d/zookeeper start - ./itests/start-marathon.sh & - make itests + +# Work around travis-ci/travis-ci#5227 +before_install: + - cat /etc/hosts # optionally check the content *before* + - sudo hostname "$(hostname | cut -c1-63)" + - sed -e "s/^\\(127\\.0\\.0\\.1.*\\)/\\1 $(hostname | cut -c1-63)/" /etc/hosts | sudo tee /etc/hosts + - cat /etc/hosts # optionally check the content *after* From 42e0531504cb08ba3d86172026b2f9f1010c04f2 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Thu, 28 Jan 2016 20:51:46 -0800 Subject: [PATCH 019/292] Only run on travisci legacy infrastructure for now --- .travis.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0b9ad3c..ce296ed 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,6 @@ language: python python: - 2.7 - 3.4 -sudo: true install: - pip install tox script: @@ -27,3 +26,8 @@ before_install: - sudo hostname "$(hostname | cut -c1-63)" - sed -e "s/^\\(127\\.0\\.0\\.1.*\\)/\\1 $(hostname | cut -c1-63)/" /etc/hosts | sudo tee /etc/hosts - cat /etc/hosts # optionally check the content *after* + +# Work around and avoid travis on gce as it is dog slow +sudo: true +dist: precise +group: legacy From 91583c352bb4c3ead7434e4988ba035f84b38e6f Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Thu, 28 Jan 2016 21:00:05 -0800 Subject: [PATCH 020/292] Try harder to get on bluebox infrastructure --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ce296ed..b8b40f5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,6 @@ before_install: - cat /etc/hosts # optionally check the content *after* # Work around and avoid travis on gce as it is dog slow -sudo: true +sudo: required dist: precise group: legacy From 38c38d046d5493828873cd67e837b3132fea192c Mon Sep 17 00:00:00 2001 From: Corentin Chary Date: Fri, 29 Jan 2016 12:51:22 +0100 Subject: [PATCH 021/292] Don't drop 0s when transforming to JSON This fixes issue #75 --- marathon/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/marathon/util.py b/marathon/util.py index 22a7451..eacab87 100644 --- a/marathon/util.py +++ b/marathon/util.py @@ -45,9 +45,9 @@ def default(self, obj): if isinstance(obj, collections.Iterable) and not is_stringy(obj): try: - return {k: self.default(v) for k, v in obj.items() if (v or v is False)} + return {k: self.default(v) for k, v in obj.items() if (v or v is False or v is 0)} except AttributeError: - return [self.default(e) for e in obj if (e or e is False)] + return [self.default(e) for e in obj if (e or e is False or e is 0)] return obj From 2f616eeb202b257110fccb971e39ff7986d9979a Mon Sep 17 00:00:00 2001 From: Guanglu Guo Date: Sat, 19 Dec 2015 12:22:51 +0800 Subject: [PATCH 022/292] Change MarathonDockerContainer.parameters to type list --- itests/steps/marathon_steps.py | 3 +++ marathon/models/container.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/itests/steps/marathon_steps.py b/itests/steps/marathon_steps.py index cf37d7b..24b0eb6 100644 --- a/itests/steps/marathon_steps.py +++ b/itests/steps/marathon_steps.py @@ -45,6 +45,9 @@ def create_complex_new_app_with_unicode(context): 'portMappings': [{'protocol': 'tcp', 'containerPort': 8888, 'hostPort': 0}], 'image': u'localhost/fake_docker_url', 'network': 'BRIDGE', + 'parameters': [ + {'key': 'add-host', 'value': 'google-public-dns-a.google.com:8.8.8.8'}, + ], }, 'volumes': [{'hostPath': u'/etc/stuff', 'containerPath': u'/etc/stuff', 'mode': 'RO'}], }, diff --git a/marathon/models/container.py b/marathon/models/container.py index af0b177..3e88558 100644 --- a/marathon/models/container.py +++ b/marathon/models/container.py @@ -38,7 +38,7 @@ class MarathonDockerContainer(MarathonObject): :param str network: :param port_mappings: :type port_mappings: list[:class:`marathon.models.container.MarathonContainerPortMapping`] or list[dict] - :param dict parameters: + :param list[dict] parameters: :param bool privileged: run container in privileged mode :param bool force_pull_image: Force a docker pull before launching """ @@ -57,7 +57,7 @@ def __init__(self, image=None, network='HOST', port_mappings=None, parameters=No pm if isinstance(pm, MarathonContainerPortMapping) else MarathonContainerPortMapping().from_json(pm) for pm in (port_mappings or []) ] - self.parameters = parameters or {} + self.parameters = parameters or [] self.privileged = privileged or False self.force_pull_image = force_pull_image or False From c4c2dd736400ebbbe04088aa0ea9e6f2deafc97c Mon Sep 17 00:00:00 2001 From: Ben Sanchez Date: Thu, 28 Jan 2016 10:52:49 -0500 Subject: [PATCH 023/292] Make meploy work with marathon 0.14.0 --- marathon/models/app.py | 3 ++- marathon/models/task.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/marathon/models/app.py b/marathon/models/app.py index 6bd3d8d..65c7534 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -75,7 +75,8 @@ def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, executor=None, health_checks=None, id=None, instances=None, labels=None, last_task_failure=None, max_launch_delay_seconds=None, mem=None, ports=None, require_ports=None, store_urls=None, task_rate_limit=None, tasks=None, tasks_running=None, tasks_staged=None, tasks_healthy=None, - tasks_unhealthy=None, upgrade_strategy=None, uris=None, user=None, version=None, version_info=None): + tasks_unhealthy=None, upgrade_strategy=None, uris=None, user=None, version=None, version_info=None, + ip_address=None): # self.args = args or [] self.accepted_resource_roles = accepted_resource_roles diff --git a/marathon/models/task.py b/marathon/models/task.py index 764b1b9..f4dad71 100644 --- a/marathon/models/task.py +++ b/marathon/models/task.py @@ -24,7 +24,7 @@ class MarathonTask(MarathonResource): DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ' def __init__(self, app_id=None, health_check_results=None, host=None, id=None, ports=None, service_ports=None, - slave_id=None, staged_at=None, started_at=None, version=None): + slave_id=None, staged_at=None, started_at=None, version=None, ip_addresses=None): self.app_id = app_id self.health_check_results = health_check_results or [] self.health_check_results = [ From 2423b042959fa8c2551d7077f25d7da8c4dbd02b Mon Sep 17 00:00:00 2001 From: Denis Ilyn Date: Thu, 28 Jan 2016 17:25:36 -0800 Subject: [PATCH 024/292] NOJIRA marathon 0.14.0 support --- marathon/models/base.py | 2 +- marathon/models/task.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/marathon/models/base.py b/marathon/models/base.py index 01a528a..773c563 100644 --- a/marathon/models/base.py +++ b/marathon/models/base.py @@ -21,7 +21,7 @@ def json_repr(self, minimal=False): :rtype: dict """ if minimal: - return {to_camel_case(k):v for k,v in vars(self).items() if (v or v == False)} + return {to_camel_case(k):v for k,v in vars(self).items() if (v or v == False or v == 0 )} else: return {to_camel_case(k):v for k,v in vars(self).items()} diff --git a/marathon/models/task.py b/marathon/models/task.py index f4dad71..70c2b10 100644 --- a/marathon/models/task.py +++ b/marathon/models/task.py @@ -24,7 +24,7 @@ class MarathonTask(MarathonResource): DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ' def __init__(self, app_id=None, health_check_results=None, host=None, id=None, ports=None, service_ports=None, - slave_id=None, staged_at=None, started_at=None, version=None, ip_addresses=None): + slave_id=None, staged_at=None, started_at=None, version=None, ip_addresses=[] ): self.app_id = app_id self.health_check_results = health_check_results or [] self.health_check_results = [ From e4c30788b3d7d8f90c86bbdf8cb95e35b894d448 Mon Sep 17 00:00:00 2001 From: "denis.ilyn" Date: Thu, 28 Jan 2016 18:43:37 -0800 Subject: [PATCH 025/292] Update util.py --- marathon/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/marathon/util.py b/marathon/util.py index eacab87..e88fb3a 100644 --- a/marathon/util.py +++ b/marathon/util.py @@ -45,9 +45,9 @@ def default(self, obj): if isinstance(obj, collections.Iterable) and not is_stringy(obj): try: - return {k: self.default(v) for k, v in obj.items() if (v or v is False or v is 0)} + return {k: self.default(v) for k, v in obj.items() if (v or v in (False, 0))} except AttributeError: - return [self.default(e) for e in obj if (e or e is False or e is 0)] + return [self.default(e) for e in obj if (e or e in (False, 0))] return obj From efe01121b75c5dd49a6bb406dde4ee8fba33f382 Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Thu, 11 Feb 2016 15:31:43 +0200 Subject: [PATCH 026/292] Update travis build matrix - Include only latest minor version from every release - Add 0.13 & 0.14 --- .travis.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index b8b40f5..9f1fda6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,10 @@ env: - - MARATHONVERSION: 0.8.1 - MARATHONVERSION: 0.8.2 - - MARATHONVERSION: 0.9.0 - MARATHONVERSION: 0.9.1 - - MARATHONVERSION: 0.10.0 - MARATHONVERSION: 0.10.1 - - MARATHONVERSION: 0.11.0 + - MARATHONVERSION: 0.11.1 + - MARATHONVERSION: 0.13.1 + - MARATHONVERSION: 0.14.1 language: python python: From 920219e9ccd5685b03bea37efdcff6974ae96961 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Thu, 11 Feb 2016 19:12:47 -0800 Subject: [PATCH 027/292] Release 0.7.6 --- CHANGELOG.md | 19 +++++++++++++++++++ README.md | 2 +- setup.py | 2 +- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa4ff1a..1d52828 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Change Log +## [0.7.6](https://github.com/thefactory/marathon-python/tree/0.7.6) (2016-02-11) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.7.5...0.7.6) + +**Closed issues:** + +- MarathonClient is not compatible with marathon 0.14.1-1.0.455.ubuntu1404 [\#81](https://github.com/thefactory/marathon-python/issues/81) +- Zero values in apps/groups doesnt work [\#75](https://github.com/thefactory/marathon-python/issues/75) +- does marathon-python supports marathon Version 0.11.1? [\#66](https://github.com/thefactory/marathon-python/issues/66) +- Why MarathonDockerContainer.parameters is defined as dict? [\#47](https://github.com/thefactory/marathon-python/issues/47) + +**Merged pull requests:** + +- 0.14 support [\#85](https://github.com/thefactory/marathon-python/pull/85) ([itamaro](https://github.com/itamaro)) +- Change MarathonDockerContainer.parameters to type list [\#83](https://github.com/thefactory/marathon-python/pull/83) ([fengyehong](https://github.com/fengyehong)) +- Don't drop 0s when transforming to JSON [\#79](https://github.com/thefactory/marathon-python/pull/79) ([iksaif](https://github.com/iksaif)) +- Itest fixes and stick to legacy travis infrastructure for now. [\#78](https://github.com/thefactory/marathon-python/pull/78) ([solarkennedy](https://github.com/solarkennedy)) +- Modify list\_apps so user can input app\_id without the starting slash [\#71](https://github.com/thefactory/marathon-python/pull/71) ([fengyehong](https://github.com/fengyehong)) +- Use the /v2/tasks/delete endpoint for taskkill [\#67](https://github.com/thefactory/marathon-python/pull/67) ([fengyehong](https://github.com/fengyehong)) + ## [0.7.5](https://github.com/thefactory/marathon-python/tree/0.7.5) (2015-12-09) [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.7.4...0.7.5) diff --git a/README.md b/README.md index b5f4bd6..df1ecf7 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ This is a Python library for interfacing with [Marathon](https://github.com/meso marathon-python is primarily developed against Marathon 0.8.x (see [Marathon docs](https://mesosphere.github.io/marathon/)) -* For Marathon greater than 0.11.x: Not supported yet. Please submit a patch! +* For Marathon greater than 0.14.x: Experimental support in 0.7.6 * For Marathon 0.8.x-0.11.x, use marathon-python 0.7.5 * For Marathon 0.8.x-0.9.x, use marathon-python 0.6.11 - 0.7.4 * For Marathon 0.7.x, use marathon-python 0.6.10 diff --git a/setup.py b/setup.py index fb8d550..e7df8ec 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='marathon', - version='0.7.5', + version='0.7.6', description='Marathon Client Library', long_description="""Python interface to the Mesos Marathon REST API.""", author='Mike Babineau', From 80a840b772060daf1a6430b22584bfbeb3af39a5 Mon Sep 17 00:00:00 2001 From: Burak Bostancioglu Date: Sun, 14 Feb 2016 20:45:06 +0000 Subject: [PATCH 028/292] a small fix for fetching apps for marathon v0.15 --- marathon/models/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marathon/models/app.py b/marathon/models/app.py index 65c7534..749e1f4 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -76,7 +76,7 @@ def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, max_launch_delay_seconds=None, mem=None, ports=None, require_ports=None, store_urls=None, task_rate_limit=None, tasks=None, tasks_running=None, tasks_staged=None, tasks_healthy=None, tasks_unhealthy=None, upgrade_strategy=None, uris=None, user=None, version=None, version_info=None, - ip_address=None): + ip_address=None, fetch=None): # self.args = args or [] self.accepted_resource_roles = accepted_resource_roles From 9727b737817c1d1eec63c0ba6f3e68bfafd3d069 Mon Sep 17 00:00:00 2001 From: Burak Bostancioglu Date: Tue, 16 Feb 2016 17:37:07 +0000 Subject: [PATCH 029/292] restart endpoint --- marathon/client.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/marathon/client.py b/marathon/client.py index d76cd04..ad06faf 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -145,6 +145,18 @@ def get_app(self, app_id, embed_tasks=False): response = self._do_request('GET', '/v2/apps/{app_id}'.format(app_id=app_id), params=params) return self._parse_response(response, MarathonApp, resource_name='app') + def restart_app(self, app_id, force=False): + """ + Restarts given application by app_id + :param str app_id: application ID + :param bool force: apply even if a deployment is in progress + :returns: a dict containing the deployment id and version + :rtype: dict + """ + params = {'force': force} + response = self._do_request('POST', '/v2/apps/{appId}/restart'.format(app_id=app_id), params=params) + return response.json() + def update_app(self, app_id, app, force=False, minimal=True): """Update an app. From 44c7c31a6f32979c3fddee8177ee0592d9f3b862 Mon Sep 17 00:00:00 2001 From: Burak Bostancioglu Date: Tue, 16 Feb 2016 17:51:13 +0000 Subject: [PATCH 030/292] fix for typo --- marathon/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marathon/client.py b/marathon/client.py index ad06faf..992d0f8 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -154,7 +154,7 @@ def restart_app(self, app_id, force=False): :rtype: dict """ params = {'force': force} - response = self._do_request('POST', '/v2/apps/{appId}/restart'.format(app_id=app_id), params=params) + response = self._do_request('POST', '/v2/apps/{app_id}/restart'.format(app_id=app_id), params=params) return response.json() def update_app(self, app_id, app, force=False, minimal=True): From 44d411b282e7aa770fb7ea7711ae72365bbdfc16 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Mon, 29 Feb 2016 07:52:04 -0800 Subject: [PATCH 031/292] Release 0.7.7 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e7df8ec..a41a6f8 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='marathon', - version='0.7.6', + version='0.7.7', description='Marathon Client Library', long_description="""Python interface to the Mesos Marathon REST API.""", author='Mike Babineau', From bd6303e713bf3c1f3b795556669dcec611363f7b Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Mon, 29 Feb 2016 07:54:46 -0800 Subject: [PATCH 032/292] Release 0.7.7 changlog --- CHANGELOG.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d52828..64a5435 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,14 @@ # Change Log -## [0.7.6](https://github.com/thefactory/marathon-python/tree/0.7.6) (2016-02-11) +## [0.7.7](https://github.com/thefactory/marathon-python/tree/0.7.7) (2016-02-29) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.7.6...0.7.7) + +**Merged pull requests:** + +- restart endpoint [\#88](https://github.com/thefactory/marathon-python/pull/88) ([burakbostancioglu](https://github.com/burakbostancioglu)) +- a small fix for fetching apps for marathon v0.15 [\#87](https://github.com/thefactory/marathon-python/pull/87) ([burakbostancioglu](https://github.com/burakbostancioglu)) + +## [0.7.6](https://github.com/thefactory/marathon-python/tree/0.7.6) (2016-02-12) [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.7.5...0.7.6) **Closed issues:** From 693a9612668bc382b6ca56f3ac21a82172d2a326 Mon Sep 17 00:00:00 2001 From: Bekir Dogan Date: Tue, 8 Mar 2016 03:50:25 +0000 Subject: [PATCH 033/292] update for v2/queue and v2/apps?embed=apps.taskStats --- marathon/client.py | 16 ++++++- marathon/models/app.py | 95 +++++++++++++++++++++++++++++++++++++++- marathon/models/queue.py | 28 +++++++++++- 3 files changed, 135 insertions(+), 4 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index 992d0f8..9ec9c77 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -10,7 +10,7 @@ import requests.exceptions import marathon -from .models import MarathonApp, MarathonDeployment, MarathonGroup, MarathonInfo, MarathonTask, MarathonEndpoint +from .models import MarathonApp, MarathonDeployment, MarathonGroup, MarathonInfo, MarathonTask, MarathonEndpoint, MarathonQueueItem from .exceptions import InternalServerError, NotFoundError, MarathonHttpError, MarathonError @@ -105,13 +105,14 @@ def create_app(self, app_id, app): else: return False - def list_apps(self, cmd=None, embed_tasks=False, embed_failures=False, **kwargs): + def list_apps(self, cmd=None, embed_tasks=False, embed_failures=False, embed_task_stats=False, **kwargs): """List all apps. :param str app_id: application ID :param str cmd: if passed, only show apps with a matching `cmd` :param bool embed_tasks: embed tasks in result :param bool embed_failures: embed tasks and last task failure in result + :param bool embed_task_stats: embed task stats in result :param kwargs: arbitrary search filters :returns: list of applications @@ -125,6 +126,8 @@ def list_apps(self, cmd=None, embed_tasks=False, embed_failures=False, **kwargs) params['embed'] = 'apps.failures' elif embed_tasks: params['embed'] = 'apps.tasks' + elif embed_task_stats: + params['embed'] = 'apps.taskStats' response = self._do_request('GET', '/v2/apps', params=params) apps = self._parse_response(response, MarathonApp, is_list=True, resource_name='apps') @@ -518,6 +521,15 @@ def list_deployments(self): response = self._do_request('GET', '/v2/deployments') return self._parse_response(response, MarathonDeployment, is_list=True) + def list_queue(self): + """List all the tasks queued up or waiting to be scheduled. + + :returns: list of queue items + :rtype: list[:class:`marathon.models.queue.MarathonQueueItem`] + """ + response = self._do_request('GET', '/v2/queue') + return self._parse_response(response, MarathonQueueItem, is_list=True, resource_name='queue') + def delete_deployment(self, deployment_id, force=False): """Cancel a deployment. diff --git a/marathon/models/app.py b/marathon/models/app.py index 749e1f4..238faca 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -54,6 +54,8 @@ class MarathonApp(MarathonResource): :param str version: version id :param version_info: time of last scaling, last config change :type version_info: :class:`marathon.models.app.MarathonAppVersionInfo` or dict + :param task_stats: task statistics + :type task_stats: :class:`marathon.models.app.MarathonTaskStats` or dict :param dict labels """ @@ -76,7 +78,7 @@ def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, max_launch_delay_seconds=None, mem=None, ports=None, require_ports=None, store_urls=None, task_rate_limit=None, tasks=None, tasks_running=None, tasks_staged=None, tasks_healthy=None, tasks_unhealthy=None, upgrade_strategy=None, uris=None, user=None, version=None, version_info=None, - ip_address=None, fetch=None): + ip_address=None, fetch=None, task_stats=None): # self.args = args or [] self.accepted_resource_roles = accepted_resource_roles @@ -133,6 +135,8 @@ def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, self.version = version self.version_info = version_info if (isinstance(version_info, MarathonAppVersionInfo) or version_info is None) \ else MarathonAppVersionInfo.from_json(version_info) + self.task_stats = version_info if (isinstance(task_stats, MarathonTaskStats) or task_stats is None) \ + else MarathonTaskStats.from_json(task_stats) class MarathonHealthCheck(MarathonObject): @@ -229,3 +233,92 @@ def _to_datetime(self, timestamp): return timestamp else: return datetime.strptime(timestamp, self.DATETIME_FORMAT) + + +class MarathonTaskStats(MarathonObject): + """Marathon task statistics + + See https://mesosphere.github.io/marathon/docs/rest-api.html#taskstats-object-v0-11 + + :param started_after_last_scaling: contains statistics about all tasks that were started after the last scaling or restart operation. + :type started_after_last_scaling: :class:`marathon.models.app.MarathonTaskStatsType` or dict + :param with_latest_config: contains statistics about all tasks that run with the same config as the latest app version. + :type with_latest_config: :class:`marathon.models.app.MarathonTaskStatsType` or dict + :param with_outdated_config: contains statistics about all tasks that were started before the last config change which was not simply a restart or scaling operation. + :type with_outdated_config: :class:`marathon.models.app.MarathonTaskStatsType` or dict + :param total_summary: contains statistics about all tasks. + :type total_summary: :class:`marathon.models.app.MarathonTaskStatsType` or dict + """ + + def __init__(self, started_after_last_scaling=None, with_latest_config=None, with_outdated_config=None, total_summary=None): + self.started_after_last_scaling = started_after_last_scaling if \ + (isinstance(started_after_last_scaling, MarathonTaskStatsType) or started_after_last_scaling is None) \ + else MarathonTaskStatsType.from_json(started_after_last_scaling) + self.with_latest_config = with_latest_config if \ + (isinstance(with_latest_config , MarathonTaskStatsType) or with_latest_config is None) \ + else MarathonTaskStatsType.from_json(with_latest_config ) + self.with_outdated_config = with_outdated_config if \ + (isinstance(with_outdated_config, MarathonTaskStatsType) or with_outdated_config is None) \ + else MarathonTaskStatsType.from_json(with_outdated_config) + self.total_summary = total_summary if \ + (isinstance(total_summary, MarathonTaskStatsType) or total_summary is None) \ + else MarathonTaskStatsType.from_json(total_summary) + + +class MarathonTaskStatsType(MarathonObject): + """Marathon app task stats + + :param stats: stast about app tasks + :type stats: :class:`marathon.models.app.MarathonTaskStatsStats` or dict + """ + + def __init__(self, stats=None): + self.stats = stats if (isinstance(stats, MarathonTaskStatsStats) or stats is None)\ + else MarathonTaskStatsStats.from_json(stats) + + +class MarathonTaskStatsStats(MarathonObject): + """Marathon app task stats + + :param counts: app task count breakdown + :type counts: :class:`marathon.models.app.MarathonTaskStatsCounts` or dict + :param life_time: app task life time stats + :type life_time: :class:`marathon.models.app.MarathonTaskStatsLifeTime` or dict + """ + + def __init__(self, counts=None, life_time=None): + self.counts = counts if (isinstance(counts, MarathonTaskStatsCounts) or counts is None)\ + else MarathonTaskStatsCounts.from_json(counts) + self.life_time = life_time if (isinstance(life_time, MarathonTaskStatsLifeTime) or life_time is None)\ + else MarathonTaskStatsLifeTime.from_json(life_time) + + +class MarathonTaskStatsCounts(MarathonObject): + """Marathon app task counts + + Equivalent to tasksStaged, tasksRunning, tasksHealthy, tasksUnhealthy. + + :param int staged: Staged task count + :param int running: Running task count + :param int healthy: Healthy task count + :param int unhealthy: unhealthy task count + """ + + def __init__(self, staged=None, running=None, healthy=None, unhealthy=None): + self.staged = staged + self.running = running + self.healthy = healthy + self.unhealthy = unhealthy + +class MarathonTaskStatsLifeTime(MarathonObject): + """Marathon app life time statistics + + Measured from `"startedAt"` (timestamp of the Mesos TASK_RUNNING status update) of each running task until now + + :param float average_seconds: Average seconds + :param float median_seconds: Median seconds + """ + + def __init__(self, average_seconds=None, median_seconds=None): + self.average_seconds = average_seconds + self.median_seconds = median_seconds diff --git a/marathon/models/queue.py b/marathon/models/queue.py index acd7c9a..9bb43eb 100644 --- a/marathon/models/queue.py +++ b/marathon/models/queue.py @@ -7,11 +7,37 @@ class MarathonQueueItem(MarathonResource): See: https://mesosphere.github.io/marathon/docs/rest-api.html#queue + List all the tasks queued up or waiting to be scheduled. This is mainly + used for troubleshooting and occurs when scaling changes are requested and the + volume of scaling changes out paces the ability to schedule those tasks. In + addition to the application in the queue, you see also the task count that + needs to be started. + + If the task has a rate limit, then a delay to the start gets applied. You + can see this delay for every application with the seconds to wait before + the next launch will be tried. + :param app: :type app: :class:`marathon.models.app.MarathonApp` or dict + :param delay: queue item delay + :type delay: :class:`marathon.models.app.MarathonQueueItemDelay` or dict :param bool overdue: """ - def __init__(self, app=None, overdue=None): + def __init__(self, app=None, overdue=None, count=None, delay=None): self.app = app if isinstance(app, MarathonApp) else MarathonApp().from_json(app) self.overdue = overdue + self.count = count + self.delay = delay if isinstance(app, MarathonQueueItemDelay) else MarathonQueueItemDelay().from_json(app) + + +class MarathonQueueItemDelay(MarathonResource): + """Marathon queue item delay. + + :param int time_left_seconds: Seconds to wait before the next launch will be tried. + :param bool overdue: Is the queue item overdue. + """ + + def __init__(self, time_left_seconds=None, overdue=None): + self.count = count + self.delay = delay From 1bbb8bd7bc49b1a7fdf24b4ae2c5c0cc30d2e1c4 Mon Sep 17 00:00:00 2001 From: Bekir Dogan Date: Tue, 8 Mar 2016 04:03:38 +0000 Subject: [PATCH 034/292] Fixes for previous broken commit --- marathon/models/queue.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/marathon/models/queue.py b/marathon/models/queue.py index 9bb43eb..54d80e2 100644 --- a/marathon/models/queue.py +++ b/marathon/models/queue.py @@ -28,7 +28,7 @@ def __init__(self, app=None, overdue=None, count=None, delay=None): self.app = app if isinstance(app, MarathonApp) else MarathonApp().from_json(app) self.overdue = overdue self.count = count - self.delay = delay if isinstance(app, MarathonQueueItemDelay) else MarathonQueueItemDelay().from_json(app) + self.delay = delay if isinstance(delay, MarathonQueueItemDelay) else MarathonQueueItemDelay().from_json(delay) class MarathonQueueItemDelay(MarathonResource): @@ -39,5 +39,5 @@ class MarathonQueueItemDelay(MarathonResource): """ def __init__(self, time_left_seconds=None, overdue=None): - self.count = count - self.delay = delay + self.time_left_seconds = time_left_seconds + self.overdue = overdue From 6df9214837b3deb1673540971ec4c2ccaf8836aa Mon Sep 17 00:00:00 2001 From: Anatolii Lapytskyi Date: Thu, 14 Apr 2016 19:18:39 +0300 Subject: [PATCH 035/292] Add support for /v2/events stream --- marathon/client.py | 37 +++++++++++++++++++++++++++++++++++++ marathon/models/events.py | 9 ++++++++- requirements.txt | 1 + 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/marathon/client.py b/marathon/client.py index 9ec9c77..39fb965 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -12,6 +12,7 @@ import marathon from .models import MarathonApp, MarathonDeployment, MarathonGroup, MarathonInfo, MarathonTask, MarathonEndpoint, MarathonQueueItem from .exceptions import InternalServerError, NotFoundError, MarathonHttpError, MarathonError +from .models.events import EventFactory class MarathonClient(object): @@ -80,6 +81,26 @@ def _do_request(self, method, path, params=None, data=None): return response + def _do_sse_request(self, path, params=None, data=None): + from sseclient import SSEClient + + headers = {'Accept': 'text/event-stream'} + messages = None + servers = list(self.servers) + while servers and messages is None: + server = servers.pop(0) + url = ''.join([server.rstrip('/'), path]) + try: + messages = SSEClient(url,params=params, data=data, headers=headers, + auth=self.auth) + except Exception as e: + marathon.log.error('Error while calling %s: %s', url, e.message) + + if messages is None: + raise MarathonError('No remaining Marathon servers to try') + + return messages + def list_endpoints(self): """List the current endpoints for all applications @@ -592,3 +613,19 @@ def get_metrics(self): """ response = self._do_request('GET', '/metrics') return response.json() + + def event_stream(self): + """Polls event bus using /v2/events + + :returns: iterator with events + :rtype: iterator + """ + + messages = self._do_sse_request('/v2/events') + + ef = EventFactory() + for message in messages: + if not message.data: + continue + data = json.loads(message.data) + yield ef.process(data) diff --git a/marathon/models/events.py b/marathon/models/events.py index e9189d1..9b51b6c 100644 --- a/marathon/models/events.py +++ b/marathon/models/events.py @@ -85,6 +85,11 @@ class MarathonDeploymentStepSuccess(MarathonEvent): class MarathonDeploymentStepFailure(MarathonEvent): KNOWN_ATTRIBUTES = ['plan'] +class MarathonEventStreamAttached(MarathonEvent): + KNOWN_ATTRIBUTES = ['remote_address'] + +class MarathonEventStreamDetached(MarathonEvent): + KNOWN_ATTRIBUTES = ['remote_address'] class EventFactory: """ @@ -111,7 +116,9 @@ def __init__(self): 'deployment_failed': MarathonDeploymentFailed, 'deployment_info': MarathonDeploymentInfo, 'deployment_step_success': MarathonDeploymentStepSuccess, - 'deployment_step_failure': MarathonDeploymentStepFailure + 'deployment_step_failure': MarathonDeploymentStepFailure, + 'event_stream_attached': MarathonEventStreamAttached, + 'event_stream_detached': MarathonEventStreamDetached, } def process(self, event): diff --git a/requirements.txt b/requirements.txt index 40e7bdf..cb1d49e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ requests-mock +sseclient From 7d337cdbbde6d07adc1634060bfce10997761a12 Mon Sep 17 00:00:00 2001 From: Anatolii Lapytskyi Date: Thu, 14 Apr 2016 20:25:02 +0300 Subject: [PATCH 036/292] Add sseclient dependency to setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a41a6f8..2eb60ef 100755 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ long_description="""Python interface to the Mesos Marathon REST API.""", author='Mike Babineau', author_email='michael.babineau@gmail.com', - install_requires=['requests>=2.0.0'], + install_requires=['requests>=2.0.0', 'sseclient'], url='https://github.com/thefactory/marathon-python', packages=['marathon', 'marathon.models'], license='MIT', From 953be382abe24086d935c7fc75269cba95f07243 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Thu, 14 Apr 2016 16:43:13 -0700 Subject: [PATCH 037/292] Added pep8 checking --- tox.ini | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 0c164c4..5bf285a 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ passenv = TRAVIS usedevelop=True basepython = python2.7 -envlist = py +envlist = py,pep8 [testenv:itests] passenv = TRAVIS MARATHONVERSION @@ -41,4 +41,11 @@ deps = commands = py.test -s {posargs:tests} +[testenv:pep8] +deps = flake8 +commands = flake8 . +[flake8] +exclude = .tox,*.egg,docs,build +ignore = E226,E302,E41 +max-line-length = 160 From f5f94c4b7d67aead57833717b95b3d3ed2f29aa6 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Thu, 14 Apr 2016 16:45:16 -0700 Subject: [PATCH 038/292] Remove lots of extra whitespace for flake8 --- marathon/exceptions.py | 4 +- marathon/models/app.py | 4 +- marathon/models/base.py | 2 +- marathon/models/task.py | 2 +- tests/test_api.py | 88 ++++++++++++++++++++--------------------- 5 files changed, 50 insertions(+), 50 deletions(-) diff --git a/marathon/exceptions.py b/marathon/exceptions.py index 666935b..34e3b6f 100644 --- a/marathon/exceptions.py +++ b/marathon/exceptions.py @@ -11,7 +11,7 @@ def __init__(self, response): content = response.json() self.status_code = response.status_code self.error_message = content['message'] - super(MarathonHttpError, self).__init__(self.__str__() ) + super(MarathonHttpError, self).__init__(self.__str__()) def __repr__(self): return 'MarathonHttpError: HTTP %s returned with message, "%s"' % \ @@ -36,4 +36,4 @@ def __init__(self, param, value, options): 'Invalid choice "{value}" for param "{param}". Must be one of {options}'.format( param=param, value=value, options=options ) - ) \ No newline at end of file + ) diff --git a/marathon/models/app.py b/marathon/models/app.py index 238faca..d8d8773 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -256,7 +256,7 @@ def __init__(self, started_after_last_scaling=None, with_latest_config=None, wit else MarathonTaskStatsType.from_json(started_after_last_scaling) self.with_latest_config = with_latest_config if \ (isinstance(with_latest_config , MarathonTaskStatsType) or with_latest_config is None) \ - else MarathonTaskStatsType.from_json(with_latest_config ) + else MarathonTaskStatsType.from_json(with_latest_config) self.with_outdated_config = with_outdated_config if \ (isinstance(with_outdated_config, MarathonTaskStatsType) or with_outdated_config is None) \ else MarathonTaskStatsType.from_json(with_outdated_config) @@ -305,7 +305,7 @@ class MarathonTaskStatsCounts(MarathonObject): """ def __init__(self, staged=None, running=None, healthy=None, unhealthy=None): - self.staged = staged + self.staged = staged self.running = running self.healthy = healthy self.unhealthy = unhealthy diff --git a/marathon/models/base.py b/marathon/models/base.py index 773c563..75fc011 100644 --- a/marathon/models/base.py +++ b/marathon/models/base.py @@ -21,7 +21,7 @@ def json_repr(self, minimal=False): :rtype: dict """ if minimal: - return {to_camel_case(k):v for k,v in vars(self).items() if (v or v == False or v == 0 )} + return {to_camel_case(k):v for k,v in vars(self).items() if (v or v == False or v == 0)} else: return {to_camel_case(k):v for k,v in vars(self).items()} diff --git a/marathon/models/task.py b/marathon/models/task.py index 70c2b10..5b9dd77 100644 --- a/marathon/models/task.py +++ b/marathon/models/task.py @@ -24,7 +24,7 @@ class MarathonTask(MarathonResource): DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ' def __init__(self, app_id=None, health_check_results=None, host=None, id=None, ports=None, service_ports=None, - slave_id=None, staged_at=None, started_at=None, version=None, ip_addresses=[] ): + slave_id=None, staged_at=None, started_at=None, version=None, ip_addresses=[]): self.app_id = app_id self.health_check_results = health_check_results or [] self.health_check_results = [ diff --git a/tests/test_api.py b/tests/test_api.py index 23e8259..1425903 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -10,7 +10,7 @@ def test_get_deployments(m): m.get('http://fake_server/v2/deployments', text=fake_response) mock_client = MarathonClient(servers='http://fake_server') actual_deployments = mock_client.list_deployments() - expected_deployments = [ models.MarathonDeployment( + expected_deployments = [models.MarathonDeployment( id=u"fakeid", steps=[[models.MarathonDeploymentAction(action="ScaleApplication", app="/test")]], current_actions=[models.MarathonDeploymentAction(action="ScaleApplication", app="/test")], @@ -28,29 +28,29 @@ def test_list_tasks_with_app_id(m): m.get('http://fake_server/v2/tasks', text=fake_response) mock_client = MarathonClient(servers='http://fake_server') actual_deployments = mock_client.list_tasks(app_id='/anapp') - expected_deployments = [ models.task.MarathonTask( + expected_deployments = [models.task.MarathonTask( app_id="/anapp", - health_check_results= [ + health_check_results=[ models.task.MarathonHealthCheckResult( - alive= True, - consecutive_failures= 0, - first_success= "2014-10-03T22:57:02.246Z", - last_failure= None, - last_success= "2014-10-03T22:57:41.643Z", - task_id= "bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799" + alive=True, + consecutive_failures=0, + first_success="2014-10-03T22:57:02.246Z", + last_failure=None, + last_success="2014-10-03T22:57:41.643Z", + task_id="bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799" ) ], - host= "10.141.141.10", - id= "bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799", - ports= [ + host="10.141.141.10", + id="bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799", + ports=[ 31000 ], - service_ports= [ + service_ports=[ 9000 ], - staged_at= "2014-10-03T22:16:27.811Z", - started_at= "2014-10-03T22:57:41.587Z", - version= "2014-10-03T22:16:23.634Z" + staged_at="2014-10-03T22:16:27.811Z", + started_at="2014-10-03T22:57:41.587Z", + version="2014-10-03T22:16:23.634Z" )] assert actual_deployments == expected_deployments @@ -64,46 +64,46 @@ def test_list_tasks_without_app_id(m): expected_deployments = [ models.task.MarathonTask( app_id="/anapp", - health_check_results= [ + health_check_results=[ models.task.MarathonHealthCheckResult( - alive= True, - consecutive_failures= 0, - first_success= "2014-10-03T22:57:02.246Z", - last_failure= None, - last_success= "2014-10-03T22:57:41.643Z", - task_id= "bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799" + alive=True, + consecutive_failures=0, + first_success="2014-10-03T22:57:02.246Z", + last_failure=None, + last_success="2014-10-03T22:57:41.643Z", + task_id="bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799" ) ], - host= "10.141.141.10", + host="10.141.141.10", id="bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799", - ports= [ + ports=[ 31000 ], - service_ports= [ + service_ports=[ 9000 ], - staged_at= "2014-10-03T22:16:27.811Z", - started_at= "2014-10-03T22:57:41.587Z", - version= "2014-10-03T22:16:23.634Z" + staged_at="2014-10-03T22:16:27.811Z", + started_at="2014-10-03T22:57:41.587Z", + version="2014-10-03T22:16:23.634Z" ), models.task.MarathonTask( - app_id= "/anotherapp", - health_check_results= [ + app_id="/anotherapp", + health_check_results=[ models.task.MarathonHealthCheckResult( - alive= True, - consecutive_failures= 0, - first_success = "2014-10-03T22:57:02.246Z", - last_failure= None, - last_success= "2014-10-03T22:57:41.649Z", - task_id= "bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799" + alive=True, + consecutive_failures=0, + first_success="2014-10-03T22:57:02.246Z", + last_failure=None, + last_success="2014-10-03T22:57:41.649Z", + task_id="bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799" ) ], - host= "10.141.141.10", - id= "bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799", - ports= [ 31001 ], - service_ports= [ 9000 ], - staged_at = "2014-10-03T22:16:33.814Z", - started_at= "2014-10-03T22:57:41.593Z", - version= "2014-10-03T22:16:23.634Z" + host="10.141.141.10", + id="bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799", + ports=[31001], + service_ports=[9000], + staged_at="2014-10-03T22:16:33.814Z", + started_at="2014-10-03T22:57:41.593Z", + version="2014-10-03T22:16:23.634Z" )] assert actual_deployments == expected_deployments From 937d02af4f1b47f4534f8235d2aae865b3f9dd04 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Thu, 14 Apr 2016 16:46:06 -0700 Subject: [PATCH 039/292] Remove more whitespace violations --- marathon/exceptions.py | 2 +- marathon/models/app.py | 10 +++++----- marathon/models/base.py | 6 +++--- marathon/models/container.py | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/marathon/exceptions.py b/marathon/exceptions.py index 34e3b6f..5ce7b26 100644 --- a/marathon/exceptions.py +++ b/marathon/exceptions.py @@ -10,7 +10,7 @@ def __init__(self, response): """ content = response.json() self.status_code = response.status_code - self.error_message = content['message'] + self.error_message = content['message'] super(MarathonHttpError, self).__init__(self.__str__()) def __repr__(self): diff --git a/marathon/models/app.py b/marathon/models/app.py index d8d8773..b953c45 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -229,10 +229,10 @@ def __init__(self, last_scaling_at=None, last_config_change_at=None): self.last_config_change_at = self._to_datetime(last_config_change_at) def _to_datetime(self, timestamp): - if (timestamp is None or isinstance(timestamp, datetime)): - return timestamp - else: - return datetime.strptime(timestamp, self.DATETIME_FORMAT) + if (timestamp is None or isinstance(timestamp, datetime)): + return timestamp + else: + return datetime.strptime(timestamp, self.DATETIME_FORMAT) class MarathonTaskStats(MarathonObject): @@ -319,6 +319,6 @@ class MarathonTaskStatsLifeTime(MarathonObject): :param float median_seconds: Median seconds """ - def __init__(self, average_seconds=None, median_seconds=None): + def __init__(self, average_seconds=None, median_seconds=None): self.average_seconds = average_seconds self.median_seconds = median_seconds diff --git a/marathon/models/base.py b/marathon/models/base.py index 75fc011..23e96d0 100644 --- a/marathon/models/base.py +++ b/marathon/models/base.py @@ -21,9 +21,9 @@ def json_repr(self, minimal=False): :rtype: dict """ if minimal: - return {to_camel_case(k):v for k,v in vars(self).items() if (v or v == False or v == 0)} + return {to_camel_case(k): v for k, v in vars(self).items() if (v or v == False or v == 0)} else: - return {to_camel_case(k):v for k,v in vars(self).items()} + return {to_camel_case(k): v for k, v in vars(self).items()} @classmethod def from_json(cls, attributes): @@ -31,7 +31,7 @@ def from_json(cls, attributes): :param dict attributes: object attributes from parsed response """ - return cls(**{to_snake_case(k): v for k,v in attributes.items()}) + return cls(**{to_snake_case(k): v for k, v in attributes.items()}) def to_json(self, minimal=True): """Encode an object as a JSON string. diff --git a/marathon/models/container.py b/marathon/models/container.py index 3e88558..a171ec5 100644 --- a/marathon/models/container.py +++ b/marathon/models/container.py @@ -43,7 +43,7 @@ class MarathonDockerContainer(MarathonObject): :param bool force_pull_image: Force a docker pull before launching """ - NETWORK_MODES=['BRIDGE', 'HOST'] + NETWORK_MODES = ['BRIDGE', 'HOST'] """Valid network modes""" def __init__(self, image=None, network='HOST', port_mappings=None, parameters=None, privileged=None, @@ -72,7 +72,7 @@ class MarathonContainerPortMapping(MarathonObject): :param str protocol: """ - PROTOCOLS=['tcp', 'udp'] + PROTOCOLS = ['tcp', 'udp'] """Valid protocols""" def __init__(self, container_port=None, host_port=0, service_port=None, protocol='tcp'): @@ -94,7 +94,7 @@ class MarathonContainerVolume(MarathonObject): :param str mode: one of ['RO', 'RW'] """ - MODES=['RO', 'RW'] + MODES = ['RO', 'RW'] def __init__(self, container_path=None, host_path=None, mode='RW'): self.container_path = container_path From d601f4252ecfbff41e8e41eb6742656c1e4f13a2 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Thu, 14 Apr 2016 17:00:15 -0700 Subject: [PATCH 040/292] Fix some hanging indent line length violations and ignore others --- itests/itest_utils.py | 4 +- itests/steps/marathon_steps.py | 25 +++++-- marathon/client.py | 121 ++++++++++++++++++++++----------- marathon/models/app.py | 40 ++++++++--- marathon/models/base.py | 17 +++-- marathon/models/constraint.py | 1 + marathon/models/container.py | 19 ++++-- marathon/models/deployment.py | 27 ++++++-- marathon/models/endpoint.py | 7 +- marathon/models/events.py | 28 +++++++- marathon/models/group.py | 4 +- marathon/models/info.py | 17 +++-- marathon/models/queue.py | 8 ++- marathon/models/task.py | 11 ++- marathon/util.py | 2 + tests/test_api.py | 95 +++++++++++++------------- tox.ini | 2 +- 17 files changed, 291 insertions(+), 137 deletions(-) diff --git a/itests/itest_utils.py b/itests/itest_utils.py index fb0dcea..04a4500 100644 --- a/itests/itest_utils.py +++ b/itests/itest_utils.py @@ -13,6 +13,7 @@ class TimeoutError(Exception): pass + def timeout(seconds=10, error_message=os.strerror(errno.ETIME)): def decorator(func): def _handle_timeout(signum, frame): @@ -39,7 +40,8 @@ def wait_for_marathon(): while True: print 'Connecting to marathon on %s' % marathon_service try: - response = requests.get('http://%s/ping' % marathon_service, timeout=2) + response = requests.get( + 'http://%s/ping' % marathon_service, timeout=2) except ( requests.exceptions.ConnectionError, requests.exceptions.Timeout, diff --git a/itests/steps/marathon_steps.py b/itests/steps/marathon_steps.py index 24b0eb6..1f133aa 100644 --- a/itests/steps/marathon_steps.py +++ b/itests/steps/marathon_steps.py @@ -25,7 +25,8 @@ def get_marathon_info(context): @when(u'we create a trivial new app') def create_trivial_new_app(context): - context.client.create_app('test-trivial-app', marathon.MarathonApp(cmd='sleep 3600', mem=16, cpus=1, instances=5)) + context.client.create_app('test-trivial-app', marathon.MarathonApp( + cmd='sleep 3600', mem=16, cpus=1, instances=5)) @then(u'we should be able to kill the tasks') @@ -33,7 +34,8 @@ def kill_a_task(context): time.sleep(5) app = context.client.get_app('test-trivial-app') tasks = app.tasks - context.client.kill_task(app_id='test-trivial-app', task_id=tasks[0].id, scale=True) + context.client.kill_task( + app_id='test-trivial-app', task_id=tasks[0].id, scale=True) @when(u'we create a complex new app') @@ -42,14 +44,21 @@ def create_complex_new_app_with_unicode(context): 'container': { 'type': 'DOCKER', 'docker': { - 'portMappings': [{'protocol': 'tcp', 'containerPort': 8888, 'hostPort': 0}], + 'portMappings': + [{'protocol': 'tcp', + 'containerPort': 8888, + 'hostPort': 0}], 'image': u'localhost/fake_docker_url', 'network': 'BRIDGE', 'parameters': [ - {'key': 'add-host', 'value': 'google-public-dns-a.google.com:8.8.8.8'}, + {'key': 'add-host', 'value': + 'google-public-dns-a.google.com:8.8.8.8'}, ], }, - 'volumes': [{'hostPath': u'/etc/stuff', 'containerPath': u'/etc/stuff', 'mode': 'RO'}], + 'volumes': + [{'hostPath': u'/etc/stuff', + 'containerPath': u'/etc/stuff', + 'mode': 'RO'}], }, 'instances': 1, 'mem': 30, @@ -72,7 +81,8 @@ def create_complex_new_app_with_unicode(context): }, ], } - context.client.create_app('test-complex-app', marathon.MarathonApp(**app_config)) + context.client.create_app( + 'test-complex-app', marathon.MarathonApp(**app_config)) @then(u'we should see the {which} app running via the marathon api') @@ -92,7 +102,8 @@ def wait_deployment_finish(context, which): @then(u'we should be able to kill the #{to_kill} tasks of the {which} app') def kill_tasks(context, to_kill, which): - app_tasks = context.client.get_app('test-%s-app' % which, embed_tasks=True).tasks + app_tasks = context.client.get_app( + 'test-%s-app' % which, embed_tasks=True).tasks index_to_kill = eval("[" + to_kill + "]") task_to_kill = [app_tasks[index].id for index in index_to_kill] diff --git a/marathon/client.py b/marathon/client.py index 9ec9c77..8e59f7d 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -15,6 +15,7 @@ class MarathonClient(object): + """Client interface for the Marathon REST API.""" def __init__(self, servers, username=None, password=None, timeout=10): @@ -40,7 +41,8 @@ def __repr__(self): @staticmethod def _parse_response(response, clazz, is_list=False, resource_name=None): """Parse a Marathon response into an object or list of objects.""" - target = response.json()[resource_name] if resource_name else response.json() + target = response.json()[ + resource_name] if resource_name else response.json() if is_list: return [clazz.from_json(resource) for resource in target] else: @@ -48,35 +50,42 @@ def _parse_response(response, clazz, is_list=False, resource_name=None): def _do_request(self, method, path, params=None, data=None): """Query Marathon server.""" - headers = {'Content-Type': 'application/json', 'Accept': 'application/json'} + headers = { + 'Content-Type': 'application/json', 'Accept': 'application/json'} response = None servers = list(self.servers) while servers and response is None: server = servers.pop(0) url = ''.join([server.rstrip('/'), path]) try: - response = requests.request(method, url, params=params, data=data, headers=headers, + response = requests.request( + method, url, params=params, data=data, headers=headers, auth=self.auth, timeout=self.timeout) marathon.log.info('Got response from %s', server) except requests.exceptions.RequestException as e: - marathon.log.error('Error while calling %s: %s', url, e.message) + marathon.log.error( + 'Error while calling %s: %s', url, e.message) if response is None: raise MarathonError('No remaining Marathon servers to try') if response.status_code >= 500: - marathon.log.error('Got HTTP {code}: {body}'.format(code=response.status_code, body=response.text)) + marathon.log.error('Got HTTP {code}: {body}'.format( + code=response.status_code, body=response.text)) raise InternalServerError(response) elif response.status_code >= 400: - marathon.log.error('Got HTTP {code}: {body}'.format(code=response.status_code, body=response.text)) + marathon.log.error('Got HTTP {code}: {body}'.format( + code=response.status_code, body=response.text)) if response.status_code == 404: raise NotFoundError(response) else: raise MarathonHttpError(response) elif response.status_code >= 300: - marathon.log.warn('Got HTTP {code}: {body}'.format(code=response.status_code, body=response.text)) + marathon.log.warn('Got HTTP {code}: {body}'.format( + code=response.status_code, body=response.text)) else: - marathon.log.debug('Got HTTP {code}: {body}'.format(code=response.status_code, body=response.text)) + marathon.log.debug('Got HTTP {code}: {body}'.format( + code=response.status_code, body=response.text)) return response @@ -105,7 +114,8 @@ def create_app(self, app_id, app): else: return False - def list_apps(self, cmd=None, embed_tasks=False, embed_failures=False, embed_task_stats=False, **kwargs): + def list_apps(self, cmd=None, embed_tasks=False, + embed_failures=False, embed_task_stats=False, **kwargs): """List all apps. :param str app_id: application ID @@ -130,7 +140,8 @@ def list_apps(self, cmd=None, embed_tasks=False, embed_failures=False, embed_tas params['embed'] = 'apps.taskStats' response = self._do_request('GET', '/v2/apps', params=params) - apps = self._parse_response(response, MarathonApp, is_list=True, resource_name='apps') + apps = self._parse_response( + response, MarathonApp, is_list=True, resource_name='apps') for k, v in kwargs.items(): apps = [o for o in apps if getattr(o, k) == v] return apps @@ -145,7 +156,8 @@ def get_app(self, app_id, embed_tasks=False): :rtype: :class:`marathon.models.app.MarathonApp` """ params = {'embed': 'apps.tasks'} if embed_tasks else {} - response = self._do_request('GET', '/v2/apps/{app_id}'.format(app_id=app_id), params=params) + response = self._do_request( + 'GET', '/v2/apps/{app_id}'.format(app_id=app_id), params=params) return self._parse_response(response, MarathonApp, resource_name='app') def restart_app(self, app_id, force=False): @@ -157,7 +169,8 @@ def restart_app(self, app_id, force=False): :rtype: dict """ params = {'force': force} - response = self._do_request('POST', '/v2/apps/{app_id}/restart'.format(app_id=app_id), params=params) + response = self._do_request( + 'POST', '/v2/apps/{app_id}/restart'.format(app_id=app_id), params=params) return response.json() def update_app(self, app_id, app, force=False, minimal=True): @@ -181,7 +194,8 @@ def update_app(self, app_id, app, force=False, minimal=True): params = {'force': force} data = app.to_json(minimal=minimal) - response = self._do_request('PUT', '/v2/apps/{app_id}'.format(app_id=app_id), params=params, data=data) + response = self._do_request( + 'PUT', '/v2/apps/{app_id}'.format(app_id=app_id), params=params, data=data) return response.json() def rollback_app(self, app_id, version, force=False): @@ -196,7 +210,8 @@ def rollback_app(self, app_id, version, force=False): """ params = {'force': force} data = json.dumps({'version': version}) - response = self._do_request('PUT', '/v2/apps/{app_id}'.format(app_id=app_id), params=params, data=data) + response = self._do_request( + 'PUT', '/v2/apps/{app_id}'.format(app_id=app_id), params=params, data=data) return response.json() def delete_app(self, app_id, force=False): @@ -209,7 +224,8 @@ def delete_app(self, app_id, force=False): :rtype: dict """ params = {'force': force} - response = self._do_request('DELETE', '/v2/apps/{app_id}'.format(app_id=app_id), params=params) + response = self._do_request( + 'DELETE', '/v2/apps/{app_id}'.format(app_id=app_id), params=params) return response.json() def scale_app(self, app_id, instances=None, delta=None, force=False): @@ -239,7 +255,8 @@ def scale_app(self, app_id, instances=None, delta=None, force=False): marathon.log.error('App "{app}" not found'.format(app=app_id)) return - desired = instances if instances is not None else (app.instances + delta) + desired = instances if instances is not None else ( + app.instances + delta) return self.update_app(app.id, MarathonApp(instances=desired), force=force) def create_group(self, group): @@ -263,7 +280,8 @@ def list_groups(self, **kwargs): :rtype: list[:class:`marathon.models.group.MarathonGroup`] """ response = self._do_request('GET', '/v2/groups') - groups = self._parse_response(response, MarathonGroup, is_list=True, resource_name='groups') + groups = self._parse_response( + response, MarathonGroup, is_list=True, resource_name='groups') for k, v in kwargs.items(): groups = [o for o in groups if getattr(o, k) == v] return groups @@ -276,7 +294,8 @@ def get_group(self, group_id): :returns: group :rtype: :class:`marathon.models.group.MarathonGroup` """ - response = self._do_request('GET', '/v2/groups/{group_id}'.format(group_id=group_id)) + response = self._do_request( + 'GET', '/v2/groups/{group_id}'.format(group_id=group_id)) return self._parse_response(response, MarathonGroup, resource_name='group') def update_group(self, group_id, group, force=False, minimal=True): @@ -300,7 +319,8 @@ def update_group(self, group_id, group, force=False, minimal=True): params = {'force': force} data = group.to_json(minimal=minimal) - response = self._do_request('PUT', '/v2/groups/{group_id}'.format(group_id=group_id), data=data, params=params) + response = self._do_request( + 'PUT', '/v2/groups/{group_id}'.format(group_id=group_id), data=data, params=params) return response.json() def rollback_group(self, group_id, version, force=False): @@ -314,8 +334,9 @@ def rollback_group(self, group_id, version, force=False): :rtype: dict """ params = {'force': force} - response = self._do_request('PUT', '/v2/groups/{group_id}/versions/{version}'.format(group_id=group_id, - version=version), + response = self._do_request( + 'PUT', '/v2/groups/{group_id}/versions/{version}'.format(group_id=group_id, + version=version), params=params) return response.json() @@ -329,7 +350,8 @@ def delete_group(self, group_id, force=False): :rtype: dict """ params = {'force': force} - response = self._do_request('DELETE', '/v2/groups/{group_id}'.format(group_id=group_id), params=params) + response = self._do_request( + 'DELETE', '/v2/groups/{group_id}'.format(group_id=group_id), params=params) return response.json() def scale_group(self, group_id, scale_by): @@ -342,7 +364,8 @@ def scale_group(self, group_id, scale_by): :rtype: dict """ params = {'scaleBy': scale_by} - response = self._do_request('PUT', '/v2/groups/{group_id}'.format(group_id=group_id), params=params) + response = self._do_request( + 'PUT', '/v2/groups/{group_id}'.format(group_id=group_id), params=params) return response.json() def list_tasks(self, app_id=None, **kwargs): @@ -355,11 +378,14 @@ def list_tasks(self, app_id=None, **kwargs): :rtype: list[:class:`marathon.models.task.MarathonTask`] """ response = self._do_request('GET', '/v2/tasks') - tasks = self._parse_response(response, MarathonTask, is_list=True, resource_name='tasks') + tasks = self._parse_response( + response, MarathonTask, is_list=True, resource_name='tasks') if app_id: - tasks = [task for task in tasks if task.app_id.lstrip('/') == app_id.lstrip('/')] + tasks = [ + task for task in tasks if task.app_id.lstrip('/') == app_id.lstrip('/')] - [setattr(t, 'app_id', app_id) for t in tasks if app_id and t.app_id is None] + [setattr(t, 'app_id', app_id) + for t in tasks if app_id and t.app_id is None] for k, v in kwargs.items(): tasks = [o for o in tasks if getattr(o, k) == v] @@ -376,10 +402,12 @@ def kill_given_tasks(self, task_ids, scale=False): """ params = {'scale': scale} data = json.dumps({"ids": task_ids}) - response = self._do_request('POST', '/v2/tasks/delete', params=params, data=data) + response = self._do_request( + 'POST', '/v2/tasks/delete', params=params, data=data) return response == 200 - def kill_tasks(self, app_id, scale=False, host=None, batch_size=0, batch_delay=0): + def kill_tasks(self, app_id, scale=False, + host=None, batch_size=0, batch_delay=0): """Kill all tasks belonging to app. :param str app_id: application ID @@ -400,27 +428,33 @@ def batch(iterable, size): if batch_size == 0: # Terminate all at once params = {'scale': scale} - if host: params['host'] = host - response = self._do_request('DELETE', '/v2/apps/{app_id}/tasks'.format(app_id=app_id), params) + if host: + params['host'] = host + response = self._do_request( + 'DELETE', '/v2/apps/{app_id}/tasks'.format(app_id=app_id), params) # Marathon is inconsistent about what type of object it returns on the multi # task deletion endpoint, depending on the version of Marathon. See: # https://github.com/mesosphere/marathon/blob/06a6f763a75fb6d652b4f1660685ae234bd15387/src/main/scala/mesosphere/marathon/api/v2/AppTasksResource.scala#L88-L95 - if response.json().has_key("tasks"): + if "tasks" in response.json(): return self._parse_response(response, MarathonTask, is_list=True, resource_name='tasks') else: return response.json() else: # Terminate in batches - tasks = self.list_tasks(app_id, host=host) if host else self.list_tasks(app_id) + tasks = self.list_tasks( + app_id, host=host) if host else self.list_tasks(app_id) for tbatch in batch(tasks, batch_size): - killed_tasks = [self.kill_task(app_id, t.id, scale=scale) for t in tbatch] + killed_tasks = [self.kill_task(app_id, t.id, scale=scale) + for t in tbatch] - # Pause until the tasks have been killed to avoid race conditions + # Pause until the tasks have been killed to avoid race + # conditions killed_task_ids = set(t.id for t in killed_tasks) running_task_ids = killed_task_ids while killed_task_ids.intersection(running_task_ids): time.sleep(1) - running_task_ids = set(t.id for t in self.get_app(app_id).tasks) + running_task_ids = set( + t.id for t in self.get_app(app_id).tasks) if batch_delay == 0: # Pause until the replacement tasks are healthy @@ -428,7 +462,8 @@ def batch(iterable, size): running_instances = 0 while running_instances < desired_instances: time.sleep(1) - running_instances = sum(t.started_at is None for t in self.get_app(app_id).tasks) + running_instances = sum( + t.started_at is None for t in self.get_app(app_id).tasks) else: time.sleep(batch_delay) @@ -450,7 +485,7 @@ def kill_task(self, app_id, task_id, scale=False): # Marathon is inconsistent about what type of object it returns on the multi # task deletion endpoint, depending on the version of Marathon. See: # https://github.com/mesosphere/marathon/blob/06a6f763a75fb6d652b4f1660685ae234bd15387/src/main/scala/mesosphere/marathon/api/v2/AppTasksResource.scala#L88-L95 - if response.json().has_key("task"): + if "task" in response.json(): return self._parse_response(response, MarathonTask, is_list=False, resource_name='task') else: return response.json() @@ -463,7 +498,8 @@ def list_versions(self, app_id): :returns: list of versions :rtype: list[str] """ - response = self._do_request('GET', '/v2/apps/{app_id}/versions'.format(app_id=app_id)) + response = self._do_request( + 'GET', '/v2/apps/{app_id}/versions'.format(app_id=app_id)) return [version for version in response.json()['versions']] def get_version(self, app_id, version): @@ -541,11 +577,14 @@ def delete_deployment(self, deployment_id, force=False): """ if force: params = {'force': True} - self._do_request('DELETE', '/v2/deployments/{deployment}'.format(deployment=deployment_id), params=params) - # Successful DELETE with ?force=true returns empty text (and status code 202). Client code should poll until deployment is removed. + self._do_request('DELETE', '/v2/deployments/{deployment}'.format( + deployment=deployment_id), params=params) + # Successful DELETE with ?force=true returns empty text (and status + # code 202). Client code should poll until deployment is removed. return {} else: - response = self._do_request('DELETE', '/v2/deployments/{deployment}'.format(deployment=deployment_id)) + response = self._do_request( + 'DELETE', '/v2/deployments/{deployment}'.format(deployment=deployment_id)) return response.json() def get_info(self): diff --git a/marathon/models/app.py b/marathon/models/app.py index b953c45..cd1560a 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -8,6 +8,7 @@ class MarathonApp(MarathonResource): + """Marathon Application resource. See: https://mesosphere.github.io/marathon/docs/rest-api.html#post-/v2/apps @@ -69,10 +70,12 @@ class MarathonApp(MarathonResource): CREATE_ONLY_ATTRIBUTES = ['id', 'accepted_resource_roles'] """List of attributes that should only be passed on creation""" - READ_ONLY_ATTRIBUTES = ['deployments', 'tasks', 'tasks_running', 'tasks_staged', 'tasks_healthy', 'tasks_unhealthy'] + READ_ONLY_ATTRIBUTES = [ + 'deployments', 'tasks', 'tasks_running', 'tasks_staged', 'tasks_healthy', 'tasks_unhealthy'] """List of read-only attributes""" - def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, backoff_seconds=None, cmd=None, + def __init__( + self, accepted_resource_roles=None, args=None, backoff_factor=None, backoff_seconds=None, cmd=None, constraints=None, container=None, cpus=None, dependencies=None, deployments=None, disk=None, env=None, executor=None, health_checks=None, id=None, instances=None, labels=None, last_task_failure=None, max_launch_delay_seconds=None, mem=None, ports=None, require_ports=None, store_urls=None, @@ -98,7 +101,8 @@ def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, self.cpus = cpus self.dependencies = dependencies or [] self.deployments = [ - d if isinstance(d, MarathonDeployment) else MarathonDeployment().from_json(d) + d if isinstance( + d, MarathonDeployment) else MarathonDeployment().from_json(d) for d in (deployments or []) ] self.disk = disk @@ -106,7 +110,8 @@ def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, self.executor = executor self.health_checks = health_checks or [] self.health_checks = [ - hc if isinstance(hc, MarathonHealthCheck) else MarathonHealthCheck().from_json(hc) + hc if isinstance( + hc, MarathonHealthCheck) else MarathonHealthCheck().from_json(hc) for hc in (health_checks or []) ] self.id = assert_valid_path(id) @@ -140,6 +145,7 @@ def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, class MarathonHealthCheck(MarathonObject): + """Marathon health check. See https://mesosphere.github.io/marathon/docs/health-checks.html @@ -156,7 +162,8 @@ class MarathonHealthCheck(MarathonObject): :param dict kwargs: additional arguments for forward compatibility """ - def __init__(self, command=None, grace_period_seconds=None, interval_seconds=None, max_consecutive_failures=None, + def __init__( + self, command=None, grace_period_seconds=None, interval_seconds=None, max_consecutive_failures=None, path=None, port_index=None, protocol=None, timeout_seconds=None, ignore_http1xx=None, **kwargs): self.command = command self.grace_period_seconds = grace_period_seconds @@ -173,6 +180,7 @@ def __init__(self, command=None, grace_period_seconds=None, interval_seconds=Non class MarathonTaskFailure(MarathonObject): + """Marathon Task Failure. :param str app_id: application id @@ -187,7 +195,8 @@ class MarathonTaskFailure(MarathonObject): DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ' - def __init__(self, app_id=None, host=None, message=None, task_id=None, slave_id=None, state=None, timestamp=None, version=None): + def __init__(self, app_id=None, host=None, message=None, task_id=None, + slave_id=None, state=None, timestamp=None, version=None): self.app_id = app_id self.host = host self.message = message @@ -200,6 +209,7 @@ def __init__(self, app_id=None, host=None, message=None, task_id=None, slave_id= class MarathonUpgradeStrategy(MarathonObject): + """Marathon health check. See https://mesosphere.github.io/marathon/docs/health-checks.html @@ -207,12 +217,14 @@ class MarathonUpgradeStrategy(MarathonObject): :param float minimum_health_capacity: minimum % of instances kept healthy on deploy """ - def __init__(self, maximum_over_capacity=None, minimum_health_capacity=None): + def __init__(self, maximum_over_capacity=None, + minimum_health_capacity=None): self.maximum_over_capacity = maximum_over_capacity self.minimum_health_capacity = minimum_health_capacity class MarathonAppVersionInfo(MarathonObject): + """Marathon App version info. See release notes for Marathon v0.11.0 @@ -236,6 +248,7 @@ def _to_datetime(self, timestamp): class MarathonTaskStats(MarathonObject): + """Marathon task statistics See https://mesosphere.github.io/marathon/docs/rest-api.html#taskstats-object-v0-11 @@ -250,12 +263,13 @@ class MarathonTaskStats(MarathonObject): :type total_summary: :class:`marathon.models.app.MarathonTaskStatsType` or dict """ - def __init__(self, started_after_last_scaling=None, with_latest_config=None, with_outdated_config=None, total_summary=None): + def __init__(self, started_after_last_scaling=None, + with_latest_config=None, with_outdated_config=None, total_summary=None): self.started_after_last_scaling = started_after_last_scaling if \ (isinstance(started_after_last_scaling, MarathonTaskStatsType) or started_after_last_scaling is None) \ else MarathonTaskStatsType.from_json(started_after_last_scaling) self.with_latest_config = with_latest_config if \ - (isinstance(with_latest_config , MarathonTaskStatsType) or with_latest_config is None) \ + (isinstance(with_latest_config, MarathonTaskStatsType) or with_latest_config is None) \ else MarathonTaskStatsType.from_json(with_latest_config) self.with_outdated_config = with_outdated_config if \ (isinstance(with_outdated_config, MarathonTaskStatsType) or with_outdated_config is None) \ @@ -266,6 +280,7 @@ def __init__(self, started_after_last_scaling=None, with_latest_config=None, wit class MarathonTaskStatsType(MarathonObject): + """Marathon app task stats :param stats: stast about app tasks @@ -278,6 +293,7 @@ def __init__(self, stats=None): class MarathonTaskStatsStats(MarathonObject): + """Marathon app task stats :param counts: app task count breakdown @@ -294,6 +310,7 @@ def __init__(self, counts=None, life_time=None): class MarathonTaskStatsCounts(MarathonObject): + """Marathon app task counts Equivalent to tasksStaged, tasksRunning, tasksHealthy, tasksUnhealthy. @@ -304,13 +321,16 @@ class MarathonTaskStatsCounts(MarathonObject): :param int unhealthy: unhealthy task count """ - def __init__(self, staged=None, running=None, healthy=None, unhealthy=None): + def __init__(self, staged=None, + running=None, healthy=None, unhealthy=None): self.staged = staged self.running = running self.healthy = healthy self.unhealthy = unhealthy + class MarathonTaskStatsLifeTime(MarathonObject): + """Marathon app life time statistics Measured from `"startedAt"` (timestamp of the Mesos TASK_RUNNING status update) of each running task until now diff --git a/marathon/models/base.py b/marathon/models/base.py index 23e96d0..6c4ad83 100644 --- a/marathon/models/base.py +++ b/marathon/models/base.py @@ -5,6 +5,7 @@ class MarathonObject(object): + """Base Marathon object.""" def __repr__(self): @@ -47,6 +48,7 @@ def to_json(self, minimal=True): class MarathonResource(MarathonObject): + """Base Marathon resource.""" def __repr__(self): @@ -61,8 +63,10 @@ def __eq__(self, other): def __str__(self): return "{clazz}::".format(clazz=self.__class__.__name__) + str(self.__dict__) -# See: https://github.com/mesosphere/marathon/blob/2a9d1d20ec2f1cfcc49fbb1c0e7348b26418ef38/src/main/scala/mesosphere/marathon/api/ModelValidation.scala#L224 -ID_PATTERN = re.compile('^(([a-z0-9]|[a-z0-9][a-z0-9\\-]*[a-z0-9])\\.)*([a-z0-9]|[a-z0-9][a-z0-9\\-]*[a-z0-9])|(\\.|\\.\\.)$') +# See: +# https://github.com/mesosphere/marathon/blob/2a9d1d20ec2f1cfcc49fbb1c0e7348b26418ef38/src/main/scala/mesosphere/marathon/api/ModelValidation.scala#L224 +ID_PATTERN = re.compile( + '^(([a-z0-9]|[a-z0-9][a-z0-9\\-]*[a-z0-9])\\.)*([a-z0-9]|[a-z0-9][a-z0-9\\-]*[a-z0-9])|(\\.|\\.\\.)$') def assert_valid_path(path): @@ -74,10 +78,12 @@ def assert_valid_path(path): """ if path is None: return - # As seen in: https://github.com/mesosphere/marathon/blob/0c11661ca2f259f8a903d114ef79023649a6f04b/src/main/scala/mesosphere/marathon/state/PathId.scala#L71 + # As seen in: + # https://github.com/mesosphere/marathon/blob/0c11661ca2f259f8a903d114ef79023649a6f04b/src/main/scala/mesosphere/marathon/state/PathId.scala#L71 for id in filter(None, path.strip('/').split('/')): if not ID_PATTERN.match(id): - raise ValueError('invalid path (allowed: lowercase letters, digits, hyphen, "/", ".", ".."): %r' % path) + raise ValueError( + 'invalid path (allowed: lowercase letters, digits, hyphen, "/", ".", ".."): %r' % path) return path @@ -91,5 +97,6 @@ def assert_valid_id(id): if id is None: return if not ID_PATTERN.match(id.strip('/')): - raise ValueError('invalid id (allowed: lowercase letters, digits, hyphen, ".", ".."): %r' % id) + raise ValueError( + 'invalid id (allowed: lowercase letters, digits, hyphen, ".", ".."): %r' % id) return id diff --git a/marathon/models/constraint.py b/marathon/models/constraint.py index 9d87663..2264f16 100644 --- a/marathon/models/constraint.py +++ b/marathon/models/constraint.py @@ -3,6 +3,7 @@ class MarathonConstraint(MarathonObject): + """Marathon placement constraint. See https://mesosphere.github.io/marathon/docs/constraints.html diff --git a/marathon/models/container.py b/marathon/models/container.py index a171ec5..4719afd 100644 --- a/marathon/models/container.py +++ b/marathon/models/container.py @@ -3,6 +3,7 @@ class MarathonContainer(MarathonObject): + """Marathon health check. See https://mesosphere.github.io/marathon/docs/native-docker.html @@ -24,12 +25,14 @@ def __init__(self, docker=None, type='DOCKER', volumes=None): self.docker = docker if isinstance(docker, MarathonDockerContainer) \ else MarathonDockerContainer().from_json(docker) self.volumes = [ - v if isinstance(v, MarathonContainerVolume) else MarathonContainerVolume().from_json(v) + v if isinstance( + v, MarathonContainerVolume) else MarathonContainerVolume().from_json(v) for v in (volumes or []) ] class MarathonDockerContainer(MarathonObject): + """Docker options. See https://mesosphere.github.io/marathon/docs/native-docker.html @@ -46,15 +49,18 @@ class MarathonDockerContainer(MarathonObject): NETWORK_MODES = ['BRIDGE', 'HOST'] """Valid network modes""" - def __init__(self, image=None, network='HOST', port_mappings=None, parameters=None, privileged=None, + def __init__( + self, image=None, network='HOST', port_mappings=None, parameters=None, privileged=None, force_pull_image=None, **kwargs): self.image = image if network: if not network in self.NETWORK_MODES: - raise InvalidChoiceError('network', network, self.NETWORK_MODES) + raise InvalidChoiceError( + 'network', network, self.NETWORK_MODES) self.network = network self.port_mappings = [ - pm if isinstance(pm, MarathonContainerPortMapping) else MarathonContainerPortMapping().from_json(pm) + pm if isinstance( + pm, MarathonContainerPortMapping) else MarathonContainerPortMapping().from_json(pm) for pm in (port_mappings or []) ] self.parameters = parameters or [] @@ -63,6 +69,7 @@ def __init__(self, image=None, network='HOST', port_mappings=None, parameters=No class MarathonContainerPortMapping(MarathonObject): + """Container port mapping. See https://mesosphere.github.io/marathon/docs/native-docker.html @@ -75,7 +82,8 @@ class MarathonContainerPortMapping(MarathonObject): PROTOCOLS = ['tcp', 'udp'] """Valid protocols""" - def __init__(self, container_port=None, host_port=0, service_port=None, protocol='tcp'): + def __init__(self, container_port=None, + host_port=0, service_port=None, protocol='tcp'): self.container_port = container_port self.host_port = host_port self.service_port = service_port @@ -85,6 +93,7 @@ def __init__(self, container_port=None, host_port=0, service_port=None, protocol class MarathonContainerVolume(MarathonObject): + """Volume options. See https://mesosphere.github.io/marathon/docs/native-docker.html diff --git a/marathon/models/deployment.py b/marathon/models/deployment.py index e73afe8..6c864d8 100644 --- a/marathon/models/deployment.py +++ b/marathon/models/deployment.py @@ -2,6 +2,7 @@ class MarathonDeployment(MarathonResource): + """Marathon Application resource. See: https://mesosphere.github.io/marathon/docs/rest-api.html#deployments @@ -17,17 +18,20 @@ class MarathonDeployment(MarathonResource): :param str version: version id """ - def __init__(self, affected_apps=None, current_actions=None, current_step=None, id=None, steps=None, + def __init__( + self, affected_apps=None, current_actions=None, current_step=None, id=None, steps=None, total_steps=None, version=None): self.affected_apps = affected_apps self.current_actions = [ - a if isinstance(a, MarathonDeploymentAction) else MarathonDeploymentAction().from_json(a) + a if isinstance( + a, MarathonDeploymentAction) else MarathonDeploymentAction().from_json(a) for a in (current_actions or []) ] self.current_step = current_step self.id = id self.steps = [ - [step if isinstance(step, MarathonDeploymentAction) else MarathonDeploymentAction().from_json(step) for step in s] + [step if isinstance(step, MarathonDeploymentAction) else MarathonDeploymentAction().from_json(step) + for step in s] for s in (steps or []) ] self.total_steps = total_steps @@ -35,6 +39,7 @@ def __init__(self, affected_apps=None, current_actions=None, current_step=None, class MarathonDeploymentAction(MarathonObject): + """Marathon Application resource. See: https://mesosphere.github.io/marathon/docs/rest-api.html#deployments @@ -52,27 +57,37 @@ def __init__(self, action=None, app=None, apps=None, type=None): class MarathonDeploymentPlan(MarathonObject): - def __init__(self, original=None, target=None, steps=None, id=None, version=None): + + def __init__(self, original=None, target=None, + steps=None, id=None, version=None): self.original = MarathonDeploymentOriginalState.from_json(original) self.target = MarathonDeploymentTargetState.from_json(target) self.steps = [MarathonDeploymentStep.from_json(x) for x in steps] self.id = id self.version = version + class MarathonDeploymentStep(MarathonObject): + def __init__(self, actions=None): self.actions = [MarathonDeploymentAction.from_json(x) for x in actions] + class MarathonDeploymentOriginalState(MarathonObject): - def __init__(self, dependencies=None, apps=None, id=None, version=None, groups=None): + + def __init__(self, dependencies=None, + apps=None, id=None, version=None, groups=None): self.apps = apps self.groups = groups self.id = id self.version = version self.dependencies = dependencies + class MarathonDeploymentTargetState(MarathonObject): - def __init__(self, groups=None, apps=None, dependencies=None, id=None, version=None): + + def __init__(self, groups=None, apps=None, + dependencies=None, id=None, version=None): self.apps = apps self.groups = groups self.id = id diff --git a/marathon/models/endpoint.py b/marathon/models/endpoint.py index 3493222..7866eaf 100644 --- a/marathon/models/endpoint.py +++ b/marathon/models/endpoint.py @@ -7,6 +7,7 @@ class MarathonEndpoint(MarathonObject): + """Marathon Endpoint helper object for service discovery. It describes a single port mapping for a running task. :param str app_id: application id @@ -25,7 +26,8 @@ def __repr__(self): task_port=self.task_port ) - def __init__(self, app_id=None, service_port=None, host=None, task_id=None, task_port=None): + def __init__(self, app_id=None, service_port=None, + host=None, task_id=None, task_port=None): self.app_id = app_id self.service_port = service_port self.host = host @@ -43,7 +45,8 @@ def from_tasks(cls, tasks): endpoints = [ [ - MarathonEndpoint(task.app_id, task.service_ports[port_index], task.host, task.id, port) + MarathonEndpoint(task.app_id, task.service_ports[ + port_index], task.host, task.id, port) for port_index, port in enumerate(task.ports) ] for task in tasks diff --git a/marathon/models/events.py b/marathon/models/events.py index e9189d1..6157deb 100644 --- a/marathon/models/events.py +++ b/marathon/models/events.py @@ -10,7 +10,9 @@ import marathon + class MarathonEvent(MarathonObject): + """ The MarathonEvent base class handles the translation of Event objects sent by the Marathon server into library MarathonObjects. @@ -29,64 +31,84 @@ def __init__(self, event_type, timestamp, **kwargs): try: self._set(attribute, kwargs[attribute]) except KeyError: - marathon.log.warn('Unknown event attribute processing event {}: {}'.format(event_type, attribute)) + marathon.log.warn( + 'Unknown event attribute processing event {}: {}'.format(event_type, attribute)) def _set(self, attribute_name, attribute): if attribute_name in self.attribute_name_to_marathon_object: clazz = self.attribute_name_to_marathon_object[attribute_name] - attribute = clazz.from_json(attribute) # If this attribute already has a Marathon object instantiate it. + attribute = clazz.from_json( + attribute) # If this attribute already has a Marathon object instantiate it. setattr(self, attribute_name, attribute) + class MarathonApiPostEvent(MarathonEvent): KNOWN_ATTRIBUTES = ['client_ip', 'app_definition', 'uri'] + class MarathonStatusUpdateEvent(MarathonEvent): - KNOWN_ATTRIBUTES = ['slave_id', 'task_id', 'task_status', 'app_id', 'host', 'ports', 'version'] + KNOWN_ATTRIBUTES = [ + 'slave_id', 'task_id', 'task_status', 'app_id', 'host', 'ports', 'version'] + class MarathonFrameworkMessageEvent(MarathonEvent): KNOWN_ATTRIBUTES = ['slave_id', 'executor_id', 'message'] + class MarathonSubscribeEvent(MarathonEvent): KNOWN_ATTRIBUTES = ['client_ip', 'callback_url'] + class MarathonUnsubscribeEvent(MarathonEvent): KNOWN_ATTRIBUTES = ['client_ip', 'callback_url'] + class MarathonAddHealthCheckEvent(MarathonEvent): KNOWN_ATTRIBUTES = ['app_id', 'health_check', 'version'] + class MarathonRemoveHealthCheckEvent(MarathonEvent): KNOWN_ATTRIBUTES = ['app_id', 'health_check'] + class MarathonFailedHealthCheckEvent(MarathonEvent): KNOWN_ATTRIBUTES = ['app_id', 'health_check', 'task_id'] + class MarathonHealthStatusChangedEvent(MarathonEvent): KNOWN_ATTRIBUTES = ['app_id', 'health_check', 'task_id', 'alive'] + class MarathonGroupChangeSuccess(MarathonEvent): KNOWN_ATTRIBUTES = ['group_id', 'version'] + class MarathonGroupChangeFailed(MarathonEvent): KNOWN_ATTRIBUTES = ['group_id', 'version', 'reason'] + class MarathonDeploymentSuccess(MarathonEvent): KNOWN_ATTRIBUTES = ['id'] + class MarathonDeploymentFailed(MarathonEvent): KNOWN_ATTRIBUTES = ['id'] + class MarathonDeploymentInfo(MarathonEvent): KNOWN_ATTRIBUTES = ['plan'] + class MarathonDeploymentStepSuccess(MarathonEvent): KNOWN_ATTRIBUTES = ['plan'] + class MarathonDeploymentStepFailure(MarathonEvent): KNOWN_ATTRIBUTES = ['plan'] class EventFactory: + """ Handle an event emitted from the Marathon EventBus See: https://mesosphere.github.io/marathon/docs/event-bus.html diff --git a/marathon/models/group.py b/marathon/models/group.py index 06ecd3e..40cf6bf 100644 --- a/marathon/models/group.py +++ b/marathon/models/group.py @@ -3,6 +3,7 @@ class MarathonGroup(MarathonResource): + """Marathon group resource. See: https://mesosphere.github.io/marathon/docs/rest-api.html#groups @@ -16,7 +17,8 @@ class MarathonGroup(MarathonResource): :param str version: """ - def __init__(self, apps=None, dependencies=None, groups=None, id=None, version=None): + def __init__(self, apps=None, dependencies=None, + groups=None, id=None, version=None): self.apps = [ a if isinstance(a, MarathonApp) else MarathonApp().from_json(a) for a in (apps or []) diff --git a/marathon/models/info.py b/marathon/models/info.py index 140088e..3ab77d5 100644 --- a/marathon/models/info.py +++ b/marathon/models/info.py @@ -2,6 +2,7 @@ class MarathonInfo(MarathonResource): + """Marathon Info. See: https://mesosphere.github.io/marathon/docs/rest-api.html#get-v2-info @@ -21,12 +22,14 @@ class MarathonInfo(MarathonResource): :param bool elected: """ - def __init__(self, event_subscriber=None, framework_id=None, http_config=None, leader=None, marathon_config=None, + def __init__( + self, event_subscriber=None, framework_id=None, http_config=None, leader=None, marathon_config=None, name=None, version=None, elected=None, zookeeper_config=None): if isinstance(event_subscriber, MarathonEventSubscriber): self.event_subscriber = event_subscriber elif event_subscriber is not None: - self.event_subscriber = MarathonEventSubscriber().from_json(event_subscriber) + self.event_subscriber = MarathonEventSubscriber().from_json( + event_subscriber) else: self.event_subscriber = None self.framework_id = framework_id @@ -43,6 +46,7 @@ def __init__(self, event_subscriber=None, framework_id=None, http_config=None, l class MarathonConfig(MarathonObject): + """Marathon config resource. See: https://mesosphere.github.io/marathon/docs/rest-api.html#get-/v2/info @@ -68,7 +72,8 @@ class MarathonConfig(MarathonObject): :param int marathon_store_timeout: """ - def __init__(self, checkpoint=None, executor=None, failover_timeout=None, framework_name=None, ha=None, + def __init__( + self, checkpoint=None, executor=None, failover_timeout=None, framework_name=None, ha=None, hostname=None, leader_proxy_connection_timeout_ms=None, leader_proxy_read_timeout_ms=None, local_port_min=None, local_port_max=None, master=None, mesos_leader_ui_url=None, mesos_role=None, mesos_user=None, webui_url=None, reconciliation_initial_delay=None, reconciliation_interval=None, @@ -92,6 +97,7 @@ def __init__(self, checkpoint=None, executor=None, failover_timeout=None, framew class MarathonZooKeeperConfig(MarathonObject): + """Marathon zookeeper config resource. See: https://mesosphere.github.io/marathon/docs/rest-api.html#get-/v2/info @@ -106,7 +112,8 @@ class MarathonZooKeeperConfig(MarathonObject): :param int zk_timeout: """ - def __init__(self, zk=None, zk_future_timeout=None, zk_hosts=None, zk_max_versions=None, zk_path=None, + def __init__( + self, zk=None, zk_future_timeout=None, zk_hosts=None, zk_max_versions=None, zk_path=None, zk_session_timeout=None, zk_state=None, zk_timeout=None): self.zk = zk self.zk_future_timeout = zk_future_timeout @@ -117,6 +124,7 @@ def __init__(self, zk=None, zk_future_timeout=None, zk_hosts=None, zk_max_versio class MarathonHttpConfig(MarathonObject): + """Marathon http config resource. See: https://mesosphere.github.io/marathon/docs/rest-api.html#get-/v2/info @@ -133,6 +141,7 @@ def __init__(self, assets_path=None, http_port=None, https_port=None): class MarathonEventSubscriber(MarathonObject): + """Marathon event subscriber resource. See: https://mesosphere.github.io/marathon/docs/rest-api.html#get-/v2/info diff --git a/marathon/models/queue.py b/marathon/models/queue.py index 54d80e2..5f14af8 100644 --- a/marathon/models/queue.py +++ b/marathon/models/queue.py @@ -3,6 +3,7 @@ class MarathonQueueItem(MarathonResource): + """Marathon queue item. See: https://mesosphere.github.io/marathon/docs/rest-api.html#queue @@ -25,13 +26,16 @@ class MarathonQueueItem(MarathonResource): """ def __init__(self, app=None, overdue=None, count=None, delay=None): - self.app = app if isinstance(app, MarathonApp) else MarathonApp().from_json(app) + self.app = app if isinstance( + app, MarathonApp) else MarathonApp().from_json(app) self.overdue = overdue self.count = count - self.delay = delay if isinstance(delay, MarathonQueueItemDelay) else MarathonQueueItemDelay().from_json(delay) + self.delay = delay if isinstance( + delay, MarathonQueueItemDelay) else MarathonQueueItemDelay().from_json(delay) class MarathonQueueItemDelay(MarathonResource): + """Marathon queue item delay. :param int time_left_seconds: Seconds to wait before the next launch will be tried. diff --git a/marathon/models/task.py b/marathon/models/task.py index 5b9dd77..dbaa4d4 100644 --- a/marathon/models/task.py +++ b/marathon/models/task.py @@ -4,6 +4,7 @@ class MarathonTask(MarathonResource): + """Marathon Task resource. :param str app_id: application id @@ -23,12 +24,14 @@ class MarathonTask(MarathonResource): DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ' - def __init__(self, app_id=None, health_check_results=None, host=None, id=None, ports=None, service_ports=None, + def __init__( + self, app_id=None, health_check_results=None, host=None, id=None, ports=None, service_ports=None, slave_id=None, staged_at=None, started_at=None, version=None, ip_addresses=[]): self.app_id = app_id self.health_check_results = health_check_results or [] self.health_check_results = [ - hcr if isinstance(hcr, MarathonHealthCheckResult) else MarathonHealthCheckResult().from_json(hcr) + hcr if isinstance( + hcr, MarathonHealthCheckResult) else MarathonHealthCheckResult().from_json(hcr) for hcr in (health_check_results or []) if any(health_check_results) ] self.host = host @@ -44,6 +47,7 @@ def __init__(self, app_id=None, health_check_results=None, host=None, id=None, p class MarathonHealthCheckResult(MarathonObject): + """Marathon health check result. See https://mesosphere.github.io/marathon/docs/health-checks.html @@ -59,7 +63,8 @@ class MarathonHealthCheckResult(MarathonObject): DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ' - def __init__(self, alive=None, consecutive_failures=None, first_success=None, + def __init__( + self, alive=None, consecutive_failures=None, first_success=None, last_failure=None, last_success=None, task_id=None, last_failure_cause=None): self.alive = alive self.consecutive_failures = consecutive_failures diff --git a/marathon/util.py b/marathon/util.py index e88fb3a..7363913 100644 --- a/marathon/util.py +++ b/marathon/util.py @@ -15,6 +15,7 @@ def is_stringy(obj): class MarathonJsonEncoder(json.JSONEncoder): + """Custom JSON encoder for Marathon object serialization.""" def default(self, obj): @@ -34,6 +35,7 @@ def default(self, obj): class MarathonMinimalJsonEncoder(json.JSONEncoder): + """Custom JSON encoder for Marathon object serialization.""" def default(self, obj): diff --git a/tests/test_api.py b/tests/test_api.py index 1425903..a8c9174 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -12,8 +12,11 @@ def test_get_deployments(m): actual_deployments = mock_client.list_deployments() expected_deployments = [models.MarathonDeployment( id=u"fakeid", - steps=[[models.MarathonDeploymentAction(action="ScaleApplication", app="/test")]], - current_actions=[models.MarathonDeploymentAction(action="ScaleApplication", app="/test")], + steps=[ + [models.MarathonDeploymentAction( + action="ScaleApplication", app="/test")]], + current_actions=[models.MarathonDeploymentAction( + action="ScaleApplication", app="/test")], current_step=1, total_steps=1, affected_apps=[u"/test"], @@ -62,48 +65,48 @@ def test_list_tasks_without_app_id(m): mock_client = MarathonClient(servers='http://fake_server') actual_deployments = mock_client.list_tasks() expected_deployments = [ - models.task.MarathonTask( - app_id="/anapp", - health_check_results=[ - models.task.MarathonHealthCheckResult( - alive=True, - consecutive_failures=0, - first_success="2014-10-03T22:57:02.246Z", - last_failure=None, - last_success="2014-10-03T22:57:41.643Z", - task_id="bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799" - ) - ], - host="10.141.141.10", - id="bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799", - ports=[ - 31000 - ], - service_ports=[ - 9000 - ], - staged_at="2014-10-03T22:16:27.811Z", - started_at="2014-10-03T22:57:41.587Z", - version="2014-10-03T22:16:23.634Z" - ), - models.task.MarathonTask( - app_id="/anotherapp", - health_check_results=[ - models.task.MarathonHealthCheckResult( - alive=True, - consecutive_failures=0, - first_success="2014-10-03T22:57:02.246Z", - last_failure=None, - last_success="2014-10-03T22:57:41.649Z", - task_id="bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799" - ) - ], - host="10.141.141.10", - id="bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799", - ports=[31001], - service_ports=[9000], - staged_at="2014-10-03T22:16:33.814Z", - started_at="2014-10-03T22:57:41.593Z", - version="2014-10-03T22:16:23.634Z" - )] + models.task.MarathonTask( + app_id="/anapp", + health_check_results=[ + models.task.MarathonHealthCheckResult( + alive=True, + consecutive_failures=0, + first_success="2014-10-03T22:57:02.246Z", + last_failure=None, + last_success="2014-10-03T22:57:41.643Z", + task_id="bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799" + ) + ], + host="10.141.141.10", + id="bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799", + ports=[ + 31000 + ], + service_ports=[ + 9000 + ], + staged_at="2014-10-03T22:16:27.811Z", + started_at="2014-10-03T22:57:41.587Z", + version="2014-10-03T22:16:23.634Z" + ), + models.task.MarathonTask( + app_id="/anotherapp", + health_check_results=[ + models.task.MarathonHealthCheckResult( + alive=True, + consecutive_failures=0, + first_success="2014-10-03T22:57:02.246Z", + last_failure=None, + last_success="2014-10-03T22:57:41.649Z", + task_id="bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799" + ) + ], + host="10.141.141.10", + id="bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799", + ports=[31001], + service_ports=[9000], + staged_at="2014-10-03T22:16:33.814Z", + started_at="2014-10-03T22:57:41.593Z", + version="2014-10-03T22:16:23.634Z" + )] assert actual_deployments == expected_deployments diff --git a/tox.ini b/tox.ini index 5bf285a..4058c25 100644 --- a/tox.ini +++ b/tox.ini @@ -47,5 +47,5 @@ commands = flake8 . [flake8] exclude = .tox,*.egg,docs,build -ignore = E226,E302,E41 +ignore = E226,E302,E41,E501,E131 max-line-length = 160 From bcc3e0e497c7241c6cb2fb315e446aba6de77874 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Thu, 14 Apr 2016 17:06:16 -0700 Subject: [PATCH 041/292] More misc pep8 fixes and ignore the rest --- itests/itest_utils.py | 2 -- itests/steps/marathon_steps.py | 5 +---- marathon/models/base.py | 2 +- marathon/models/constraint.py | 2 +- marathon/models/container.py | 8 ++++---- marathon/models/deployment.py | 2 +- marathon/models/endpoint.py | 5 ----- tests/test_api.py | 1 - tox.ini | 2 +- 9 files changed, 9 insertions(+), 20 deletions(-) diff --git a/itests/itest_utils.py b/itests/itest_utils.py index 04a4500..66be2b9 100644 --- a/itests/itest_utils.py +++ b/itests/itest_utils.py @@ -2,8 +2,6 @@ from functools import wraps import os import signal -import sys -import threading import time import requests diff --git a/itests/steps/marathon_steps.py b/itests/steps/marathon_steps.py index 1f133aa..ac329c6 100644 --- a/itests/steps/marathon_steps.py +++ b/itests/steps/marathon_steps.py @@ -50,10 +50,7 @@ def create_complex_new_app_with_unicode(context): 'hostPort': 0}], 'image': u'localhost/fake_docker_url', 'network': 'BRIDGE', - 'parameters': [ - {'key': 'add-host', 'value': - 'google-public-dns-a.google.com:8.8.8.8'}, - ], + 'parameters': [{'key': 'add-host', 'value': 'google-public-dns-a.google.com:8.8.8.8'}], }, 'volumes': [{'hostPath': u'/etc/stuff', diff --git a/marathon/models/base.py b/marathon/models/base.py index 6c4ad83..2ddc798 100644 --- a/marathon/models/base.py +++ b/marathon/models/base.py @@ -22,7 +22,7 @@ def json_repr(self, minimal=False): :rtype: dict """ if minimal: - return {to_camel_case(k): v for k, v in vars(self).items() if (v or v == False or v == 0)} + return {to_camel_case(k): v for k, v in vars(self).items() if (v or v is False or v == 0)} else: return {to_camel_case(k): v for k, v in vars(self).items()} diff --git a/marathon/models/constraint.py b/marathon/models/constraint.py index 2264f16..184d9f8 100644 --- a/marathon/models/constraint.py +++ b/marathon/models/constraint.py @@ -20,7 +20,7 @@ class MarathonConstraint(MarathonObject): """Valid operators""" def __init__(self, field, operator, value=None): - if not operator in self.OPERATORS: + if operator not in self.OPERATORS: raise InvalidChoiceError('operator', operator, self.OPERATORS) self.field = field self.operator = operator diff --git a/marathon/models/container.py b/marathon/models/container.py index 4719afd..3a3d2ce 100644 --- a/marathon/models/container.py +++ b/marathon/models/container.py @@ -19,7 +19,7 @@ class MarathonContainer(MarathonObject): """Valid container types""" def __init__(self, docker=None, type='DOCKER', volumes=None): - if not type in self.TYPES: + if type not in self.TYPES: raise InvalidChoiceError('type', type, self.TYPES) self.type = type self.docker = docker if isinstance(docker, MarathonDockerContainer) \ @@ -54,7 +54,7 @@ def __init__( force_pull_image=None, **kwargs): self.image = image if network: - if not network in self.NETWORK_MODES: + if network not in self.NETWORK_MODES: raise InvalidChoiceError( 'network', network, self.NETWORK_MODES) self.network = network @@ -87,7 +87,7 @@ def __init__(self, container_port=None, self.container_port = container_port self.host_port = host_port self.service_port = service_port - if not protocol in self.PROTOCOLS: + if protocol not in self.PROTOCOLS: raise InvalidChoiceError('protocol', protocol, self.PROTOCOLS) self.protocol = protocol @@ -108,6 +108,6 @@ class MarathonContainerVolume(MarathonObject): def __init__(self, container_path=None, host_path=None, mode='RW'): self.container_path = container_path self.host_path = host_path - if not mode in self.MODES: + if mode not in self.MODES: raise InvalidChoiceError('mode', mode, self.MODES) self.mode = mode diff --git a/marathon/models/deployment.py b/marathon/models/deployment.py index 6c864d8..29e24b6 100644 --- a/marathon/models/deployment.py +++ b/marathon/models/deployment.py @@ -31,7 +31,7 @@ def __init__( self.id = id self.steps = [ [step if isinstance(step, MarathonDeploymentAction) else MarathonDeploymentAction().from_json(step) - for step in s] + for step in s] for s in (steps or []) ] self.total_steps = total_steps diff --git a/marathon/models/endpoint.py b/marathon/models/endpoint.py index 7866eaf..4e9c45c 100644 --- a/marathon/models/endpoint.py +++ b/marathon/models/endpoint.py @@ -1,8 +1,3 @@ -try: - import json -except ImportError: - import simplejson as json - from .base import MarathonObject diff --git a/tests/test_api.py b/tests/test_api.py index a8c9174..fe58861 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,4 +1,3 @@ -import mock import requests_mock from marathon import MarathonClient from marathon import models diff --git a/tox.ini b/tox.ini index 4058c25..0e93077 100644 --- a/tox.ini +++ b/tox.ini @@ -46,6 +46,6 @@ deps = flake8 commands = flake8 . [flake8] -exclude = .tox,*.egg,docs,build +exclude = .tox,*.egg,docs,build,__init__.py ignore = E226,E302,E41,E501,E131 max-line-length = 160 From b91915d7c036b560e00d7597e935b67bf118886a Mon Sep 17 00:00:00 2001 From: Anatolii Lapytskyi Date: Fri, 15 Apr 2016 09:14:06 +0300 Subject: [PATCH 042/292] Make it possible to run itests at osx with docker-machine --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 0c164c4..45b45ea 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ basepython = python2.7 envlist = py [testenv:itests] -passenv = TRAVIS MARATHONVERSION +passenv = TRAVIS MARATHONVERSION DOCKER_HOST DOCKER_TLS_VERIFY DOCKER_CERT_PATH DOCKER_MACHINE_NAME basepython = python2.7 whitelist_externals=/bin/bash skipsdist=True From 48229c87d849c3d7eb7aed5b201f948068af821e Mon Sep 17 00:00:00 2001 From: Anatolii Lapytskyi Date: Fri, 15 Apr 2016 09:14:06 +0300 Subject: [PATCH 043/292] Make it possible to run itests at osx with docker-machine --- itests/itest_utils.py | 10 +++++++++- tox.ini | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/itests/itest_utils.py b/itests/itest_utils.py index fb0dcea..efca1e3 100644 --- a/itests/itest_utils.py +++ b/itests/itest_utils.py @@ -4,6 +4,7 @@ import signal import sys import threading +import re import time import requests @@ -64,7 +65,14 @@ def get_marathon_connection_string(): return 'localhost:8080' else: service_port = get_service_internal_port('marathon') - return get_compose_service('marathon').get_container().get_local_port(service_port) + local_port = get_compose_service('marathon').get_container().get_local_port(service_port) + + # Check if we're at OSX. Use ip from DOCKER_HOST + if sys.platform == 'darwin': + m = re.match("(.*?)://(.*?):(\d+)", os.environ["DOCKER_HOST"]) + local_port = "{}:{}".format(m.group(2), local_port.split(":")[1]) + + return local_port def get_service_internal_port(service_name): diff --git a/tox.ini b/tox.ini index 0c164c4..45b45ea 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ basepython = python2.7 envlist = py [testenv:itests] -passenv = TRAVIS MARATHONVERSION +passenv = TRAVIS MARATHONVERSION DOCKER_HOST DOCKER_TLS_VERIFY DOCKER_CERT_PATH DOCKER_MACHINE_NAME basepython = python2.7 whitelist_externals=/bin/bash skipsdist=True From 4806a5a8c1b12346d29890e0e36ab19f482672f2 Mon Sep 17 00:00:00 2001 From: Anatolii Lapytskyi Date: Fri, 15 Apr 2016 13:49:38 +0300 Subject: [PATCH 044/292] Increase timeout to allow Marathon to start, update CPU resource for test task --- itests/itest_utils.py | 2 +- itests/steps/marathon_steps.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/itests/itest_utils.py b/itests/itest_utils.py index efca1e3..d852033 100644 --- a/itests/itest_utils.py +++ b/itests/itest_utils.py @@ -33,7 +33,7 @@ def wrapper(*args, **kwargs): return decorator -@timeout(10) +@timeout(30) def wait_for_marathon(): """Blocks until marathon is up""" marathon_service = get_marathon_connection_string() diff --git a/itests/steps/marathon_steps.py b/itests/steps/marathon_steps.py index 24b0eb6..a1cfda2 100644 --- a/itests/steps/marathon_steps.py +++ b/itests/steps/marathon_steps.py @@ -25,7 +25,7 @@ def get_marathon_info(context): @when(u'we create a trivial new app') def create_trivial_new_app(context): - context.client.create_app('test-trivial-app', marathon.MarathonApp(cmd='sleep 3600', mem=16, cpus=1, instances=5)) + context.client.create_app('test-trivial-app', marathon.MarathonApp(cmd='sleep 3600', mem=16, cpus=0.1, instances=5)) @then(u'we should be able to kill the tasks') From f7e02df355c066d7f15fb0f7c0e02a5445aa98ae Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Thu, 14 Apr 2016 15:50:45 -0700 Subject: [PATCH 045/292] Support marathon .15 and 1.1 --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 9f1fda6..cec6395 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,8 @@ env: - MARATHONVERSION: 0.11.1 - MARATHONVERSION: 0.13.1 - MARATHONVERSION: 0.14.1 + - MARATHONVERSION: 0.15.3 + - MARATHONVERSION: 1.1.0 language: python python: From a1d9cef76d129e8df5fdf98bad366d927c9a3953 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Thu, 14 Apr 2016 16:23:00 -0700 Subject: [PATCH 046/292] Added readinessChecks model --- marathon/models/app.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/marathon/models/app.py b/marathon/models/app.py index cd1560a..4c6f019 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -58,6 +58,7 @@ class MarathonApp(MarathonResource): :param task_stats: task statistics :type task_stats: :class:`marathon.models.app.MarathonTaskStats` or dict :param dict labels + :type readiness_checks: list[:class:`marathon.models.app.ReadinessChecks`] or list[dict] """ UPDATE_OK_ATTRIBUTES = [ @@ -81,7 +82,7 @@ def __init__( max_launch_delay_seconds=None, mem=None, ports=None, require_ports=None, store_urls=None, task_rate_limit=None, tasks=None, tasks_running=None, tasks_staged=None, tasks_healthy=None, tasks_unhealthy=None, upgrade_strategy=None, uris=None, user=None, version=None, version_info=None, - ip_address=None, fetch=None, task_stats=None): + ip_address=None, fetch=None, task_stats=None, readiness_checks=None): # self.args = args or [] self.accepted_resource_roles = accepted_resource_roles @@ -122,6 +123,7 @@ def __init__( self.max_launch_delay_seconds = max_launch_delay_seconds self.mem = mem self.ports = ports or [] + self.readiness_checks = readiness_checks or [] self.require_ports = require_ports self.store_urls = store_urls or [] self.task_rate_limit = task_rate_limit @@ -342,3 +344,27 @@ class MarathonTaskStatsLifeTime(MarathonObject): def __init__(self, average_seconds=None, median_seconds=None): self.average_seconds = average_seconds self.median_seconds = median_seconds + +class ReadinessCheck(MarathonObject): + """Marathon readiness check: https://mesosphere.github.io/marathon/docs/readiness-checks.html + + :param string name (Optional. Default: "readinessCheck"): The name used to identify this readiness check. + :param string protocol (Optional. Default: "HTTP"): Protocol of the requests to be performed. Either HTTP or HTTPS. + :param string path (Optional. Default: "/"): Path to the endpoint the task exposes to provide readiness status. Example: /path/to/readiness. + :param string port_name (Optional. Default: "http-api"): Name of the port to query as described in the portDefinitions. Example: http-api. + :param int interval_seconds (Optional. Default: 30 seconds): Number of seconds to wait between readiness checks. + :param int timeout_seconds (Optional. Default: 10 seconds): Number of seconds after which a readiness check times out, regardless of the response. This value must be smaller than interval_seconds. + :param list http_status_codes_for_ready (Optional. Default: [200]): The HTTP/HTTPS status code to treat as ready. + :param bool preserve_last_response (Optional. Default: false): If true, the last readiness check response will be preserved and exposed in the API as part of a deployment. + + """ + + def __init__(self, name=None, protocol=None, path=None, port_name=None, interval_seconds=None, + http_status_codes_for_ready=None, preserve_last_response=None): + self.name = name + self.protocol = protocol + self.path = path + self.port_name = port_name + self.interval_seconds = interval_seconds + self.http_status_codes_for_ready = http_status_codes_for_ready + self.preserve_last_response = preserve_last_response From 24cddb9065ad0967334492a61eafa97afe88cc51 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Thu, 14 Apr 2016 17:49:43 -0700 Subject: [PATCH 047/292] Added port_definitions to the model --- marathon/models/app.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/marathon/models/app.py b/marathon/models/app.py index 4c6f019..967ab4f 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -38,6 +38,7 @@ class MarathonApp(MarathonResource): :param last_task_failure: last task failure :type last_task_failure: :class:`marathon.models.app.MarathonTaskFailure` or dict :param float mem: memory (in MB) required per instance + :type port_definitions: list[:class:`marathon.models.app.PortDefinitions`] or list[dict] :param list[int] ports: ports :param bool require_ports: require the specified `ports` to be available in the resource offer :param list[str] store_urls: store URLs @@ -58,7 +59,7 @@ class MarathonApp(MarathonResource): :param task_stats: task statistics :type task_stats: :class:`marathon.models.app.MarathonTaskStats` or dict :param dict labels - :type readiness_checks: list[:class:`marathon.models.app.ReadinessChecks`] or list[dict] + :type readiness_checks: list[:class:`marathon.models.app.ReadinessChecks`] or list[dict] """ UPDATE_OK_ATTRIBUTES = [ @@ -82,7 +83,7 @@ def __init__( max_launch_delay_seconds=None, mem=None, ports=None, require_ports=None, store_urls=None, task_rate_limit=None, tasks=None, tasks_running=None, tasks_staged=None, tasks_healthy=None, tasks_unhealthy=None, upgrade_strategy=None, uris=None, user=None, version=None, version_info=None, - ip_address=None, fetch=None, task_stats=None, readiness_checks=None): + ip_address=None, fetch=None, task_stats=None, readiness_checks=None, port_definitions=None): # self.args = args or [] self.accepted_resource_roles = accepted_resource_roles @@ -123,6 +124,7 @@ def __init__( self.max_launch_delay_seconds = max_launch_delay_seconds self.mem = mem self.ports = ports or [] + self.port_definitions = port_definitions or [] self.readiness_checks = readiness_checks or [] self.require_ports = require_ports self.store_urls = store_urls or [] @@ -368,3 +370,18 @@ def __init__(self, name=None, protocol=None, path=None, port_name=None, interval self.interval_seconds = interval_seconds self.http_status_codes_for_ready = http_status_codes_for_ready self.preserve_last_response = preserve_last_response + +class PortDefinition(MarathonObject): + """Marathon port definitions: https://mesosphere.github.io/marathon/docs/ports.html + + :param int port: The port + :param string protocol: tcp or udp + :param string name: (optional) the name of the port + :param dict labels: undocumented + """ + + def __init__(self, port=None, protocol=None, name=None, labels=None): + self.port = port + self.protocol = protocol + self.name = name + self.labels = labels From ae609bc6189bda61933fadd7e6b84a51e81ce7bd Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Fri, 15 Apr 2016 12:47:15 -0700 Subject: [PATCH 048/292] Added residency to the app api --- marathon/models/app.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/marathon/models/app.py b/marathon/models/app.py index 967ab4f..c208ebf 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -60,6 +60,7 @@ class MarathonApp(MarathonResource): :type task_stats: :class:`marathon.models.app.MarathonTaskStats` or dict :param dict labels :type readiness_checks: list[:class:`marathon.models.app.ReadinessChecks`] or list[dict] + :type residency: :class:`marathon.models.app.Residency` or dict """ UPDATE_OK_ATTRIBUTES = [ @@ -83,7 +84,7 @@ def __init__( max_launch_delay_seconds=None, mem=None, ports=None, require_ports=None, store_urls=None, task_rate_limit=None, tasks=None, tasks_running=None, tasks_staged=None, tasks_healthy=None, tasks_unhealthy=None, upgrade_strategy=None, uris=None, user=None, version=None, version_info=None, - ip_address=None, fetch=None, task_stats=None, readiness_checks=None, port_definitions=None): + ip_address=None, fetch=None, task_stats=None, readiness_checks=None, port_definitions=None, residency=None): # self.args = args or [] self.accepted_resource_roles = accepted_resource_roles @@ -126,6 +127,7 @@ def __init__( self.ports = ports or [] self.port_definitions = port_definitions or [] self.readiness_checks = readiness_checks or [] + self.residency = residency self.require_ports = require_ports self.store_urls = store_urls or [] self.task_rate_limit = task_rate_limit @@ -148,6 +150,7 @@ def __init__( else MarathonTaskStats.from_json(task_stats) + class MarathonHealthCheck(MarathonObject): """Marathon health check. @@ -385,3 +388,15 @@ def __init__(self, port=None, protocol=None, name=None, labels=None): self.protocol = protocol self.name = name self.labels = labels + +class Residency(MarathonObject): + """Declares how "resident" an app is: https://mesosphere.github.io/marathon/docs/persistent-volumes.html + + :param int relaunch_escalation_timeout_seconds: How long marathon will try to relaunch where the volumes is, defaults to 3600 + :param string task_lost_behavior: What to do after a TASK_LOST. See the official Marathon docs for options + + """ + + def __init__(self, relaunch_escalation_timeout_seconds=None, task_lost_behavior=None): + self.relaunch_escalation_timeout_seconds = relaunch_escalation_timeout_seconds + self.task_lost_behavior = task_lost_behavior From 74720034820c92d9d66798d7156da6ee2fcca0ed Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Fri, 15 Apr 2016 13:21:12 -0700 Subject: [PATCH 049/292] Added labels to portmapping api --- marathon/models/app.py | 1 - marathon/models/container.py | 5 +++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/marathon/models/app.py b/marathon/models/app.py index c208ebf..d26b3f9 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -150,7 +150,6 @@ def __init__( else MarathonTaskStats.from_json(task_stats) - class MarathonHealthCheck(MarathonObject): """Marathon health check. diff --git a/marathon/models/container.py b/marathon/models/container.py index 3a3d2ce..2224200 100644 --- a/marathon/models/container.py +++ b/marathon/models/container.py @@ -77,19 +77,20 @@ class MarathonContainerPortMapping(MarathonObject): :param int container_port: :param int host_port: :param str protocol: + :param object labels: """ PROTOCOLS = ['tcp', 'udp'] """Valid protocols""" - def __init__(self, container_port=None, - host_port=0, service_port=None, protocol='tcp'): + def __init__(self, container_port=None, host_port=0, service_port=None, protocol='tcp', labels=None): self.container_port = container_port self.host_port = host_port self.service_port = service_port if protocol not in self.PROTOCOLS: raise InvalidChoiceError('protocol', protocol, self.PROTOCOLS) self.protocol = protocol + self.labels = labels class MarathonContainerVolume(MarathonObject): From 0ee8646a1ab53c1cb90b7341a8dd8c03a69c8372 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Fri, 15 Apr 2016 13:40:58 -0700 Subject: [PATCH 050/292] Added task_reservation_timeout to v2/info endpoint --- marathon/models/info.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/marathon/models/info.py b/marathon/models/info.py index 3ab77d5..b07b88a 100644 --- a/marathon/models/info.py +++ b/marathon/models/info.py @@ -5,7 +5,8 @@ class MarathonInfo(MarathonResource): """Marathon Info. - See: https://mesosphere.github.io/marathon/docs/rest-api.html#get-v2-info + See: https://mesosphere.github.io/marathon/docs/rest-api.html#get-v2-info + Also: https://mesosphere.github.io/marathon/docs/generated/api.html#v2_info_get :param str framework_id: :param str leader: @@ -69,6 +70,7 @@ class MarathonConfig(MarathonObject): :param int reconciliation_initial_delay: :param int reconciliation_interval: :param int task_launch_timeout: + :param int task_reservation_timeout: :param int marathon_store_timeout: """ @@ -77,7 +79,7 @@ def __init__( hostname=None, leader_proxy_connection_timeout_ms=None, leader_proxy_read_timeout_ms=None, local_port_min=None, local_port_max=None, master=None, mesos_leader_ui_url=None, mesos_role=None, mesos_user=None, webui_url=None, reconciliation_initial_delay=None, reconciliation_interval=None, - task_launch_timeout=None, marathon_store_timeout=None): + task_launch_timeout=None, marathon_store_timeout=None, task_reservation_timeout=None): self.checkpoint = checkpoint self.executor = executor self.failover_timeout = failover_timeout @@ -93,6 +95,7 @@ def __init__( self.reconciliation_initial_delay = reconciliation_initial_delay self.reconciliation_interval = reconciliation_interval self.task_launch_timeout = task_launch_timeout + self.task_reservation_timeout = task_reservation_timeout self.marathon_store_timeout = marathon_store_timeout From 67b7380146d7d0410f0931795ca9db9b229227c5 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Fri, 15 Apr 2016 15:00:26 -0700 Subject: [PATCH 051/292] Added features attribute to MarathonConfig --- marathon/models/info.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/marathon/models/info.py b/marathon/models/info.py index b07b88a..d3144d6 100644 --- a/marathon/models/info.py +++ b/marathon/models/info.py @@ -55,6 +55,7 @@ class MarathonConfig(MarathonObject): :param bool checkpoint: :param str executor: :param int failover_timeout: + :param type features: Undocumented object :param str framework_name: :param bool ha: :param str hostname: @@ -79,10 +80,11 @@ def __init__( hostname=None, leader_proxy_connection_timeout_ms=None, leader_proxy_read_timeout_ms=None, local_port_min=None, local_port_max=None, master=None, mesos_leader_ui_url=None, mesos_role=None, mesos_user=None, webui_url=None, reconciliation_initial_delay=None, reconciliation_interval=None, - task_launch_timeout=None, marathon_store_timeout=None, task_reservation_timeout=None): + task_launch_timeout=None, marathon_store_timeout=None, task_reservation_timeout=None, features=None): self.checkpoint = checkpoint self.executor = executor self.failover_timeout = failover_timeout + self.features = features self.ha = ha self.hostname = hostname self.local_port_min = local_port_min From 526c9fecd3978dec6c050c9502e38659c1a7ffd4 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Fri, 15 Apr 2016 15:22:55 -0700 Subject: [PATCH 052/292] Remove some travis and update the readme --- .travis.yml | 4 +--- README.md | 21 +++++++++++++++------ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index cec6395..5478adb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,10 @@ env: - - MARATHONVERSION: 0.8.2 - - MARATHONVERSION: 0.9.1 - MARATHONVERSION: 0.10.1 - MARATHONVERSION: 0.11.1 - MARATHONVERSION: 0.13.1 - MARATHONVERSION: 0.14.1 - MARATHONVERSION: 0.15.3 - - MARATHONVERSION: 1.1.0 + - MARATHONVERSION: 1.1.1 language: python python: diff --git a/README.md b/README.md index df1ecf7..d578db8 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,23 @@ This is a Python library for interfacing with [Marathon](https://github.com/meso #### Compatibility -marathon-python is primarily developed against Marathon 0.8.x (see [Marathon docs](https://mesosphere.github.io/marathon/)) - -* For Marathon greater than 0.14.x: Experimental support in 0.7.6 -* For Marathon 0.8.x-0.11.x, use marathon-python 0.7.5 -* For Marathon 0.8.x-0.9.x, use marathon-python 0.6.11 - 0.7.4 -* For Marathon 0.7.x, use marathon-python 0.6.10 +* For Marathon 1.1.1 and 0.15.x, use at least 0.7.8 +* For Marathon 0.14.x, use at least 0.7.6 +* For Marathon 0.8.x-0.11.x, use at least marathon-python 0.7.5 +* For Marathon 0.8.x-0.9.x, use as least marathon-python 0.6.11 - 0.7.4 +* For Marathon 0.7.x, use at least marathon-python 0.6.10 * For all version changes, please see `CHANGELOG.md` +Note: Not all versions of Python are tested against every version of Marathon. + +If you find a feature that is broken, please submit a PR that adds a test for +it so it will be fixed and will continue to stay fixed as Marathon changes over +time. + +Just because this library is tested against a specific version of Marathon, +doesn't necessarily mean that it supports every feature and API Marathon +provides. + ## Installation #### From PyPi (recommended) From a4424fd1634d018a9709c11b6ce4e6273d67f847 Mon Sep 17 00:00:00 2001 From: Anatolii Lapytskyi Date: Sat, 16 Apr 2016 12:03:02 +0300 Subject: [PATCH 053/292] Add tests for event stream --- itests/marathon_tasks.feature | 8 +++++++ itests/steps/marathon_steps.py | 38 ++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/itests/marathon_tasks.feature b/itests/marathon_tasks.feature index f9b68f4..c32cc7e 100644 --- a/itests/marathon_tasks.feature +++ b/itests/marathon_tasks.feature @@ -17,3 +17,11 @@ Feature: marathon-python can operate marathon app tasks When we create a trivial new app And we wait the trivial app deployment finish Then we should be able to kill the #0,1,2 tasks of the trivial app + + Scenario: Events can be listened in stream + Given a working marathon instance + When we start listening for events + And we create a trivial new app + And we wait the trivial app deployment finish + Then we should be able to kill the tasks + And we should see list of events diff --git a/itests/steps/marathon_steps.py b/itests/steps/marathon_steps.py index a1cfda2..0b415e9 100644 --- a/itests/steps/marathon_steps.py +++ b/itests/steps/marathon_steps.py @@ -1,5 +1,6 @@ import sys import time +import multiprocessing import marathon from behave import given, when, then @@ -105,3 +106,40 @@ def list_tasks(context, which): app = context.client.get_app('test-%s-app' % which) tasks = context.client.list_tasks('test-%s-app' % which) assert len(tasks) == app.instances + + +def listen_for_events(client, events): + for msg in client.event_stream(): + events.append(msg) + + +@when(u'we start listening for events') +def start_listening_stream(context): + manager = multiprocessing.Manager() + mlist = manager.list() + context.manager = manager + context.events = mlist + p = multiprocessing.Process(target=listen_for_events, args=(context.client, mlist)) + p.start() + context.p = p + +@then(u'we should see list of events') +def stop_listening_stream(context): + time.sleep(10) + context.p.terminate() + + # event list should contain 5 status_update_event with taskStatus == TASK_RUNNING + filtered_events = [e for e in context.events if e.event_type == "status_update_event" and e.task_status == "TASK_RUNNING"] + assert len(filtered_events) == 5 + + # and 1 status_update_event with taskStatus == TASK_KILLED + filtered_events = [e for e in context.events if e.event_type == "status_update_event" and e.task_status == "TASK_KILLED"] + assert len(filtered_events) == 1 + + # and 1 deployment_step_success events with target instances == 5 + filtered_events = [e for e in context.events if e.event_type == "deployment_step_success" and e.plan.target.apps[0]['instances'] == 5] + assert len(filtered_events) == 1 + + # and 1 deployment_step_success events with target instances == 4 + filtered_events = [e for e in context.events if e.event_type == "deployment_step_success" and e.plan.target.apps[0]['instances'] == 4] + assert len(filtered_events) == 1 From e8ba5cac24bd9d5616fdc26b03359becd90d79fd Mon Sep 17 00:00:00 2001 From: Anatolii Lapytskyi Date: Sat, 16 Apr 2016 14:45:56 +0300 Subject: [PATCH 054/292] Add checking marathon version --- itests/marathon_tasks.feature | 3 ++- itests/steps/marathon_steps.py | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/itests/marathon_tasks.feature b/itests/marathon_tasks.feature index c32cc7e..de8d042 100644 --- a/itests/marathon_tasks.feature +++ b/itests/marathon_tasks.feature @@ -20,7 +20,8 @@ Feature: marathon-python can operate marathon app tasks Scenario: Events can be listened in stream Given a working marathon instance - When we start listening for events + When marathon version is greater than 0.9.0 + And we start listening for events And we create a trivial new app And we wait the trivial app deployment finish Then we should be able to kill the tasks diff --git a/itests/steps/marathon_steps.py b/itests/steps/marathon_steps.py index 0b415e9..8c1dcfb 100644 --- a/itests/steps/marathon_steps.py +++ b/itests/steps/marathon_steps.py @@ -1,6 +1,7 @@ import sys import time import multiprocessing +from distutils.version import LooseVersion, StrictVersion import marathon from behave import given, when, then @@ -112,6 +113,11 @@ def listen_for_events(client, events): for msg in client.event_stream(): events.append(msg) +@when(u'marathon version is greater than {version}') +def marathon_version_chech(context, version): + info = context.client.get_info() + if StrictVersion(info.version) < StrictVersion(version): + context.scenario.skip(reason='Marathon version is too low for this scenario') @when(u'we start listening for events') def start_listening_stream(context): From 6fc5896529983c4111a338261898e9f11ab2fb0c Mon Sep 17 00:00:00 2001 From: Anatolii Lapytskyi Date: Sat, 16 Apr 2016 19:47:55 +0300 Subject: [PATCH 055/292] Reorganize test slightly --- itests/steps/marathon_steps.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/itests/steps/marathon_steps.py b/itests/steps/marathon_steps.py index 8c1dcfb..89507d7 100644 --- a/itests/steps/marathon_steps.py +++ b/itests/steps/marathon_steps.py @@ -138,14 +138,14 @@ def stop_listening_stream(context): filtered_events = [e for e in context.events if e.event_type == "status_update_event" and e.task_status == "TASK_RUNNING"] assert len(filtered_events) == 5 - # and 1 status_update_event with taskStatus == TASK_KILLED - filtered_events = [e for e in context.events if e.event_type == "status_update_event" and e.task_status == "TASK_KILLED"] - assert len(filtered_events) == 1 - # and 1 deployment_step_success events with target instances == 5 filtered_events = [e for e in context.events if e.event_type == "deployment_step_success" and e.plan.target.apps[0]['instances'] == 5] assert len(filtered_events) == 1 + # and 1 status_update_event with taskStatus == TASK_KILLED + filtered_events = [e for e in context.events if e.event_type == "status_update_event" and e.task_status == "TASK_KILLED"] + assert len(filtered_events) == 1 + # and 1 deployment_step_success events with target instances == 4 filtered_events = [e for e in context.events if e.event_type == "deployment_step_success" and e.plan.target.apps[0]['instances'] == 4] assert len(filtered_events) == 1 From e0940a0252002be5b2123feabdd77a0e086b22b8 Mon Sep 17 00:00:00 2001 From: Anatolii Lapytskyi Date: Thu, 14 Apr 2016 19:18:39 +0300 Subject: [PATCH 056/292] Add support for /v2/events stream --- marathon/client.py | 37 +++++++++++++++++++++++++++++++++++++ marathon/models/events.py | 9 ++++++++- requirements.txt | 1 + 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/marathon/client.py b/marathon/client.py index 8e59f7d..f4e4503 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -12,6 +12,7 @@ import marathon from .models import MarathonApp, MarathonDeployment, MarathonGroup, MarathonInfo, MarathonTask, MarathonEndpoint, MarathonQueueItem from .exceptions import InternalServerError, NotFoundError, MarathonHttpError, MarathonError +from .models.events import EventFactory class MarathonClient(object): @@ -89,6 +90,26 @@ def _do_request(self, method, path, params=None, data=None): return response + def _do_sse_request(self, path, params=None, data=None): + from sseclient import SSEClient + + headers = {'Accept': 'text/event-stream'} + messages = None + servers = list(self.servers) + while servers and messages is None: + server = servers.pop(0) + url = ''.join([server.rstrip('/'), path]) + try: + messages = SSEClient(url,params=params, data=data, headers=headers, + auth=self.auth) + except Exception as e: + marathon.log.error('Error while calling %s: %s', url, e.message) + + if messages is None: + raise MarathonError('No remaining Marathon servers to try') + + return messages + def list_endpoints(self): """List the current endpoints for all applications @@ -631,3 +652,19 @@ def get_metrics(self): """ response = self._do_request('GET', '/metrics') return response.json() + + def event_stream(self): + """Polls event bus using /v2/events + + :returns: iterator with events + :rtype: iterator + """ + + messages = self._do_sse_request('/v2/events') + + ef = EventFactory() + for message in messages: + if not message.data: + continue + data = json.loads(message.data) + yield ef.process(data) diff --git a/marathon/models/events.py b/marathon/models/events.py index 6157deb..1c7a0f9 100644 --- a/marathon/models/events.py +++ b/marathon/models/events.py @@ -106,6 +106,11 @@ class MarathonDeploymentStepSuccess(MarathonEvent): class MarathonDeploymentStepFailure(MarathonEvent): KNOWN_ATTRIBUTES = ['plan'] +class MarathonEventStreamAttached(MarathonEvent): + KNOWN_ATTRIBUTES = ['remote_address'] + +class MarathonEventStreamDetached(MarathonEvent): + KNOWN_ATTRIBUTES = ['remote_address'] class EventFactory: @@ -133,7 +138,9 @@ def __init__(self): 'deployment_failed': MarathonDeploymentFailed, 'deployment_info': MarathonDeploymentInfo, 'deployment_step_success': MarathonDeploymentStepSuccess, - 'deployment_step_failure': MarathonDeploymentStepFailure + 'deployment_step_failure': MarathonDeploymentStepFailure, + 'event_stream_attached': MarathonEventStreamAttached, + 'event_stream_detached': MarathonEventStreamDetached, } def process(self, event): diff --git a/requirements.txt b/requirements.txt index 40e7bdf..cb1d49e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ requests-mock +sseclient From aff5bb4ce4b1db6140536045ad980665b647ab9c Mon Sep 17 00:00:00 2001 From: Anatolii Lapytskyi Date: Thu, 14 Apr 2016 20:25:02 +0300 Subject: [PATCH 057/292] Add sseclient dependency to setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a41a6f8..2eb60ef 100755 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ long_description="""Python interface to the Mesos Marathon REST API.""", author='Mike Babineau', author_email='michael.babineau@gmail.com', - install_requires=['requests>=2.0.0'], + install_requires=['requests>=2.0.0', 'sseclient'], url='https://github.com/thefactory/marathon-python', packages=['marathon', 'marathon.models'], license='MIT', From 34f8244091fd32f605a5c2b45ff89c580fcf8113 Mon Sep 17 00:00:00 2001 From: Anatolii Lapytskyi Date: Fri, 15 Apr 2016 09:14:06 +0300 Subject: [PATCH 058/292] Make it possible to run itests at osx with docker-machine --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 0e93077..005f07a 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ basepython = python2.7 envlist = py,pep8 [testenv:itests] -passenv = TRAVIS MARATHONVERSION +passenv = TRAVIS MARATHONVERSION DOCKER_HOST DOCKER_TLS_VERIFY DOCKER_CERT_PATH DOCKER_MACHINE_NAME basepython = python2.7 whitelist_externals=/bin/bash skipsdist=True From 6c44763e008a0f69d1f20852472fb9d89032691b Mon Sep 17 00:00:00 2001 From: Anatolii Lapytskyi Date: Fri, 15 Apr 2016 09:14:06 +0300 Subject: [PATCH 059/292] Make it possible to run itests at osx with docker-machine --- itests/itest_utils.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/itests/itest_utils.py b/itests/itest_utils.py index 66be2b9..9bd6f79 100644 --- a/itests/itest_utils.py +++ b/itests/itest_utils.py @@ -2,6 +2,9 @@ from functools import wraps import os import signal +import sys +import threading +import re import time import requests @@ -64,7 +67,14 @@ def get_marathon_connection_string(): return 'localhost:8080' else: service_port = get_service_internal_port('marathon') - return get_compose_service('marathon').get_container().get_local_port(service_port) + local_port = get_compose_service('marathon').get_container().get_local_port(service_port) + + # Check if we're at OSX. Use ip from DOCKER_HOST + if sys.platform == 'darwin': + m = re.match("(.*?)://(.*?):(\d+)", os.environ["DOCKER_HOST"]) + local_port = "{}:{}".format(m.group(2), local_port.split(":")[1]) + + return local_port def get_service_internal_port(service_name): From 9c2536c9e8a77c0679db153c251d33241b709d1b Mon Sep 17 00:00:00 2001 From: Anatolii Lapytskyi Date: Fri, 15 Apr 2016 13:49:38 +0300 Subject: [PATCH 060/292] Increase timeout to allow Marathon to start, update CPU resource for test task --- itests/itest_utils.py | 2 +- itests/steps/marathon_steps.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/itests/itest_utils.py b/itests/itest_utils.py index 9bd6f79..2b54c00 100644 --- a/itests/itest_utils.py +++ b/itests/itest_utils.py @@ -34,7 +34,7 @@ def wrapper(*args, **kwargs): return decorator -@timeout(10) +@timeout(30) def wait_for_marathon(): """Blocks until marathon is up""" marathon_service = get_marathon_connection_string() diff --git a/itests/steps/marathon_steps.py b/itests/steps/marathon_steps.py index ac329c6..7155d18 100644 --- a/itests/steps/marathon_steps.py +++ b/itests/steps/marathon_steps.py @@ -26,7 +26,7 @@ def get_marathon_info(context): @when(u'we create a trivial new app') def create_trivial_new_app(context): context.client.create_app('test-trivial-app', marathon.MarathonApp( - cmd='sleep 3600', mem=16, cpus=1, instances=5)) + cmd='sleep 3600', mem=16, cpus=0.1, instances=5)) @then(u'we should be able to kill the tasks') From c1625a4796525b81307bcd942e11bfd0e66f8225 Mon Sep 17 00:00:00 2001 From: Anatolii Lapytskyi Date: Sat, 16 Apr 2016 12:03:02 +0300 Subject: [PATCH 061/292] Add tests for event stream --- itests/marathon_tasks.feature | 8 +++++++ itests/steps/marathon_steps.py | 38 ++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/itests/marathon_tasks.feature b/itests/marathon_tasks.feature index f9b68f4..c32cc7e 100644 --- a/itests/marathon_tasks.feature +++ b/itests/marathon_tasks.feature @@ -17,3 +17,11 @@ Feature: marathon-python can operate marathon app tasks When we create a trivial new app And we wait the trivial app deployment finish Then we should be able to kill the #0,1,2 tasks of the trivial app + + Scenario: Events can be listened in stream + Given a working marathon instance + When we start listening for events + And we create a trivial new app + And we wait the trivial app deployment finish + Then we should be able to kill the tasks + And we should see list of events diff --git a/itests/steps/marathon_steps.py b/itests/steps/marathon_steps.py index 7155d18..46db2aa 100644 --- a/itests/steps/marathon_steps.py +++ b/itests/steps/marathon_steps.py @@ -1,5 +1,6 @@ import sys import time +import multiprocessing import marathon from behave import given, when, then @@ -113,3 +114,40 @@ def list_tasks(context, which): app = context.client.get_app('test-%s-app' % which) tasks = context.client.list_tasks('test-%s-app' % which) assert len(tasks) == app.instances + + +def listen_for_events(client, events): + for msg in client.event_stream(): + events.append(msg) + + +@when(u'we start listening for events') +def start_listening_stream(context): + manager = multiprocessing.Manager() + mlist = manager.list() + context.manager = manager + context.events = mlist + p = multiprocessing.Process(target=listen_for_events, args=(context.client, mlist)) + p.start() + context.p = p + +@then(u'we should see list of events') +def stop_listening_stream(context): + time.sleep(10) + context.p.terminate() + + # event list should contain 5 status_update_event with taskStatus == TASK_RUNNING + filtered_events = [e for e in context.events if e.event_type == "status_update_event" and e.task_status == "TASK_RUNNING"] + assert len(filtered_events) == 5 + + # and 1 status_update_event with taskStatus == TASK_KILLED + filtered_events = [e for e in context.events if e.event_type == "status_update_event" and e.task_status == "TASK_KILLED"] + assert len(filtered_events) == 1 + + # and 1 deployment_step_success events with target instances == 5 + filtered_events = [e for e in context.events if e.event_type == "deployment_step_success" and e.plan.target.apps[0]['instances'] == 5] + assert len(filtered_events) == 1 + + # and 1 deployment_step_success events with target instances == 4 + filtered_events = [e for e in context.events if e.event_type == "deployment_step_success" and e.plan.target.apps[0]['instances'] == 4] + assert len(filtered_events) == 1 From 8119e9db6c0c2360c14a895ebe59ef57288f31e1 Mon Sep 17 00:00:00 2001 From: Anatolii Lapytskyi Date: Sat, 16 Apr 2016 14:45:56 +0300 Subject: [PATCH 062/292] Add checking marathon version --- itests/marathon_tasks.feature | 3 ++- itests/steps/marathon_steps.py | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/itests/marathon_tasks.feature b/itests/marathon_tasks.feature index c32cc7e..de8d042 100644 --- a/itests/marathon_tasks.feature +++ b/itests/marathon_tasks.feature @@ -20,7 +20,8 @@ Feature: marathon-python can operate marathon app tasks Scenario: Events can be listened in stream Given a working marathon instance - When we start listening for events + When marathon version is greater than 0.9.0 + And we start listening for events And we create a trivial new app And we wait the trivial app deployment finish Then we should be able to kill the tasks diff --git a/itests/steps/marathon_steps.py b/itests/steps/marathon_steps.py index 46db2aa..ca0445b 100644 --- a/itests/steps/marathon_steps.py +++ b/itests/steps/marathon_steps.py @@ -1,6 +1,7 @@ import sys import time import multiprocessing +from distutils.version import LooseVersion, StrictVersion import marathon from behave import given, when, then @@ -120,6 +121,11 @@ def listen_for_events(client, events): for msg in client.event_stream(): events.append(msg) +@when(u'marathon version is greater than {version}') +def marathon_version_chech(context, version): + info = context.client.get_info() + if StrictVersion(info.version) < StrictVersion(version): + context.scenario.skip(reason='Marathon version is too low for this scenario') @when(u'we start listening for events') def start_listening_stream(context): From 39c219c69df626aaf42bb0f46b2de19c49c6d9ff Mon Sep 17 00:00:00 2001 From: Anatolii Lapytskyi Date: Sat, 16 Apr 2016 19:47:55 +0300 Subject: [PATCH 063/292] Reorganize test slightly --- itests/steps/marathon_steps.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/itests/steps/marathon_steps.py b/itests/steps/marathon_steps.py index ca0445b..de59f91 100644 --- a/itests/steps/marathon_steps.py +++ b/itests/steps/marathon_steps.py @@ -146,14 +146,14 @@ def stop_listening_stream(context): filtered_events = [e for e in context.events if e.event_type == "status_update_event" and e.task_status == "TASK_RUNNING"] assert len(filtered_events) == 5 - # and 1 status_update_event with taskStatus == TASK_KILLED - filtered_events = [e for e in context.events if e.event_type == "status_update_event" and e.task_status == "TASK_KILLED"] - assert len(filtered_events) == 1 - # and 1 deployment_step_success events with target instances == 5 filtered_events = [e for e in context.events if e.event_type == "deployment_step_success" and e.plan.target.apps[0]['instances'] == 5] assert len(filtered_events) == 1 + # and 1 status_update_event with taskStatus == TASK_KILLED + filtered_events = [e for e in context.events if e.event_type == "status_update_event" and e.task_status == "TASK_KILLED"] + assert len(filtered_events) == 1 + # and 1 deployment_step_success events with target instances == 4 filtered_events = [e for e in context.events if e.event_type == "deployment_step_success" and e.plan.target.apps[0]['instances'] == 4] assert len(filtered_events) == 1 From 4ddd588104fce81f1b5f003fb18aa758b48316fa Mon Sep 17 00:00:00 2001 From: Anatolii Lapytskyi Date: Sun, 17 Apr 2016 12:16:48 +0300 Subject: [PATCH 064/292] Add debug --- itests/steps/marathon_steps.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/itests/steps/marathon_steps.py b/itests/steps/marathon_steps.py index c6c665c..6948692 100644 --- a/itests/steps/marathon_steps.py +++ b/itests/steps/marathon_steps.py @@ -142,6 +142,8 @@ def stop_listening_stream(context): time.sleep(10) context.p.terminate() + print(context.events) + # event list should contain 5 status_update_event with taskStatus == TASK_RUNNING filtered_events = [e for e in context.events if e.event_type == "status_update_event" and e.task_status == "TASK_RUNNING"] assert len(filtered_events) == 5 From ae5dea9cc4fd05d5d962a8132bad99038b3dc4aa Mon Sep 17 00:00:00 2001 From: Anatolii Lapytskyi Date: Sun, 17 Apr 2016 12:56:36 +0300 Subject: [PATCH 065/292] Make event test simplier --- itests/steps/marathon_steps.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/itests/steps/marathon_steps.py b/itests/steps/marathon_steps.py index 6948692..6df44b0 100644 --- a/itests/steps/marathon_steps.py +++ b/itests/steps/marathon_steps.py @@ -148,14 +148,10 @@ def stop_listening_stream(context): filtered_events = [e for e in context.events if e.event_type == "status_update_event" and e.task_status == "TASK_RUNNING"] assert len(filtered_events) == 5 - # and 1 deployment_step_success events with target instances == 5 - filtered_events = [e for e in context.events if e.event_type == "deployment_step_success" and e.plan.target.apps[0]['instances'] == 5] - assert len(filtered_events) == 1 - # and 1 status_update_event with taskStatus == TASK_KILLED filtered_events = [e for e in context.events if e.event_type == "status_update_event" and e.task_status == "TASK_KILLED"] assert len(filtered_events) == 1 - # and 1 deployment_step_success events with target instances == 4 - filtered_events = [e for e in context.events if e.event_type == "deployment_step_success" and e.plan.target.apps[0]['instances'] == 4] + # and 2 deployment_step_success events + filtered_events = [e for e in context.events if e.event_type == "deployment_success"] assert len(filtered_events) == 1 From efc8802f4fd2f7b697aab2936afd91dc25cd0f4e Mon Sep 17 00:00:00 2001 From: Anatolii Lapytskyi Date: Sun, 17 Apr 2016 13:07:41 +0300 Subject: [PATCH 066/292] Fix test --- itests/steps/marathon_steps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/itests/steps/marathon_steps.py b/itests/steps/marathon_steps.py index 6df44b0..a004f89 100644 --- a/itests/steps/marathon_steps.py +++ b/itests/steps/marathon_steps.py @@ -154,4 +154,4 @@ def stop_listening_stream(context): # and 2 deployment_step_success events filtered_events = [e for e in context.events if e.event_type == "deployment_success"] - assert len(filtered_events) == 1 + assert len(filtered_events) == 2 From 8a46ea7ca00d5b0d5895fff4d567225e590e9837 Mon Sep 17 00:00:00 2001 From: Anatolii Lapytskyi Date: Sun, 17 Apr 2016 13:20:11 +0300 Subject: [PATCH 067/292] It looks like the events are correctly captured starting from 0.11 --- itests/marathon_tasks.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/itests/marathon_tasks.feature b/itests/marathon_tasks.feature index de8d042..c3e53fc 100644 --- a/itests/marathon_tasks.feature +++ b/itests/marathon_tasks.feature @@ -20,7 +20,7 @@ Feature: marathon-python can operate marathon app tasks Scenario: Events can be listened in stream Given a working marathon instance - When marathon version is greater than 0.9.0 + When marathon version is greater than 0.11.0 And we start listening for events And we create a trivial new app And we wait the trivial app deployment finish From 0c9d0795423f29bf4a0584c9272705b08fcdf339 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Mon, 18 Apr 2016 13:46:03 -0700 Subject: [PATCH 068/292] Release 0.8.0 --- CHANGELOG.md | 14 ++++++++++++++ README.md | 2 +- setup.py | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64a5435..489ae38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Change Log +## [0.8.0](https://github.com/thefactory/marathon-python/tree/0.8.0) (2016-04-18) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.7.7...0.8.0) + +**Closed issues:** + +- 0.7.7 release [\#90](https://github.com/thefactory/marathon-python/issues/90) + +**Merged pull requests:** + +- Comply with pep8 betterer [\#93](https://github.com/thefactory/marathon-python/pull/93) ([solarkennedy](https://github.com/solarkennedy)) +- Support marathon .15 and 1.1 [\#92](https://github.com/thefactory/marathon-python/pull/92) ([solarkennedy](https://github.com/solarkennedy)) +- Add support for /v2/events stream [\#91](https://github.com/thefactory/marathon-python/pull/91) ([nuclon](https://github.com/nuclon)) +- update for v2/queue and v2/apps?embed=apps.taskStats [\#89](https://github.com/thefactory/marathon-python/pull/89) ([bergerx](https://github.com/bergerx)) + ## [0.7.7](https://github.com/thefactory/marathon-python/tree/0.7.7) (2016-02-29) [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.7.6...0.7.7) diff --git a/README.md b/README.md index d578db8..9397d88 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ This is a Python library for interfacing with [Marathon](https://github.com/meso #### Compatibility -* For Marathon 1.1.1 and 0.15.x, use at least 0.7.8 +* For Marathon 1.1.1 and 0.15.x, use at least 0.8.0 * For Marathon 0.14.x, use at least 0.7.6 * For Marathon 0.8.x-0.11.x, use at least marathon-python 0.7.5 * For Marathon 0.8.x-0.9.x, use as least marathon-python 0.6.11 - 0.7.4 diff --git a/setup.py b/setup.py index 2eb60ef..7936ee4 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='marathon', - version='0.7.7', + version='0.8.0', description='Marathon Client Library', long_description="""Python interface to the Mesos Marathon REST API.""", author='Mike Babineau', From c4c8e17b1e07726b4fa9c7c4b9da463c1ca0fcb7 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Mon, 18 Apr 2016 14:49:08 -0700 Subject: [PATCH 069/292] Added itest for the deployments endpoint --- itests/marathon_deployments.feature | 6 ++++++ itests/steps/marathon_steps.py | 4 ++++ 2 files changed, 10 insertions(+) create mode 100644 itests/marathon_deployments.feature diff --git a/itests/marathon_deployments.feature b/itests/marathon_deployments.feature new file mode 100644 index 0000000..cfacb2f --- /dev/null +++ b/itests/marathon_deployments.feature @@ -0,0 +1,6 @@ +Feature: marathon-python read deployments + + Scenario: deployments can be read + Given a working marathon instance + When we create a trivial new app + Then we should be able to see a deployment diff --git a/itests/steps/marathon_steps.py b/itests/steps/marathon_steps.py index a004f89..8f4b2ef 100644 --- a/itests/steps/marathon_steps.py +++ b/itests/steps/marathon_steps.py @@ -155,3 +155,7 @@ def stop_listening_stream(context): # and 2 deployment_step_success events filtered_events = [e for e in context.events if e.event_type == "deployment_success"] assert len(filtered_events) == 2 + +@then('we should be able to see a deployment') +def see_a_deployment(context): + assert len(context.client.list_deployments() == 1) From 154fce4681b85f6842cff5ee89eaae92bddc667a Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Mon, 18 Apr 2016 16:09:53 -0700 Subject: [PATCH 070/292] Added support for readiness_check_results in the deployments model --- itests/steps/marathon_steps.py | 2 +- marathon/models/deployment.py | 22 +++++++++++++++------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/itests/steps/marathon_steps.py b/itests/steps/marathon_steps.py index 8f4b2ef..0678df2 100644 --- a/itests/steps/marathon_steps.py +++ b/itests/steps/marathon_steps.py @@ -158,4 +158,4 @@ def stop_listening_stream(context): @then('we should be able to see a deployment') def see_a_deployment(context): - assert len(context.client.list_deployments() == 1) + assert len(context.client.list_deployments()) == 1 diff --git a/marathon/models/deployment.py b/marathon/models/deployment.py index 29e24b6..3ca4a5f 100644 --- a/marathon/models/deployment.py +++ b/marathon/models/deployment.py @@ -6,6 +6,7 @@ class MarathonDeployment(MarathonResource): """Marathon Application resource. See: https://mesosphere.github.io/marathon/docs/rest-api.html#deployments + https://mesosphere.github.io/marathon/docs/generated/api.html#v2_deployments_get :param list[str] affected_apps: list of affected app ids :param current_actions: current actions @@ -29,14 +30,19 @@ def __init__( ] self.current_step = current_step self.id = id - self.steps = [ - [step if isinstance(step, MarathonDeploymentAction) else MarathonDeploymentAction().from_json(step) - for step in s] - for s in (steps or []) - ] + self.steps = [self.parse_deployment_step(step) for step in (steps or [])] self.total_steps = total_steps self.version = version + def parse_deployment_step(self, step): + if step.__class__ == dict: + # This is what Marathon 1.0.0 returns: steps + return MarathonDeploymentStep().from_json(step) + elif step.__class__ == list: + # This is Marathon < 1.0.0 style, a list of actions + return [MarathonDeploymentAction().from_json(s) for s in step] + else: + return step class MarathonDeploymentAction(MarathonObject): @@ -47,13 +53,15 @@ class MarathonDeploymentAction(MarathonObject): :param str action: action :param str app: app id :param str apps: app id (see https://github.com/mesosphere/marathon/pull/802) + :param type readiness_check_results: Undocumented """ - def __init__(self, action=None, app=None, apps=None, type=None): + def __init__(self, action=None, app=None, apps=None, type=None, readiness_check_results=None): self.action = action self.app = app self.apps = apps self.type = type # TODO: Remove builtin shadow + self.readiness_check_results = readiness_check_results # TODO: The docs say this is called just "readinessChecks?" class MarathonDeploymentPlan(MarathonObject): @@ -70,7 +78,7 @@ def __init__(self, original=None, target=None, class MarathonDeploymentStep(MarathonObject): def __init__(self, actions=None): - self.actions = [MarathonDeploymentAction.from_json(x) for x in actions] + self.actions = [MarathonDeploymentAction.from_json(x) for x in (actions or [])] class MarathonDeploymentOriginalState(MarathonObject): From ac8f489653d091bfe2e33e3e9a04323dd399914c Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Wed, 20 Apr 2016 11:50:16 -0700 Subject: [PATCH 071/292] Correctly test against new and old marathon deployment models --- marathon/models/__init__.py | 2 +- marathon/models/base.py | 2 +- marathon/models/deployment.py | 4 +- tests/test_api.py | 94 ++++++++++++++++++++++++++++++++++- tox.ini | 2 +- 5 files changed, 97 insertions(+), 7 deletions(-) diff --git a/marathon/models/__init__.py b/marathon/models/__init__.py index 00a0286..1591d3f 100644 --- a/marathon/models/__init__.py +++ b/marathon/models/__init__.py @@ -1,7 +1,7 @@ from .app import MarathonApp, MarathonHealthCheck from .base import MarathonResource, MarathonObject from .constraint import MarathonConstraint -from .deployment import MarathonDeployment, MarathonDeploymentAction +from .deployment import MarathonDeployment, MarathonDeploymentAction, MarathonDeploymentStep from .endpoint import MarathonEndpoint from .group import MarathonGroup from .info import MarathonInfo, MarathonConfig, MarathonZooKeeperConfig diff --git a/marathon/models/base.py b/marathon/models/base.py index 2ddc798..719b35d 100644 --- a/marathon/models/base.py +++ b/marathon/models/base.py @@ -9,7 +9,7 @@ class MarathonObject(object): """Base Marathon object.""" def __repr__(self): - return "{clazz}::{obj}".format(clazz=self.__class__.__name__, obj=self.to_json()) + return "{clazz}::{obj}".format(clazz=self.__class__.__name__, obj=self.to_json(minimal=False)) def __eq__(self, other): return self.__dict__ == other.__dict__ diff --git a/marathon/models/deployment.py b/marathon/models/deployment.py index 3ca4a5f..56945a9 100644 --- a/marathon/models/deployment.py +++ b/marathon/models/deployment.py @@ -40,7 +40,7 @@ def parse_deployment_step(self, step): return MarathonDeploymentStep().from_json(step) elif step.__class__ == list: # This is Marathon < 1.0.0 style, a list of actions - return [MarathonDeploymentAction().from_json(s) for s in step] + return [s if isinstance(s, MarathonDeploymentAction) else MarathonDeploymentAction().from_json(s) for s in step] else: return step @@ -78,7 +78,7 @@ def __init__(self, original=None, target=None, class MarathonDeploymentStep(MarathonObject): def __init__(self, actions=None): - self.actions = [MarathonDeploymentAction.from_json(x) for x in (actions or [])] + self.actions = [a if isinstance(a, MarathonDeploymentAction) else MarathonDeploymentAction.from_json(a) for a in (actions or [])] class MarathonDeploymentOriginalState(MarathonObject): diff --git a/tests/test_api.py b/tests/test_api.py index fe58861..22af1d6 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -4,8 +4,32 @@ @requests_mock.mock() -def test_get_deployments(m): - fake_response = '[ { "affectedApps": [ "/test" ], "id": "fakeid", "steps": [ [ { "action": "ScaleApplication", "app": "/test" } ] ], "currentActions": [ { "action": "ScaleApplication", "app": "/test" } ], "version": "fakeversion", "currentStep": 1, "totalSteps": 1 } ]' +def test_get_deployments_pre_1_0(m): + fake_response = """[ + { + "affectedApps": [ + "/test" + ], + "id": "fakeid", + "steps": [ + [ + { + "action": "ScaleApplication", + "app": "/test" + } + ] + ], + "currentActions": [ + { + "action": "ScaleApplication", + "app": "/test" + } + ], + "version": "fakeversion", + "currentStep": 1, + "totalSteps": 1 + } + ]""" m.get('http://fake_server/v2/deployments', text=fake_response) mock_client = MarathonClient(servers='http://fake_server') actual_deployments = mock_client.list_deployments() @@ -24,6 +48,72 @@ def test_get_deployments(m): assert expected_deployments == actual_deployments +@requests_mock.mock() +def test_get_deployments_post_1_0(m): + fake_response = """[ + { + "id": "4d2ff4d8-fbe5-4239-a886-f0831ed68d20", + "version": "2016-04-20T18:00:20.084Z", + "affectedApps": [ + "/test-trivial-app" + ], + "steps": [ + { + "actions": [ + { + "type": "StartApplication", + "app": "/test-trivial-app" + } + ] + }, + { + "actions": [ + { + "type": "ScaleApplication", + "app": "/test-trivial-app" + } + ] + } + ], + "currentActions": [ + { + "action": "ScaleApplication", + "app": "/test-trivial-app", + "readinessCheckResults": [] + } + ], + "currentStep": 2, + "totalSteps": 2 + } + ]""" + m.get('http://fake_server/v2/deployments', text=fake_response) + mock_client = MarathonClient(servers='http://fake_server') + actual_deployments = mock_client.list_deployments() + expected_deployments = [models.MarathonDeployment( + id=u"4d2ff4d8-fbe5-4239-a886-f0831ed68d20", + steps=[ + models.MarathonDeploymentStep( + actions=[models.MarathonDeploymentAction( + type="StartApplication", app="/test-trivial-app")], + ), + models.MarathonDeploymentStep( + actions=[models.MarathonDeploymentAction( + type="ScaleApplication", app="/test-trivial-app")], + ), + ], + current_actions=[models.MarathonDeploymentAction( + action="ScaleApplication", app="/test-trivial-app", readiness_check_results=[]) + ], + current_step=2, + total_steps=2, + affected_apps=[u"/test-trivial-app"], + version=u"2016-04-20T18:00:20.084Z" + )] + # Helpful for tox to see the diff + assert expected_deployments[0].__dict__ == actual_deployments[0].__dict__ + assert expected_deployments == actual_deployments + + @requests_mock.mock() def test_list_tasks_with_app_id(m): fake_response = '{ "tasks": [ { "appId": "/anapp", "healthCheckResults": [ { "alive": true, "consecutiveFailures": 0, "firstSuccess": "2014-10-03T22:57:02.246Z", "lastFailure": null, "lastSuccess": "2014-10-03T22:57:41.643Z", "taskId": "bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799" } ], "host": "10.141.141.10", "id": "bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799", "ports": [ 31000 ], "servicePorts": [ 9000 ], "stagedAt": "2014-10-03T22:16:27.811Z", "startedAt": "2014-10-03T22:57:41.587Z", "version": "2014-10-03T22:16:23.634Z" }, { "appId": "/anotherapp", "healthCheckResults": [ { "alive": true, "consecutiveFailures": 0, "firstSuccess": "2014-10-03T22:57:02.246Z", "lastFailure": null, "lastSuccess": "2014-10-03T22:57:41.649Z", "taskId": "bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799" } ], "host": "10.141.141.10", "id": "bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799", "ports": [ 31001 ], "servicePorts": [ 9000 ], "stagedAt": "2014-10-03T22:16:33.814Z", "startedAt": "2014-10-03T22:57:41.593Z", "version": "2014-10-03T22:16:23.634Z" } ] }' diff --git a/tox.ini b/tox.ini index 005f07a..267f23b 100644 --- a/tox.ini +++ b/tox.ini @@ -39,7 +39,7 @@ deps = pytest mock commands = - py.test -s {posargs:tests} + py.test -s -vv {posargs:tests} [testenv:pep8] deps = flake8 From 8c6f02f84d204cf3538f3ae4dff802e03a14f29e Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Thu, 21 Apr 2016 12:35:53 -0700 Subject: [PATCH 072/292] Added force option for kill_given_tasks --- marathon/client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/marathon/client.py b/marathon/client.py index dfd6734..96071cd 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -412,16 +412,19 @@ def list_tasks(self, app_id=None, **kwargs): return tasks - def kill_given_tasks(self, task_ids, scale=False): + def kill_given_tasks(self, task_ids, scale=False, force=None): """Kill a list of given tasks. :param list[str] task_ids: tasks to kill :param bool scale: if true, scale down the app by the number of tasks killed + :param bool force: if true, ignore any current running deployments :return: True on success :rtype: bool """ params = {'scale': scale} + if force is not None: + params['force'] = force data = json.dumps({"ids": task_ids}) response = self._do_request( 'POST', '/v2/tasks/delete', params=params, data=data) From 7e7e1657f9e1435bbcfd540ef896b7446659e689 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Thu, 21 Apr 2016 12:50:12 -0700 Subject: [PATCH 073/292] Release 0.8.1 --- README.md | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9397d88..5b199b6 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ This is a Python library for interfacing with [Marathon](https://github.com/meso #### Compatibility -* For Marathon 1.1.1 and 0.15.x, use at least 0.8.0 +* For Marathon 1.1.1 and 0.15.x, use at least 0.8.1 * For Marathon 0.14.x, use at least 0.7.6 * For Marathon 0.8.x-0.11.x, use at least marathon-python 0.7.5 * For Marathon 0.8.x-0.9.x, use as least marathon-python 0.6.11 - 0.7.4 diff --git a/setup.py b/setup.py index 7936ee4..f94be7e 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='marathon', - version='0.8.0', + version='0.8.1', description='Marathon Client Library', long_description="""Python interface to the Mesos Marathon REST API.""", author='Mike Babineau', From 15f58459ebbe00b1a329f4eb480a2c7959266ced Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Thu, 21 Apr 2016 12:56:33 -0700 Subject: [PATCH 074/292] Updated changelog for 0.8.1 --- CHANGELOG.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 489ae38..1a06fd8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Change Log +## [0.8.1](https://github.com/thefactory/marathon-python/tree/0.8.1) (2016-04-21) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.0...0.8.1) + +**Closed issues:** + +- Does not understand readiness\_check\_results [\#94](https://github.com/thefactory/marathon-python/issues/94) +- Generate marathon-python from raml [\#86](https://github.com/thefactory/marathon-python/issues/86) +- Support Marathon 0.15.1 [\#84](https://github.com/thefactory/marathon-python/issues/84) + +**Merged pull requests:** + +- Added force option for kill\_given\_tasks [\#96](https://github.com/thefactory/marathon-python/pull/96) ([solarkennedy](https://github.com/solarkennedy)) +- Support the deployments endpoint correctly in marathon 1.1.1 [\#95](https://github.com/thefactory/marathon-python/pull/95) ([solarkennedy](https://github.com/solarkennedy)) + ## [0.8.0](https://github.com/thefactory/marathon-python/tree/0.8.0) (2016-04-18) [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.7.7...0.8.0) From 49767b92cbe3578bd2f5e5ecd373cd09f1c4e5dc Mon Sep 17 00:00:00 2001 From: Robert Johnson Date: Wed, 11 May 2016 07:21:26 -0700 Subject: [PATCH 075/292] add name attribute to port mapping --- itests/steps/marathon_steps.py | 1 + marathon/models/container.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/itests/steps/marathon_steps.py b/itests/steps/marathon_steps.py index 0678df2..6c38bfc 100644 --- a/itests/steps/marathon_steps.py +++ b/itests/steps/marathon_steps.py @@ -48,6 +48,7 @@ def create_complex_new_app_with_unicode(context): 'docker': { 'portMappings': [{'protocol': 'tcp', + 'name': 'myport', 'containerPort': 8888, 'hostPort': 0}], 'image': u'localhost/fake_docker_url', diff --git a/marathon/models/container.py b/marathon/models/container.py index 2224200..efa586a 100644 --- a/marathon/models/container.py +++ b/marathon/models/container.py @@ -74,6 +74,7 @@ class MarathonContainerPortMapping(MarathonObject): See https://mesosphere.github.io/marathon/docs/native-docker.html + :param str name: :param int container_port: :param int host_port: :param str protocol: @@ -83,7 +84,8 @@ class MarathonContainerPortMapping(MarathonObject): PROTOCOLS = ['tcp', 'udp'] """Valid protocols""" - def __init__(self, container_port=None, host_port=0, service_port=None, protocol='tcp', labels=None): + def __init__(self, name=None, container_port=None, host_port=0, service_port=None, protocol='tcp', labels=None): + self.name = name self.container_port = container_port self.host_port = host_port self.service_port = service_port From a69c37e313c7d2ec232d5683fdb8495a825bf918 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Fri, 13 May 2016 10:07:10 -0700 Subject: [PATCH 076/292] Updated readme with a correct invocation of scale_app. Fixes #102 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5b199b6..3470cf7 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ MarathonApp::myapp3 ```python >>> c.get_app('myapp3').instances 1 ->>> c.scale_app('myapp3', 2) +>>> c.scale_app('myapp3', instances=3) {'deploymentId': '611b89e3-99f2-4d8a-afe1-ec0b83fdbb88', 'version': '2014-08-26T07:40:20.121Z'} >>> c.get_app('myapp3').instances 3 From 0d980503fff88eff6a140c314b6de863be917f8f Mon Sep 17 00:00:00 2001 From: oilbeater Date: Wed, 25 May 2016 19:38:48 +0800 Subject: [PATCH 077/292] Change to_json() datatime format As the time format that MarathonApp.to_json() and MarathonApp.from_json() used is different, so the json dumped by to_json() can not be loaded by frmo_json() This commit will use unify time format for both method. --- marathon/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/marathon/util.py b/marathon/util.py index 7363913..917c0b8 100644 --- a/marathon/util.py +++ b/marathon/util.py @@ -23,7 +23,7 @@ def default(self, obj): return self.default(obj.json_repr()) if isinstance(obj, datetime.datetime): - return obj.isoformat() + return obj.strftime('%Y-%m-%dT%H:%M:%S.%fZ') if isinstance(obj, collections.Iterable) and not is_stringy(obj): try: @@ -43,7 +43,7 @@ def default(self, obj): return self.default(obj.json_repr(minimal=True)) if isinstance(obj, datetime.datetime): - return obj.isoformat() + return obj.strftime('%Y-%m-%dT%H:%M:%S.%fZ') if isinstance(obj, collections.Iterable) and not is_stringy(obj): try: From 9fc85c0a7fd044ddf444bedb0fd62586fe14e0c2 Mon Sep 17 00:00:00 2001 From: oilbeater Date: Fri, 27 May 2016 23:04:02 +0800 Subject: [PATCH 078/292] task_stats bugfix and query improvements. Fix task_stats bugs in MarathonApp init that wrongly use version_info. Add more query parameters to list_apps and get_app to get more info. Add readiness_check_results to MarathonApp when query embed with app.readiness marathon will return readinessCheckResults field. --- marathon/client.py | 57 +++++++++++++++++++++++++++++++++--------- marathon/models/app.py | 6 +++-- 2 files changed, 49 insertions(+), 14 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index 96071cd..33dcef3 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -135,14 +135,19 @@ def create_app(self, app_id, app): else: return False - def list_apps(self, cmd=None, embed_tasks=False, - embed_failures=False, embed_task_stats=False, **kwargs): + def list_apps(self, cmd=None, embed_tasks=False, embed_counts=False, + embed_deployments=False, embed_readiness=False, + embed_last_task_failure=False, embed_failures=False, + embed_task_stats=False, **kwargs): """List all apps. - :param str app_id: application ID :param str cmd: if passed, only show apps with a matching `cmd` :param bool embed_tasks: embed tasks in result - :param bool embed_failures: embed tasks and last task failure in result + :param bool embed_counts: embed all task counts + :param bool embed_deployments: embed all deployment identifier + :param bool embed_readiness: embed all readiness check results + :param bool embed_last_task_failure: embeds the last task failure + :param bool embed_failures: shorthand for embed_last_task_failure :param bool embed_task_stats: embed task stats in result :param kwargs: arbitrary search filters @@ -153,12 +158,18 @@ def list_apps(self, cmd=None, embed_tasks=False, if cmd: params['cmd'] = cmd - if embed_failures: - params['embed'] = 'apps.failures' - elif embed_tasks: - params['embed'] = 'apps.tasks' - elif embed_task_stats: - params['embed'] = 'apps.taskStats' + embed_params = { + 'app.tasks': embed_tasks, + 'app.counts': embed_counts, + 'app.deployments': embed_deployments, + 'app.readiness': embed_readiness, + 'app.lastTaskFailure': embed_last_task_failure, + 'app.failures': embed_failures, + 'app.taskStats': embed_task_stats + } + filtered_embed_params = [k for (k, v) in embed_params.items() if v] + if filtered_embed_params: + params['embed'] = filtered_embed_params response = self._do_request('GET', '/v2/apps', params=params) apps = self._parse_response( @@ -167,16 +178,38 @@ def list_apps(self, cmd=None, embed_tasks=False, apps = [o for o in apps if getattr(o, k) == v] return apps - def get_app(self, app_id, embed_tasks=False): + def get_app(self, app_id, embed_tasks=False, embed_counts=False, + embed_deployments=False, embed_readiness=False, + embed_last_task_failure=False, embed_failures=False, + embed_task_stats=False): """Get a single app. :param str app_id: application ID :param bool embed_tasks: embed tasks in result + :param bool embed_counts: embed all task counts + :param bool embed_deployments: embed all deployment identifier + :param bool embed_readiness: embed all readiness check results + :param bool embed_last_task_failure: embeds the last task failure + :param bool embed_failures: shorthand for embed_last_task_failure + :param bool embed_task_stats: embed task stats in result :returns: application :rtype: :class:`marathon.models.app.MarathonApp` """ - params = {'embed': 'apps.tasks'} if embed_tasks else {} + params = {} + embed_params = { + 'app.tasks': embed_tasks, + 'app.counts': embed_counts, + 'app.deployments': embed_deployments, + 'app.readiness': embed_readiness, + 'app.lastTaskFailure': embed_last_task_failure, + 'app.failures': embed_failures, + 'app.taskStats': embed_task_stats + } + filtered_embed_params = [k for (k, v) in embed_params.items() if v] + if filtered_embed_params: + params['embed'] = filtered_embed_params + response = self._do_request( 'GET', '/v2/apps/{app_id}'.format(app_id=app_id), params=params) return self._parse_response(response, MarathonApp, resource_name='app') diff --git a/marathon/models/app.py b/marathon/models/app.py index d26b3f9..0b6b5a4 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -84,7 +84,8 @@ def __init__( max_launch_delay_seconds=None, mem=None, ports=None, require_ports=None, store_urls=None, task_rate_limit=None, tasks=None, tasks_running=None, tasks_staged=None, tasks_healthy=None, tasks_unhealthy=None, upgrade_strategy=None, uris=None, user=None, version=None, version_info=None, - ip_address=None, fetch=None, task_stats=None, readiness_checks=None, port_definitions=None, residency=None): + ip_address=None, fetch=None, task_stats=None, readiness_checks=None, + readiness_check_results=None, port_definitions=None, residency=None): # self.args = args or [] self.accepted_resource_roles = accepted_resource_roles @@ -127,6 +128,7 @@ def __init__( self.ports = ports or [] self.port_definitions = port_definitions or [] self.readiness_checks = readiness_checks or [] + self.readiness_check_results = readiness_check_results or [] self.residency = residency self.require_ports = require_ports self.store_urls = store_urls or [] @@ -146,7 +148,7 @@ def __init__( self.version = version self.version_info = version_info if (isinstance(version_info, MarathonAppVersionInfo) or version_info is None) \ else MarathonAppVersionInfo.from_json(version_info) - self.task_stats = version_info if (isinstance(task_stats, MarathonTaskStats) or task_stats is None) \ + self.task_stats = task_stats if (isinstance(task_stats, MarathonTaskStats) or task_stats is None) \ else MarathonTaskStats.from_json(task_stats) From da4f55d91cfef8d039a4c98136d237ee50ce225f Mon Sep 17 00:00:00 2001 From: Greg Hill Date: Tue, 14 Jun 2016 08:36:18 -0500 Subject: [PATCH 079/292] Add 'wipe' option to kill_task methods In order to clean up persistent volumes associated with containers, we need to pass the 'wipe' param to Marathon. See: https://mesosphere.github.io/marathon/docs/rest-api.html#delete-v2-apps-appid-tasks --- marathon/client.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index 33dcef3..67b6741 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -463,7 +463,7 @@ def kill_given_tasks(self, task_ids, scale=False, force=None): 'POST', '/v2/tasks/delete', params=params, data=data) return response == 200 - def kill_tasks(self, app_id, scale=False, + def kill_tasks(self, app_id, scale=False, wipe=False, host=None, batch_size=0, batch_delay=0): """Kill all tasks belonging to app. @@ -484,7 +484,7 @@ def batch(iterable, size): if batch_size == 0: # Terminate all at once - params = {'scale': scale} + params = {'scale': scale, 'wipe': wipe} if host: params['host'] = host response = self._do_request( @@ -501,7 +501,7 @@ def batch(iterable, size): tasks = self.list_tasks( app_id, host=host) if host else self.list_tasks(app_id) for tbatch in batch(tasks, batch_size): - killed_tasks = [self.kill_task(app_id, t.id, scale=scale) + killed_tasks = [self.kill_task(app_id, t.id, scale=scale, wipe=wipe) for t in tbatch] # Pause until the tasks have been killed to avoid race @@ -526,7 +526,7 @@ def batch(iterable, size): return tasks - def kill_task(self, app_id, task_id, scale=False): + def kill_task(self, app_id, task_id, scale=False, wipe=False): """Kill a task. :param str app_id: application ID @@ -536,7 +536,7 @@ def kill_task(self, app_id, task_id, scale=False): :returns: the killed task :rtype: :class:`marathon.models.task.MarathonTask` """ - params = {'scale': scale} + params = {'scale': scale, 'wipe': wipe} response = self._do_request('DELETE', '/v2/apps/{app_id}/tasks/{task_id}' .format(app_id=app_id, task_id=task_id), params) # Marathon is inconsistent about what type of object it returns on the multi From 8005e1acd76a3b8d89d067ab056568c7a56ba00d Mon Sep 17 00:00:00 2001 From: Greg Hill Date: Tue, 14 Jun 2016 08:45:27 -0500 Subject: [PATCH 080/292] Make exception handling work with py3k Fixes #106 --- marathon/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marathon/client.py b/marathon/client.py index 33dcef3..a56ec45 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -65,7 +65,7 @@ def _do_request(self, method, path, params=None, data=None): marathon.log.info('Got response from %s', server) except requests.exceptions.RequestException as e: marathon.log.error( - 'Error while calling %s: %s', url, e.message) + 'Error while calling %s: %s', url, str(e)) if response is None: raise MarathonError('No remaining Marathon servers to try') From 45bf463ff3ec844cb41fe23c7b1843cf35117c8e Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Tue, 14 Jun 2016 09:24:22 -0700 Subject: [PATCH 081/292] Bump to 0.8.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f94be7e..030fb8a 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='marathon', - version='0.8.1', + version='0.8.2', description='Marathon Client Library', long_description="""Python interface to the Mesos Marathon REST API.""", author='Mike Babineau', From f393e1ee5360faaeda455a4e820b490121d3cfab Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Tue, 14 Jun 2016 12:13:14 -0700 Subject: [PATCH 082/292] Added changelog for 0.8.2 --- CHANGELOG.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a06fd8..ffc77c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Change Log +## [0.8.2](https://github.com/thefactory/marathon-python/tree/0.8.2) (2016-06-14) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.1...0.8.2) + +**Closed issues:** + +- AttributeError on connection issues [\#106](https://github.com/thefactory/marathon-python/issues/106) +- Wrongly use version\_info as task\_stats [\#104](https://github.com/thefactory/marathon-python/issues/104) +- c.scale\_app\('myapp3', 2\) [\#102](https://github.com/thefactory/marathon-python/issues/102) +- AttributeError: module 'marathon' has no attribute 'MarathonClient' [\#100](https://github.com/thefactory/marathon-python/issues/100) +- when run list\_apps ,has errors [\#99](https://github.com/thefactory/marathon-python/issues/99) + +**Merged pull requests:** + +- Make exception handling work with py3k [\#108](https://github.com/thefactory/marathon-python/pull/108) ([jimbobhickville](https://github.com/jimbobhickville)) +- Add 'wipe' option to kill\_task methods [\#107](https://github.com/thefactory/marathon-python/pull/107) ([jimbobhickville](https://github.com/jimbobhickville)) +- task\_stats bugfix and query improvements. [\#105](https://github.com/thefactory/marathon-python/pull/105) ([oilbeater](https://github.com/oilbeater)) +- Change to\_json\(\) datatime format [\#103](https://github.com/thefactory/marathon-python/pull/103) ([oilbeater](https://github.com/oilbeater)) +- add name attribute to port mapping [\#101](https://github.com/thefactory/marathon-python/pull/101) ([Rob-Johnson](https://github.com/Rob-Johnson)) + ## [0.8.1](https://github.com/thefactory/marathon-python/tree/0.8.1) (2016-04-21) [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.0...0.8.1) From 390cf02fc6a4a3ac8300b1a081ee1163d78d274b Mon Sep 17 00:00:00 2001 From: Dmitry Fedorov Date: Wed, 15 Jun 2016 16:13:27 +0300 Subject: [PATCH 083/292] Issue110: MarathonTask.ip_addresses attribute is set properly --- marathon/models/task.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/marathon/models/task.py b/marathon/models/task.py index dbaa4d4..6614bdc 100644 --- a/marathon/models/task.py +++ b/marathon/models/task.py @@ -44,8 +44,19 @@ def __init__( self.started_at = started_at if (started_at is None or isinstance(started_at, datetime)) \ else datetime.strptime(started_at, self.DATETIME_FORMAT) self.version = version + self.ip_addresses = [ + ipaddr if isinstance( + ip_addresses, MarathonIpAddress) else MarathonIpAddress().from_json(ipaddr) + for ipaddr in (ip_addresses or [])] +class MarathonIpAddress(MarathonObject): + """ + """ + def __init__(self, ip_address=None, protocol=None): + self.ip_address = ip_address + self.protocol = protocol + class MarathonHealthCheckResult(MarathonObject): """Marathon health check result. From 3f7ef2935695c49659b023e6000f2cf4373ad6e6 Mon Sep 17 00:00:00 2001 From: oilbeater Date: Wed, 15 Jun 2016 00:14:57 +0800 Subject: [PATCH 084/292] Add message field for MarathonStatusUpdateEvent. --- marathon/models/events.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/marathon/models/events.py b/marathon/models/events.py index 1c7a0f9..14e7b38 100644 --- a/marathon/models/events.py +++ b/marathon/models/events.py @@ -8,8 +8,6 @@ from marathon.models.deployment import MarathonDeploymentPlan from marathon.exceptions import MarathonError -import marathon - class MarathonEvent(MarathonObject): @@ -28,13 +26,11 @@ def __init__(self, event_type, timestamp, **kwargs): self.event_type = event_type # All events have these two attributes self.timestamp = timestamp for attribute in self.KNOWN_ATTRIBUTES: - try: - self._set(attribute, kwargs[attribute]) - except KeyError: - marathon.log.warn( - 'Unknown event attribute processing event {}: {}'.format(event_type, attribute)) + self._set(attribute, kwargs.get(attribute)) def _set(self, attribute_name, attribute): + if not attribute: + return if attribute_name in self.attribute_name_to_marathon_object: clazz = self.attribute_name_to_marathon_object[attribute_name] attribute = clazz.from_json( @@ -48,7 +44,7 @@ class MarathonApiPostEvent(MarathonEvent): class MarathonStatusUpdateEvent(MarathonEvent): KNOWN_ATTRIBUTES = [ - 'slave_id', 'task_id', 'task_status', 'app_id', 'host', 'ports', 'version'] + 'slave_id', 'task_id', 'task_status', 'app_id', 'host', 'ports', 'version', 'message'] class MarathonFrameworkMessageEvent(MarathonEvent): @@ -106,12 +102,15 @@ class MarathonDeploymentStepSuccess(MarathonEvent): class MarathonDeploymentStepFailure(MarathonEvent): KNOWN_ATTRIBUTES = ['plan'] + class MarathonEventStreamAttached(MarathonEvent): KNOWN_ATTRIBUTES = ['remote_address'] + class MarathonEventStreamDetached(MarathonEvent): KNOWN_ATTRIBUTES = ['remote_address'] + class EventFactory: """ From c29e210e0d990174e89ec4de611aa75ec03274f3 Mon Sep 17 00:00:00 2001 From: Greg Hill Date: Wed, 15 Jun 2016 14:56:38 -0500 Subject: [PATCH 085/292] Add 'persistent' volume option This is required for supporting persistent local volumes: https://mesosphere.github.io/marathon/docs/persistent-volumes.html --- marathon/models/container.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/marathon/models/container.py b/marathon/models/container.py index efa586a..b11cf98 100644 --- a/marathon/models/container.py +++ b/marathon/models/container.py @@ -104,13 +104,15 @@ class MarathonContainerVolume(MarathonObject): :param str container_path: container path :param str host_path: host path :param str mode: one of ['RO', 'RW'] + :param object persistent: persistent volume options, should be of the form {'size': 1000} """ MODES = ['RO', 'RW'] - def __init__(self, container_path=None, host_path=None, mode='RW'): + def __init__(self, container_path=None, host_path=None, mode='RW', persistent=None): self.container_path = container_path self.host_path = host_path if mode not in self.MODES: raise InvalidChoiceError('mode', mode, self.MODES) self.mode = mode + self.persistent = persistent From 5417de968bab3509b1db70f973d4b1720078b032 Mon Sep 17 00:00:00 2001 From: Charles Rice Date: Fri, 17 Jun 2016 13:55:17 +0100 Subject: [PATCH 086/292] add in state to the task model --- marathon/models/task.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/marathon/models/task.py b/marathon/models/task.py index 6614bdc..c9dcf3c 100644 --- a/marathon/models/task.py +++ b/marathon/models/task.py @@ -14,6 +14,7 @@ class MarathonTask(MarathonResource): :param str id: task id :param list[int] ports: allocated ports :param list[int] service_ports: ports exposed for load balancing + :param str state: State of the task e.g. TASK_RUNNING :param str slave_id: Mesos slave id :param staged_at: when this task was staged :type staged_at: datetime or str @@ -26,7 +27,7 @@ class MarathonTask(MarathonResource): def __init__( self, app_id=None, health_check_results=None, host=None, id=None, ports=None, service_ports=None, - slave_id=None, staged_at=None, started_at=None, version=None, ip_addresses=[]): + slave_id=None, staged_at=None, started_at=None, version=None, ip_addresses=[], state=None): self.app_id = app_id self.health_check_results = health_check_results or [] self.health_check_results = [ @@ -43,6 +44,7 @@ def __init__( else datetime.strptime(staged_at, self.DATETIME_FORMAT) self.started_at = started_at if (started_at is None or isinstance(started_at, datetime)) \ else datetime.strptime(started_at, self.DATETIME_FORMAT) + self.state = state self.version = version self.ip_addresses = [ ipaddr if isinstance( From 9c5919492498cca8f2369a979a8d43b4267b1da1 Mon Sep 17 00:00:00 2001 From: oilbeater Date: Fri, 17 Jun 2016 23:25:01 +0800 Subject: [PATCH 087/292] Set fetch in MarathonApp. --- marathon/models/app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/marathon/models/app.py b/marathon/models/app.py index 0b6b5a4..3ee18ac 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -144,6 +144,7 @@ def __init__( self.upgrade_strategy = upgrade_strategy if (isinstance(upgrade_strategy, MarathonUpgradeStrategy) or upgrade_strategy is None) \ else MarathonUpgradeStrategy.from_json(upgrade_strategy) self.uris = uris or [] + self.fetch = fetch or [] self.user = user self.version = version self.version_info = version_info if (isinstance(version_info, MarathonAppVersionInfo) or version_info is None) \ From a219e22ea8de4997f65475a9c4a87fbb409ea5db Mon Sep 17 00:00:00 2001 From: Usman Masood Date: Fri, 17 Jun 2016 09:14:09 -0700 Subject: [PATCH 088/292] Don't die if no JSON found in response --- marathon/exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marathon/exceptions.py b/marathon/exceptions.py index 5ce7b26..439d972 100644 --- a/marathon/exceptions.py +++ b/marathon/exceptions.py @@ -10,7 +10,7 @@ def __init__(self, response): """ content = response.json() self.status_code = response.status_code - self.error_message = content['message'] + self.error_message = content.get('message') super(MarathonHttpError, self).__init__(self.__str__()) def __repr__(self): From f5be4548027548aefc9d9225db0b1dc3e9ac7841 Mon Sep 17 00:00:00 2001 From: oilbeater Date: Sat, 18 Jun 2016 10:02:21 +0800 Subject: [PATCH 089/292] Fix pep8 issues and strict flake8 check. --- itests/steps/marathon_steps.py | 4 ++++ marathon/client.py | 9 ++++----- marathon/models/app.py | 24 +++++++++++++++--------- marathon/models/container.py | 3 +-- marathon/models/deployment.py | 4 ++-- marathon/models/events.py | 3 +++ marathon/models/info.py | 9 +++------ marathon/models/task.py | 7 +++---- tests/test_api.py | 23 +++++++++++++++++++++-- tox.ini | 1 - 10 files changed, 56 insertions(+), 31 deletions(-) diff --git a/itests/steps/marathon_steps.py b/itests/steps/marathon_steps.py index 6c38bfc..9c460a2 100644 --- a/itests/steps/marathon_steps.py +++ b/itests/steps/marathon_steps.py @@ -122,12 +122,14 @@ def listen_for_events(client, events): for msg in client.event_stream(): events.append(msg) + @when(u'marathon version is greater than {version}') def marathon_version_chech(context, version): info = context.client.get_info() if StrictVersion(info.version) < StrictVersion(version): context.scenario.skip(reason='Marathon version is too low for this scenario') + @when(u'we start listening for events') def start_listening_stream(context): manager = multiprocessing.Manager() @@ -138,6 +140,7 @@ def start_listening_stream(context): p.start() context.p = p + @then(u'we should see list of events') def stop_listening_stream(context): time.sleep(10) @@ -157,6 +160,7 @@ def stop_listening_stream(context): filtered_events = [e for e in context.events if e.event_type == "deployment_success"] assert len(filtered_events) == 2 + @then('we should be able to see a deployment') def see_a_deployment(context): assert len(context.client.list_deployments()) == 1 diff --git a/marathon/client.py b/marathon/client.py index b79a163..37b1497 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -59,8 +59,7 @@ def _do_request(self, method, path, params=None, data=None): server = servers.pop(0) url = ''.join([server.rstrip('/'), path]) try: - response = requests.request( - method, url, params=params, data=data, headers=headers, + response = requests.request(method, url, params=params, data=data, headers=headers, auth=self.auth, timeout=self.timeout) marathon.log.info('Got response from %s', server) except requests.exceptions.RequestException as e: @@ -389,9 +388,9 @@ def rollback_group(self, group_id, version, force=False): """ params = {'force': force} response = self._do_request( - 'PUT', '/v2/groups/{group_id}/versions/{version}'.format(group_id=group_id, - version=version), - params=params) + 'PUT', + '/v2/groups/{group_id}/versions/{version}'.format(group_id=group_id, version=version), + params=params) return response.json() def delete_group(self, group_id, force=False): diff --git a/marathon/models/app.py b/marathon/models/app.py index 3ee18ac..8e81055 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -77,8 +77,7 @@ class MarathonApp(MarathonResource): 'deployments', 'tasks', 'tasks_running', 'tasks_staged', 'tasks_healthy', 'tasks_unhealthy'] """List of read-only attributes""" - def __init__( - self, accepted_resource_roles=None, args=None, backoff_factor=None, backoff_seconds=None, cmd=None, + def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, backoff_seconds=None, cmd=None, constraints=None, container=None, cpus=None, dependencies=None, deployments=None, disk=None, env=None, executor=None, health_checks=None, id=None, instances=None, labels=None, last_task_failure=None, max_launch_delay_seconds=None, mem=None, ports=None, require_ports=None, store_urls=None, @@ -171,8 +170,7 @@ class MarathonHealthCheck(MarathonObject): :param dict kwargs: additional arguments for forward compatibility """ - def __init__( - self, command=None, grace_period_seconds=None, interval_seconds=None, max_consecutive_failures=None, + def __init__(self, command=None, grace_period_seconds=None, interval_seconds=None, max_consecutive_failures=None, path=None, port_index=None, protocol=None, timeout_seconds=None, ignore_http1xx=None, **kwargs): self.command = command self.grace_period_seconds = grace_period_seconds @@ -266,7 +264,8 @@ class MarathonTaskStats(MarathonObject): :type started_after_last_scaling: :class:`marathon.models.app.MarathonTaskStatsType` or dict :param with_latest_config: contains statistics about all tasks that run with the same config as the latest app version. :type with_latest_config: :class:`marathon.models.app.MarathonTaskStatsType` or dict - :param with_outdated_config: contains statistics about all tasks that were started before the last config change which was not simply a restart or scaling operation. + :param with_outdated_config: contains statistics about all tasks that were started before the last config change + which was not simply a restart or scaling operation. :type with_outdated_config: :class:`marathon.models.app.MarathonTaskStatsType` or dict :param total_summary: contains statistics about all tasks. :type total_summary: :class:`marathon.models.app.MarathonTaskStatsType` or dict @@ -352,17 +351,22 @@ def __init__(self, average_seconds=None, median_seconds=None): self.average_seconds = average_seconds self.median_seconds = median_seconds + class ReadinessCheck(MarathonObject): """Marathon readiness check: https://mesosphere.github.io/marathon/docs/readiness-checks.html :param string name (Optional. Default: "readinessCheck"): The name used to identify this readiness check. :param string protocol (Optional. Default: "HTTP"): Protocol of the requests to be performed. Either HTTP or HTTPS. - :param string path (Optional. Default: "/"): Path to the endpoint the task exposes to provide readiness status. Example: /path/to/readiness. - :param string port_name (Optional. Default: "http-api"): Name of the port to query as described in the portDefinitions. Example: http-api. + :param string path (Optional. Default: "/"): Path to the endpoint the task exposes to provide readiness status. + Example: /path/to/readiness. + :param string port_name (Optional. Default: "http-api"): Name of the port to query as described in the + portDefinitions. Example: http-api. :param int interval_seconds (Optional. Default: 30 seconds): Number of seconds to wait between readiness checks. - :param int timeout_seconds (Optional. Default: 10 seconds): Number of seconds after which a readiness check times out, regardless of the response. This value must be smaller than interval_seconds. + :param int timeout_seconds (Optional. Default: 10 seconds): Number of seconds after which a readiness check + times out, regardless of the response. This value must be smaller than interval_seconds. :param list http_status_codes_for_ready (Optional. Default: [200]): The HTTP/HTTPS status code to treat as ready. - :param bool preserve_last_response (Optional. Default: false): If true, the last readiness check response will be preserved and exposed in the API as part of a deployment. + :param bool preserve_last_response (Optional. Default: false): If true, the last readiness check response will be + preserved and exposed in the API as part of a deployment. """ @@ -376,6 +380,7 @@ def __init__(self, name=None, protocol=None, path=None, port_name=None, interval self.http_status_codes_for_ready = http_status_codes_for_ready self.preserve_last_response = preserve_last_response + class PortDefinition(MarathonObject): """Marathon port definitions: https://mesosphere.github.io/marathon/docs/ports.html @@ -391,6 +396,7 @@ def __init__(self, port=None, protocol=None, name=None, labels=None): self.name = name self.labels = labels + class Residency(MarathonObject): """Declares how "resident" an app is: https://mesosphere.github.io/marathon/docs/persistent-volumes.html diff --git a/marathon/models/container.py b/marathon/models/container.py index b11cf98..3ef4291 100644 --- a/marathon/models/container.py +++ b/marathon/models/container.py @@ -49,8 +49,7 @@ class MarathonDockerContainer(MarathonObject): NETWORK_MODES = ['BRIDGE', 'HOST'] """Valid network modes""" - def __init__( - self, image=None, network='HOST', port_mappings=None, parameters=None, privileged=None, + def __init__(self, image=None, network='HOST', port_mappings=None, parameters=None, privileged=None, force_pull_image=None, **kwargs): self.image = image if network: diff --git a/marathon/models/deployment.py b/marathon/models/deployment.py index 56945a9..b93bf40 100644 --- a/marathon/models/deployment.py +++ b/marathon/models/deployment.py @@ -19,8 +19,7 @@ class MarathonDeployment(MarathonResource): :param str version: version id """ - def __init__( - self, affected_apps=None, current_actions=None, current_step=None, id=None, steps=None, + def __init__(self, affected_apps=None, current_actions=None, current_step=None, id=None, steps=None, total_steps=None, version=None): self.affected_apps = affected_apps self.current_actions = [ @@ -44,6 +43,7 @@ def parse_deployment_step(self, step): else: return step + class MarathonDeploymentAction(MarathonObject): """Marathon Application resource. diff --git a/marathon/models/events.py b/marathon/models/events.py index 1c7a0f9..7c57376 100644 --- a/marathon/models/events.py +++ b/marathon/models/events.py @@ -106,12 +106,15 @@ class MarathonDeploymentStepSuccess(MarathonEvent): class MarathonDeploymentStepFailure(MarathonEvent): KNOWN_ATTRIBUTES = ['plan'] + class MarathonEventStreamAttached(MarathonEvent): KNOWN_ATTRIBUTES = ['remote_address'] + class MarathonEventStreamDetached(MarathonEvent): KNOWN_ATTRIBUTES = ['remote_address'] + class EventFactory: """ diff --git a/marathon/models/info.py b/marathon/models/info.py index d3144d6..872f8f2 100644 --- a/marathon/models/info.py +++ b/marathon/models/info.py @@ -23,8 +23,7 @@ class MarathonInfo(MarathonResource): :param bool elected: """ - def __init__( - self, event_subscriber=None, framework_id=None, http_config=None, leader=None, marathon_config=None, + def __init__(self, event_subscriber=None, framework_id=None, http_config=None, leader=None, marathon_config=None, name=None, version=None, elected=None, zookeeper_config=None): if isinstance(event_subscriber, MarathonEventSubscriber): self.event_subscriber = event_subscriber @@ -75,8 +74,7 @@ class MarathonConfig(MarathonObject): :param int marathon_store_timeout: """ - def __init__( - self, checkpoint=None, executor=None, failover_timeout=None, framework_name=None, ha=None, + def __init__(self, checkpoint=None, executor=None, failover_timeout=None, framework_name=None, ha=None, hostname=None, leader_proxy_connection_timeout_ms=None, leader_proxy_read_timeout_ms=None, local_port_min=None, local_port_max=None, master=None, mesos_leader_ui_url=None, mesos_role=None, mesos_user=None, webui_url=None, reconciliation_initial_delay=None, reconciliation_interval=None, @@ -117,8 +115,7 @@ class MarathonZooKeeperConfig(MarathonObject): :param int zk_timeout: """ - def __init__( - self, zk=None, zk_future_timeout=None, zk_hosts=None, zk_max_versions=None, zk_path=None, + def __init__(self, zk=None, zk_future_timeout=None, zk_hosts=None, zk_max_versions=None, zk_path=None, zk_session_timeout=None, zk_state=None, zk_timeout=None): self.zk = zk self.zk_future_timeout = zk_future_timeout diff --git a/marathon/models/task.py b/marathon/models/task.py index c9dcf3c..3fddd39 100644 --- a/marathon/models/task.py +++ b/marathon/models/task.py @@ -25,8 +25,7 @@ class MarathonTask(MarathonResource): DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ' - def __init__( - self, app_id=None, health_check_results=None, host=None, id=None, ports=None, service_ports=None, + def __init__(self, app_id=None, health_check_results=None, host=None, id=None, ports=None, service_ports=None, slave_id=None, staged_at=None, started_at=None, version=None, ip_addresses=[], state=None): self.app_id = app_id self.health_check_results = health_check_results or [] @@ -59,6 +58,7 @@ def __init__(self, ip_address=None, protocol=None): self.ip_address = ip_address self.protocol = protocol + class MarathonHealthCheckResult(MarathonObject): """Marathon health check result. @@ -76,8 +76,7 @@ class MarathonHealthCheckResult(MarathonObject): DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ' - def __init__( - self, alive=None, consecutive_failures=None, first_success=None, + def __init__(self, alive=None, consecutive_failures=None, first_success=None, last_failure=None, last_success=None, task_id=None, last_failure_cause=None): self.alive = alive self.consecutive_failures = consecutive_failures diff --git a/tests/test_api.py b/tests/test_api.py index 22af1d6..72f6b4a 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -116,7 +116,17 @@ def test_get_deployments_post_1_0(m): @requests_mock.mock() def test_list_tasks_with_app_id(m): - fake_response = '{ "tasks": [ { "appId": "/anapp", "healthCheckResults": [ { "alive": true, "consecutiveFailures": 0, "firstSuccess": "2014-10-03T22:57:02.246Z", "lastFailure": null, "lastSuccess": "2014-10-03T22:57:41.643Z", "taskId": "bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799" } ], "host": "10.141.141.10", "id": "bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799", "ports": [ 31000 ], "servicePorts": [ 9000 ], "stagedAt": "2014-10-03T22:16:27.811Z", "startedAt": "2014-10-03T22:57:41.587Z", "version": "2014-10-03T22:16:23.634Z" }, { "appId": "/anotherapp", "healthCheckResults": [ { "alive": true, "consecutiveFailures": 0, "firstSuccess": "2014-10-03T22:57:02.246Z", "lastFailure": null, "lastSuccess": "2014-10-03T22:57:41.649Z", "taskId": "bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799" } ], "host": "10.141.141.10", "id": "bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799", "ports": [ 31001 ], "servicePorts": [ 9000 ], "stagedAt": "2014-10-03T22:16:33.814Z", "startedAt": "2014-10-03T22:57:41.593Z", "version": "2014-10-03T22:16:23.634Z" } ] }' + fake_response = '{ "tasks": [ { "appId": "/anapp", "healthCheckResults": ' \ + '[ { "alive": true, "consecutiveFailures": 0, "firstSuccess": "2014-10-03T22:57:02.246Z", ' \ + '"lastFailure": null, "lastSuccess": "2014-10-03T22:57:41.643Z", "taskId": "bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799" } ],' \ + ' "host": "10.141.141.10", "id": "bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799", "ports": [ 31000 ], ' \ + '"servicePorts": [ 9000 ], "stagedAt": "2014-10-03T22:16:27.811Z", "startedAt": "2014-10-03T22:57:41.587Z", ' \ + '"version": "2014-10-03T22:16:23.634Z" }, { "appId": "/anotherapp", ' \ + '"healthCheckResults": [ { "alive": true, "consecutiveFailures": 0, "firstSuccess": "2014-10-03T22:57:02.246Z", "lastFailure": null, ' \ + '"lastSuccess": "2014-10-03T22:57:41.649Z", "taskId": "bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799" } ], ' \ + '"host": "10.141.141.10", "id": "bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799", "ports": [ 31001 ], ' \ + '"servicePorts": [ 9000 ], "stagedAt": "2014-10-03T22:16:33.814Z", "startedAt": "2014-10-03T22:57:41.593Z", ' \ + '"version": "2014-10-03T22:16:23.634Z" } ] }' m.get('http://fake_server/v2/tasks', text=fake_response) mock_client = MarathonClient(servers='http://fake_server') actual_deployments = mock_client.list_tasks(app_id='/anapp') @@ -149,7 +159,16 @@ def test_list_tasks_with_app_id(m): @requests_mock.mock() def test_list_tasks_without_app_id(m): - fake_response = '{ "tasks": [ { "appId": "/anapp", "healthCheckResults": [ { "alive": true, "consecutiveFailures": 0, "firstSuccess": "2014-10-03T22:57:02.246Z", "lastFailure": null, "lastSuccess": "2014-10-03T22:57:41.643Z", "taskId": "bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799" } ], "host": "10.141.141.10", "id": "bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799", "ports": [ 31000 ], "servicePorts": [ 9000 ], "stagedAt": "2014-10-03T22:16:27.811Z", "startedAt": "2014-10-03T22:57:41.587Z", "version": "2014-10-03T22:16:23.634Z" }, { "appId": "/anotherapp", "healthCheckResults": [ { "alive": true, "consecutiveFailures": 0, "firstSuccess": "2014-10-03T22:57:02.246Z", "lastFailure": null, "lastSuccess": "2014-10-03T22:57:41.649Z", "taskId": "bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799" } ], "host": "10.141.141.10", "id": "bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799", "ports": [ 31001 ], "servicePorts": [ 9000 ], "stagedAt": "2014-10-03T22:16:33.814Z", "startedAt": "2014-10-03T22:57:41.593Z", "version": "2014-10-03T22:16:23.634Z" } ] }' + fake_response = '{ "tasks": [ { "appId": "/anapp", "healthCheckResults": ' \ + '[ { "alive": true, "consecutiveFailures": 0, "firstSuccess": "2014-10-03T22:57:02.246Z", "lastFailure": null, ' \ + '"lastSuccess": "2014-10-03T22:57:41.643Z", "taskId": "bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799" } ],' \ + ' "host": "10.141.141.10", "id": "bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799", "ports": [ 31000 ], ' \ + '"servicePorts": [ 9000 ], "stagedAt": "2014-10-03T22:16:27.811Z", "startedAt": "2014-10-03T22:57:41.587Z", ' \ + '"version": "2014-10-03T22:16:23.634Z" }, { "appId": "/anotherapp", ' \ + '"healthCheckResults": [ { "alive": true, "consecutiveFailures": 0, "firstSuccess": "2014-10-03T22:57:02.246Z", ' \ + '"lastFailure": null, "lastSuccess": "2014-10-03T22:57:41.649Z", "taskId": "bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799" } ], ' \ + '"host": "10.141.141.10", "id": "bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799", "ports": [ 31001 ], "servicePorts": [ 9000 ], ' \ + '"stagedAt": "2014-10-03T22:16:33.814Z", "startedAt": "2014-10-03T22:57:41.593Z", "version": "2014-10-03T22:16:23.634Z" } ] }' m.get('http://fake_server/v2/tasks', text=fake_response) mock_client = MarathonClient(servers='http://fake_server') actual_deployments = mock_client.list_tasks() diff --git a/tox.ini b/tox.ini index 267f23b..c0529b8 100644 --- a/tox.ini +++ b/tox.ini @@ -47,5 +47,4 @@ commands = flake8 . [flake8] exclude = .tox,*.egg,docs,build,__init__.py -ignore = E226,E302,E41,E501,E131 max-line-length = 160 From 919c1ffa754776b4fd1bfb99391da1037a37f91d Mon Sep 17 00:00:00 2001 From: Usman Masood Date: Mon, 20 Jun 2016 12:09:50 -0700 Subject: [PATCH 090/292] Add local_volumes to MarathonTask --- marathon/models/task.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/marathon/models/task.py b/marathon/models/task.py index 3fddd39..fad8808 100644 --- a/marathon/models/task.py +++ b/marathon/models/task.py @@ -26,7 +26,7 @@ class MarathonTask(MarathonResource): DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ' def __init__(self, app_id=None, health_check_results=None, host=None, id=None, ports=None, service_ports=None, - slave_id=None, staged_at=None, started_at=None, version=None, ip_addresses=[], state=None): + slave_id=None, staged_at=None, started_at=None, version=None, ip_addresses=[], state=None, local_volumes=None): self.app_id = app_id self.health_check_results = health_check_results or [] self.health_check_results = [ @@ -49,6 +49,7 @@ def __init__(self, app_id=None, health_check_results=None, host=None, id=None, p ipaddr if isinstance( ip_addresses, MarathonIpAddress) else MarathonIpAddress().from_json(ipaddr) for ipaddr in (ip_addresses or [])] + self.local_volumes = local_volumes or [] class MarathonIpAddress(MarathonObject): From 17a28964785f3eb39f96d07968358b20be12e30e Mon Sep 17 00:00:00 2001 From: Stefan Tjarks Date: Thu, 23 Jun 2016 08:19:57 -0700 Subject: [PATCH 091/292] Handle HTTP errors without content graceful HTTP errors like 503 do not have a content set by Marathon. Try to use the response reason string as an alternative error message. --- marathon/exceptions.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/marathon/exceptions.py b/marathon/exceptions.py index 439d972..e5cf597 100644 --- a/marathon/exceptions.py +++ b/marathon/exceptions.py @@ -8,9 +8,11 @@ def __init__(self, response): """ :param :class:`requests.Response` response: HTTP response """ - content = response.json() + self.error_message = response.reason or '' + if response.content: + content = response.json() + self.error_message = content.get('message', self.error_message) self.status_code = response.status_code - self.error_message = content.get('message') super(MarathonHttpError, self).__init__(self.__str__()) def __repr__(self): From 8879d94f687069c3e699ec7e15cea26c72dc0615 Mon Sep 17 00:00:00 2001 From: Stefan Tjarks Date: Thu, 23 Jun 2016 11:00:14 -0700 Subject: [PATCH 092/292] On MarathonApp create make sure port_definitions ends up being a list of PortDefinition instances --- marathon/models/app.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/marathon/models/app.py b/marathon/models/app.py index 8e81055..9dd94be 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -125,7 +125,11 @@ def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, self.max_launch_delay_seconds = max_launch_delay_seconds self.mem = mem self.ports = ports or [] - self.port_definitions = port_definitions or [] + self.port_definitions = [ + pd if isinstance( + pd, PortDefinition) else PortDefinition.from_json(pd) + for pd in (port_definitions or []) + ] self.readiness_checks = readiness_checks or [] self.readiness_check_results = readiness_check_results or [] self.residency = residency From 0451bf36a92082bac31c7022740a4efe9fee58de Mon Sep 17 00:00:00 2001 From: Mateusz Moneta Date: Fri, 24 Jun 2016 11:51:57 +0200 Subject: [PATCH 093/292] Fix `Client.get_version` method. --- marathon/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marathon/client.py b/marathon/client.py index 37b1497..e19daa6 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -569,7 +569,7 @@ def get_version(self, app_id, version): """ response = self._do_request('GET', '/v2/apps/{app_id}/versions/{version}' .format(app_id=app_id, version=version)) - return MarathonApp(response.json()) + return MarathonApp.from_json(response.json()) def list_event_subscriptions(self): """List the event subscriber callback URLs. From 92de114c8b3410e582c5f332c4e4c6c009a62980 Mon Sep 17 00:00:00 2001 From: Mateusz Moneta Date: Fri, 24 Jun 2016 12:11:35 +0200 Subject: [PATCH 094/292] Use requests.Session while communicating with Marathon. --- marathon/client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index 37b1497..6300248 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -32,6 +32,7 @@ def __init__(self, servers, username=None, password=None, timeout=10): :param str password: Basic auth password :param int timeout: Timeout (in seconds) for requests to Marathon """ + self.session = requests.Session() self.servers = servers if isinstance(servers, list) else [servers] self.auth = (username, password) if username and password else None self.timeout = timeout @@ -59,8 +60,9 @@ def _do_request(self, method, path, params=None, data=None): server = servers.pop(0) url = ''.join([server.rstrip('/'), path]) try: - response = requests.request(method, url, params=params, data=data, headers=headers, - auth=self.auth, timeout=self.timeout) + response = self.session.request( + method, url, params=params, data=data, headers=headers, + auth=self.auth, timeout=self.timeout) marathon.log.info('Got response from %s', server) except requests.exceptions.RequestException as e: marathon.log.error( From 8d0220a76dd19c241c9895d59e64819af0a11c2e Mon Sep 17 00:00:00 2001 From: Dmitry Fedorov Date: Fri, 24 Jun 2016 14:05:22 +0300 Subject: [PATCH 095/292] Type assertion for ReadinessCheck in MarathonApp.__init__ method added --- marathon/models/app.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/marathon/models/app.py b/marathon/models/app.py index 9dd94be..ed72027 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -59,7 +59,7 @@ class MarathonApp(MarathonResource): :param task_stats: task statistics :type task_stats: :class:`marathon.models.app.MarathonTaskStats` or dict :param dict labels - :type readiness_checks: list[:class:`marathon.models.app.ReadinessChecks`] or list[dict] + :type readiness_checks: list[:class:`marathon.models.app.ReadinessCheck`] or list[dict] :type residency: :class:`marathon.models.app.Residency` or dict """ @@ -130,7 +130,11 @@ def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, pd, PortDefinition) else PortDefinition.from_json(pd) for pd in (port_definitions or []) ] - self.readiness_checks = readiness_checks or [] + self.readiness_checks = [ + rc if isinstance( + rc, ReadinessCheck) else ReadinessCheck().from_json(rc) + for rc in (readiness_checks or []) + ] self.readiness_check_results = readiness_check_results or [] self.residency = residency self.require_ports = require_ports From eadb18ff3dd1e6a1fdba1232093f6cd7b4336a0f Mon Sep 17 00:00:00 2001 From: Stefan Tjarks Date: Fri, 24 Jun 2016 13:37:52 -0700 Subject: [PATCH 096/292] Issue #70: Remove resource_name from get_group The Marathon response for a GET /v2/groups/{group_id} call does not have a resource_name in the JSON response. --- marathon/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marathon/client.py b/marathon/client.py index 0b883a0..3abb9bb 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -351,7 +351,7 @@ def get_group(self, group_id): """ response = self._do_request( 'GET', '/v2/groups/{group_id}'.format(group_id=group_id)) - return self._parse_response(response, MarathonGroup, resource_name='group') + return self._parse_response(response, MarathonGroup) def update_group(self, group_id, group, force=False, minimal=True): """Update a group. From b22b074a67b45a621b224e1565c5764984f8a6fc Mon Sep 17 00:00:00 2001 From: Dmitry Fedorov Date: Mon, 18 Jul 2016 16:45:27 +0300 Subject: [PATCH 097/292] Issue126: secrets and taskKillGracePeriodSeconds Marathon.App fields added --- marathon/models/app.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/marathon/models/app.py b/marathon/models/app.py index ed72027..064db06 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -38,6 +38,7 @@ class MarathonApp(MarathonResource): :param last_task_failure: last task failure :type last_task_failure: :class:`marathon.models.app.MarathonTaskFailure` or dict :param float mem: memory (in MB) required per instance + :param dict secrets: A map with named secret declarations. :type port_definitions: list[:class:`marathon.models.app.PortDefinitions`] or list[dict] :param list[int] ports: ports :param bool require_ports: require the specified `ports` to be available in the resource offer @@ -61,6 +62,7 @@ class MarathonApp(MarathonResource): :param dict labels :type readiness_checks: list[:class:`marathon.models.app.ReadinessCheck`] or list[dict] :type residency: :class:`marathon.models.app.Residency` or dict + :param int task_kill_grace_period_seconds: Configures the termination signal escalation behavior of executors when stopping tasks. """ UPDATE_OK_ATTRIBUTES = [ @@ -82,9 +84,10 @@ def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, executor=None, health_checks=None, id=None, instances=None, labels=None, last_task_failure=None, max_launch_delay_seconds=None, mem=None, ports=None, require_ports=None, store_urls=None, task_rate_limit=None, tasks=None, tasks_running=None, tasks_staged=None, tasks_healthy=None, - tasks_unhealthy=None, upgrade_strategy=None, uris=None, user=None, version=None, version_info=None, + task_kill_grace_period_seconds=None, tasks_unhealthy=None, upgrade_strategy=None, + uris=None, user=None, version=None, version_info=None, ip_address=None, fetch=None, task_stats=None, readiness_checks=None, - readiness_check_results=None, port_definitions=None, residency=None): + readiness_check_results=None, secrets=None, port_definitions=None, residency=None,): # self.args = args or [] self.accepted_resource_roles = accepted_resource_roles @@ -138,6 +141,12 @@ def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, self.readiness_check_results = readiness_check_results or [] self.residency = residency self.require_ports = require_ports + + self.secrets = secrets or {} + for k, s in self.secrets.iteritems(): + if not isinstance(s, Secret): + self.secrets[k] = Secret().from_json(s) + self.store_urls = store_urls or [] self.task_rate_limit = task_rate_limit self.tasks = [ @@ -147,6 +156,7 @@ def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, self.tasks_running = tasks_running self.tasks_staged = tasks_staged self.tasks_healthy = tasks_healthy + self.task_kill_grace_period_seconds = task_kill_grace_period_seconds self.tasks_unhealthy = tasks_unhealthy self.upgrade_strategy = upgrade_strategy if (isinstance(upgrade_strategy, MarathonUpgradeStrategy) or upgrade_strategy is None) \ else MarathonUpgradeStrategy.from_json(upgrade_strategy) @@ -416,3 +426,13 @@ class Residency(MarathonObject): def __init__(self, relaunch_escalation_timeout_seconds=None, task_lost_behavior=None): self.relaunch_escalation_timeout_seconds = relaunch_escalation_timeout_seconds self.task_lost_behavior = task_lost_behavior + + +class Secret(MarathonObject): + """Declares marathon secret object. + :param str source: The source of the secret's value. The format depends on the secret store used by Mesos. + + """ + + def __init__(self, source=None): + self.source = source From 4ec36f1f0b9027cd034fbfa1c9becdeb86fe6371 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Mon, 18 Jul 2016 14:29:30 -0700 Subject: [PATCH 098/292] Try to fix java hostname issues --- .travis.yml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5478adb..2a003d0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,13 +20,5 @@ script: - make itests # Work around travis-ci/travis-ci#5227 -before_install: - - cat /etc/hosts # optionally check the content *before* - - sudo hostname "$(hostname | cut -c1-63)" - - sed -e "s/^\\(127\\.0\\.0\\.1.*\\)/\\1 $(hostname | cut -c1-63)/" /etc/hosts | sudo tee /etc/hosts - - cat /etc/hosts # optionally check the content *after* - -# Work around and avoid travis on gce as it is dog slow -sudo: required -dist: precise -group: legacy +addons: + hostname: localhost From 30b8bb7497f58f505d952570d86de882a4cdb121 Mon Sep 17 00:00:00 2001 From: Kevin Mooney Date: Tue, 19 Jul 2016 12:49:55 -0500 Subject: [PATCH 099/292] Expose id query param in list_apps --- marathon/client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/marathon/client.py b/marathon/client.py index 3abb9bb..32f3870 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -139,7 +139,7 @@ def create_app(self, app_id, app): def list_apps(self, cmd=None, embed_tasks=False, embed_counts=False, embed_deployments=False, embed_readiness=False, embed_last_task_failure=False, embed_failures=False, - embed_task_stats=False, **kwargs): + embed_task_stats=False, app_id=None, **kwargs): """List all apps. :param str cmd: if passed, only show apps with a matching `cmd` @@ -150,6 +150,7 @@ def list_apps(self, cmd=None, embed_tasks=False, embed_counts=False, :param bool embed_last_task_failure: embeds the last task failure :param bool embed_failures: shorthand for embed_last_task_failure :param bool embed_task_stats: embed task stats in result + :param bool app_id: if passed, only show apps with with an 'id' that matches or contains this value :param kwargs: arbitrary search filters :returns: list of applications @@ -158,6 +159,8 @@ def list_apps(self, cmd=None, embed_tasks=False, embed_counts=False, params = {} if cmd: params['cmd'] = cmd + if app_id: + params['id'] = app_id embed_params = { 'app.tasks': embed_tasks, From c0a761d59f462a1451d0a602a8577d814cdb93af Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Tue, 19 Jul 2016 14:14:03 -0700 Subject: [PATCH 100/292] Release 0.8.3 --- CHANGELOG.md | 29 +++++++++++++++++++++++++++++ setup.py | 2 +- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ffc77c6..e0d5416 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,34 @@ # Change Log +## [0.8.3](https://github.com/thefactory/marathon-python/tree/0.8.3) (2016-07-19) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.2...0.8.3) + +**Closed issues:** + +- New marathon application structure field [\#126](https://github.com/thefactory/marathon-python/issues/126) +- MarathonReadinessCheck class is absent [\#122](https://github.com/thefactory/marathon-python/issues/122) +- Supporting creating applications with a json file \(or json-formatted string, or json object\) [\#112](https://github.com/thefactory/marathon-python/issues/112) +- Task.ip\_addresses are not set properly [\#110](https://github.com/thefactory/marathon-python/issues/110) +- RuntimeError: maximum recursion depth exceeded in cmp when calling create\_app [\#60](https://github.com/thefactory/marathon-python/issues/60) + +**Merged pull requests:** + +- Try to fix java hostname issues [\#128](https://github.com/thefactory/marathon-python/pull/128) ([solarkennedy](https://github.com/solarkennedy)) +- Issue126: secrets and taskKillGracePeriodSeconds Marathon.App fields … [\#127](https://github.com/thefactory/marathon-python/pull/127) ([dmajere](https://github.com/dmajere)) +- Issue \#70: Remove resource\_name from get\_group [\#124](https://github.com/thefactory/marathon-python/pull/124) ([stj](https://github.com/stj)) +- Type assertion for ReadinessCheck in MarathonApp.\_\_init\_\_ method added [\#123](https://github.com/thefactory/marathon-python/pull/123) ([dmajere](https://github.com/dmajere)) +- Use requests.Session while communicating with Marathon. [\#121](https://github.com/thefactory/marathon-python/pull/121) ([Nihn](https://github.com/Nihn)) +- Fix `Client.get\_version` method. [\#120](https://github.com/thefactory/marathon-python/pull/120) ([Nihn](https://github.com/Nihn)) +- Handle HTTP errors without content graceful [\#119](https://github.com/thefactory/marathon-python/pull/119) ([stj](https://github.com/stj)) +- Add local\_volumes to MarathonTask [\#118](https://github.com/thefactory/marathon-python/pull/118) ([usmanm](https://github.com/usmanm)) +- Fix pep8 issues and strict flake8 check. [\#117](https://github.com/thefactory/marathon-python/pull/117) ([oilbeater](https://github.com/oilbeater)) +- Don't die if no JSON found in response [\#116](https://github.com/thefactory/marathon-python/pull/116) ([usmanm](https://github.com/usmanm)) +- Set fetch in MarathonApp. [\#115](https://github.com/thefactory/marathon-python/pull/115) ([oilbeater](https://github.com/oilbeater)) +- Add in state to the task model [\#114](https://github.com/thefactory/marathon-python/pull/114) ([chuckwired](https://github.com/chuckwired)) +- Add 'persistent' volume option [\#113](https://github.com/thefactory/marathon-python/pull/113) ([jimbobhickville](https://github.com/jimbobhickville)) +- Issue110: MarathonTask.ip\_addresses attribute is set properly [\#111](https://github.com/thefactory/marathon-python/pull/111) ([dmajere](https://github.com/dmajere)) +- Add message field for MarathonStatusUpdateEvent. [\#109](https://github.com/thefactory/marathon-python/pull/109) ([oilbeater](https://github.com/oilbeater)) + ## [0.8.2](https://github.com/thefactory/marathon-python/tree/0.8.2) (2016-06-14) [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.1...0.8.2) diff --git a/setup.py b/setup.py index 030fb8a..c92516e 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='marathon', - version='0.8.2', + version='0.8.3', description='Marathon Client Library', long_description="""Python interface to the Mesos Marathon REST API.""", author='Mike Babineau', From 4d25a2772d3636bf7686aa33b81709e5f2d7b008 Mon Sep 17 00:00:00 2001 From: Greg Hill Date: Wed, 20 Jul 2016 09:01:46 -0500 Subject: [PATCH 101/292] Fix py3k regression in 0.8.3 .items() works in both. This dictionary won't ever be especially large, so the performance hit for not using an iterator in py27 is negligible. --- marathon/models/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marathon/models/app.py b/marathon/models/app.py index 064db06..281ee19 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -143,7 +143,7 @@ def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, self.require_ports = require_ports self.secrets = secrets or {} - for k, s in self.secrets.iteritems(): + for k, s in self.secrets.items(): if not isinstance(s, Secret): self.secrets[k] = Secret().from_json(s) From ccf0b2eefe4dc072d8ffe3c13689b1a2aef6563a Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Wed, 20 Jul 2016 09:30:56 -0700 Subject: [PATCH 102/292] Release 0.8.4 --- CHANGELOG.md | 12 ++++++++++++ setup.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e0d5416..c43f28f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Change Log +## [0.8.4](https://github.com/thefactory/marathon-python/tree/0.8.4) (2016-07-20) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.3...0.8.4) + +**Closed issues:** + +- Can we get another Pypi release? [\#130](https://github.com/thefactory/marathon-python/issues/130) + +**Merged pull requests:** + +- Fix py3k regression in 0.8.3 [\#131](https://github.com/thefactory/marathon-python/pull/131) ([jimbobhickville](https://github.com/jimbobhickville)) +- Expose id query param in list\_apps [\#129](https://github.com/thefactory/marathon-python/pull/129) ([moonkev](https://github.com/moonkev)) + ## [0.8.3](https://github.com/thefactory/marathon-python/tree/0.8.3) (2016-07-19) [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.2...0.8.3) diff --git a/setup.py b/setup.py index c92516e..653e935 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='marathon', - version='0.8.3', + version='0.8.4', description='Marathon Client Library', long_description="""Python interface to the Mesos Marathon REST API.""", author='Mike Babineau', From 2ba5a3d7547577cadcd6b987c655f579857d405f Mon Sep 17 00:00:00 2001 From: Nathan Handler Date: Tue, 2 Aug 2016 11:34:31 -0700 Subject: [PATCH 103/292] Fix installation of requests module --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index c0529b8..1246035 100644 --- a/tox.ini +++ b/tox.ini @@ -28,7 +28,6 @@ commands = [testenv] usedevelop=True basepython = python2.7 -install_command = pip install --upgrade {opts} {packages} deps = -rrequirements.txt [testenv:py] From e2e5772b9756fa168b74c3fdcf3b2467247145fe Mon Sep 17 00:00:00 2001 From: Nathan Handler Date: Tue, 2 Aug 2016 11:44:03 -0700 Subject: [PATCH 104/292] Fix docker install issue with software-properties-common --- itests/Dockerfile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/itests/Dockerfile b/itests/Dockerfile index 5135abb..8077318 100644 --- a/itests/Dockerfile +++ b/itests/Dockerfile @@ -1,9 +1,13 @@ FROM ubuntu:14.04 -RUN apt-get install -y software-properties-common + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install \ + software-properties-common RUN add-apt-repository ppa:webupd8team/java RUN echo "debconf shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections RUN echo "debconf shared/accepted-oracle-license-v1-1 seen true" | debconf-set-selections -RUN apt-get update && apt-get -y install lsb-release oracle-java8-installer +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install \ + lsb-release \ + oracle-java8-installer # Setup ADD ./marathon-version /root/marathon-version From 591b12cd0c5099ae52cb080a4ad31302c5e4a33e Mon Sep 17 00:00:00 2001 From: Nathan Handler Date: Wed, 3 Aug 2016 10:59:42 -0700 Subject: [PATCH 105/292] Make travis test against 1.1.2 instead of 1.1.1 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2a003d0..ec4ac60 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ env: - MARATHONVERSION: 0.13.1 - MARATHONVERSION: 0.14.1 - MARATHONVERSION: 0.15.3 - - MARATHONVERSION: 1.1.1 + - MARATHONVERSION: 1.1.2 language: python python: From 4a46d2138f6fdaaf4a931ac98313eb60a3df2926 Mon Sep 17 00:00:00 2001 From: Nathan Handler Date: Wed, 3 Aug 2016 11:01:01 -0700 Subject: [PATCH 106/292] Use mesos 1.0.* instead of mesos 0.23.* --- itests/install-marathon.sh | 2 +- itests/start-marathon.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/itests/install-marathon.sh b/itests/install-marathon.sh index 4c44f79..e83a4b5 100755 --- a/itests/install-marathon.sh +++ b/itests/install-marathon.sh @@ -23,7 +23,7 @@ sudo apt-get -y purge oracle-java7-installer sudo update-java-alternatives -s java-8-oracle sudo apt-get install oracle-java8-set-default -sudo apt-get -y --force-yes install mesos=0.23.* marathon=$MARATHONVERSION* +sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes install mesos=1.0.* marathon=$MARATHONVERSION* # WTF MARATHON? # Why does the precise version have java7 hardcoded if it requires java8? diff --git a/itests/start-marathon.sh b/itests/start-marathon.sh index 6c21fa2..6b18743 100755 --- a/itests/start-marathon.sh +++ b/itests/start-marathon.sh @@ -7,4 +7,6 @@ else fi java -version +export MESOS_WORK_DIR='/tmp/mesos' +mkdir -p "$MESOS_WORK_DIR" exec /usr/bin/marathon --master local $LOGGER --hostname localhost From 1e15f12046643393b65573151b6c14a8307ff0a5 Mon Sep 17 00:00:00 2001 From: Nathan Handler Date: Wed, 3 Aug 2016 11:02:25 -0700 Subject: [PATCH 107/292] Clean up the install-marathon.sh script --- itests/install-marathon.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/itests/install-marathon.sh b/itests/install-marathon.sh index e83a4b5..8425b66 100755 --- a/itests/install-marathon.sh +++ b/itests/install-marathon.sh @@ -5,23 +5,21 @@ set -vxeu [[ -f /root/marathon-version ]] && source /root/marathon-version MARATHONVERSION="${MARATHONVERSION:-0.8.2}" -sudo apt-get update -q - # Setup -sudo apt-key adv --keyserver keyserver.ubuntu.com --recv E56151BF +sudo apt-key adv --keyserver keyserver.ubuntu.com --recv 81026D0004C44CF7EF55ADF8DF7D54CBE56151BF DISTRO=$(lsb_release -is | tr '[:upper:]' '[:lower:]') CODENAME=$(lsb_release -cs) # Add the repository echo "deb http://repos.mesosphere.com/${DISTRO} ${CODENAME} main" | sudo tee /etc/apt/sources.list.d/mesosphere.list -sudo apt-get -y update +sudo apt-get update # Install packages -sudo apt-get -y install oracle-java8-installer +sudo DEBIAN_FRONTEND=noninteractive apt-get -y install oracle-java8-installer sudo apt-get -y purge oracle-java7-installer sudo update-java-alternatives -s java-8-oracle -sudo apt-get install oracle-java8-set-default +sudo DEBIAN_FRONTEND=noninteractive apt-get install oracle-java8-set-default sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes install mesos=1.0.* marathon=$MARATHONVERSION* From 45aa1a93059edb894c36c95f26bcde9e4b8c991d Mon Sep 17 00:00:00 2001 From: Ammar Askar Date: Tue, 9 Aug 2016 18:03:00 -0700 Subject: [PATCH 108/292] Allow setting of a custom requests session This allows programs using this library to set a user-agent for all requests, set custom headers etc --- marathon/client.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index 32f3870..ac63e3b 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -19,7 +19,7 @@ class MarathonClient(object): """Client interface for the Marathon REST API.""" - def __init__(self, servers, username=None, password=None, timeout=10): + def __init__(self, servers, username=None, password=None, timeout=10, session=None): """Create a MarathonClient instance. If multiple servers are specified, each will be tried in succession until a non-"Connection Error"-type @@ -32,7 +32,10 @@ def __init__(self, servers, username=None, password=None, timeout=10): :param str password: Basic auth password :param int timeout: Timeout (in seconds) for requests to Marathon """ - self.session = requests.Session() + if session is None: + self.session = requests.Session() + else: + self.session = session self.servers = servers if isinstance(servers, list) else [servers] self.auth = (username, password) if username and password else None self.timeout = timeout From 72ff423b26b06eafb7b28364bb179ec1ee4ba4bd Mon Sep 17 00:00:00 2001 From: Kevin Mooney Date: Wed, 10 Aug 2016 12:25:17 -0500 Subject: [PATCH 109/292] Add update_apps method to client --- marathon/client.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/marathon/client.py b/marathon/client.py index 32f3870..be0dee8 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -13,6 +13,7 @@ from .models import MarathonApp, MarathonDeployment, MarathonGroup, MarathonInfo, MarathonTask, MarathonEndpoint, MarathonQueueItem from .exceptions import InternalServerError, NotFoundError, MarathonHttpError, MarathonError from .models.events import EventFactory +from .util import MarathonJsonEncoder, MarathonMinimalJsonEncoder class MarathonClient(object): @@ -256,6 +257,32 @@ def update_app(self, app_id, app, force=False, minimal=True): 'PUT', '/v2/apps/{app_id}'.format(app_id=app_id), params=params, data=data) return response.json() + def update_apps(self, apps, force=False, minimal=True): + """Update multiple apps. + + Applies writable settings in elements of apps either by upgrading existing ones or creating new ones + + :param apps: sequence of application settings + :param bool force: apply even if a deployment is in progress + :param bool minimal: ignore nulls and empty collections + + :returns: a dict containing the deployment id and version + :rtype: dict + """ + json_repr_apps = [] + for app in apps: + # Changes won't take if version is set - blank it for convenience + app.version = None + json_repr_apps.append(app.json_repr(minimal=minimal)) + + params = {'force': force} + encoder = MarathonMinimalJsonEncoder if minimal else MarathonJsonEncoder + data = json.dumps(json_repr_apps, cls=encoder, sort_keys=True) + + response = self._do_request( + 'PUT', '/v2/apps', params=params, data=data) + return response.json() + def rollback_app(self, app_id, version, force=False): """Roll an app back to a previous version. From 8ed32da03237b9778f254245266df268e6f7ea29 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Wed, 10 Aug 2016 11:46:27 -0700 Subject: [PATCH 110/292] Release 0.8.5 --- CHANGELOG.md | 18 ++++++++++++++++-- setup.py | 2 +- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c43f28f..d7a24f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Change Log +## [0.8.5](https://github.com/thefactory/marathon-python/tree/0.8.5) (2016-08-10) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.4...0.8.5) + +**Closed issues:** + +- HTTP 400 returned with message, "Invalid JSON" [\#133](https://github.com/thefactory/marathon-python/issues/133) +- \[Question\] Passing parameters to request.get [\#132](https://github.com/thefactory/marathon-python/issues/132) + +**Merged pull requests:** + +- Add update\_apps method to client [\#136](https://github.com/thefactory/marathon-python/pull/136) ([moonkev](https://github.com/moonkev)) +- Allow setting of a custom requests session [\#135](https://github.com/thefactory/marathon-python/pull/135) ([ammaraskar](https://github.com/ammaraskar)) +- Marathon 1.1.2 and Mesos 1.0.\* [\#134](https://github.com/thefactory/marathon-python/pull/134) ([nhandler](https://github.com/nhandler)) + ## [0.8.4](https://github.com/thefactory/marathon-python/tree/0.8.4) (2016-07-20) [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.3...0.8.4) @@ -29,8 +43,8 @@ - Issue126: secrets and taskKillGracePeriodSeconds Marathon.App fields … [\#127](https://github.com/thefactory/marathon-python/pull/127) ([dmajere](https://github.com/dmajere)) - Issue \#70: Remove resource\_name from get\_group [\#124](https://github.com/thefactory/marathon-python/pull/124) ([stj](https://github.com/stj)) - Type assertion for ReadinessCheck in MarathonApp.\_\_init\_\_ method added [\#123](https://github.com/thefactory/marathon-python/pull/123) ([dmajere](https://github.com/dmajere)) -- Use requests.Session while communicating with Marathon. [\#121](https://github.com/thefactory/marathon-python/pull/121) ([Nihn](https://github.com/Nihn)) -- Fix `Client.get\_version` method. [\#120](https://github.com/thefactory/marathon-python/pull/120) ([Nihn](https://github.com/Nihn)) +- Use requests.Session while communicating with Marathon. [\#121](https://github.com/thefactory/marathon-python/pull/121) ([nihn](https://github.com/nihn)) +- Fix `Client.get\_version` method. [\#120](https://github.com/thefactory/marathon-python/pull/120) ([nihn](https://github.com/nihn)) - Handle HTTP errors without content graceful [\#119](https://github.com/thefactory/marathon-python/pull/119) ([stj](https://github.com/stj)) - Add local\_volumes to MarathonTask [\#118](https://github.com/thefactory/marathon-python/pull/118) ([usmanm](https://github.com/usmanm)) - Fix pep8 issues and strict flake8 check. [\#117](https://github.com/thefactory/marathon-python/pull/117) ([oilbeater](https://github.com/oilbeater)) diff --git a/setup.py b/setup.py index 653e935..3bcd8b2 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='marathon', - version='0.8.4', + version='0.8.5', description='Marathon Client Library', long_description="""Python interface to the Mesos Marathon REST API.""", author='Mike Babineau', From 944d7d5a88b4606d1938cddcf41e107a7b1bc3fa Mon Sep 17 00:00:00 2001 From: Anatolii Lapytskyi Date: Thu, 18 Aug 2016 12:35:44 +0300 Subject: [PATCH 111/292] Add debug on unknown event type --- marathon/models/events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marathon/models/events.py b/marathon/models/events.py index 14e7b38..c2abf8d 100644 --- a/marathon/models/events.py +++ b/marathon/models/events.py @@ -148,4 +148,4 @@ def process(self, event): clazz = self.event_to_class[event_type] return clazz.from_json(event) else: - raise MarathonError('Unknown event_type: {}'.format(event_type)) + raise MarathonError('Unknown event_type: {}, data: {}'.format(event_type, event)) From 30e0c7883878cceaa830aea0b4fc0e10d4cca35e Mon Sep 17 00:00:00 2001 From: Anatolii Lapytskyi Date: Thu, 18 Aug 2016 12:53:31 +0300 Subject: [PATCH 112/292] Add support for unhealthy_task_kill_event --- marathon/models/events.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/marathon/models/events.py b/marathon/models/events.py index c2abf8d..e698674 100644 --- a/marathon/models/events.py +++ b/marathon/models/events.py @@ -111,6 +111,10 @@ class MarathonEventStreamDetached(MarathonEvent): KNOWN_ATTRIBUTES = ['remote_address'] +class MarathonUnhealthyTaskKillEvent(MarathonEvent): + KNOWN_ATTRIBUTES = ['app_id', 'task_id', 'version', 'reason'] + + class EventFactory: """ @@ -131,6 +135,7 @@ def __init__(self): 'remove_health_check_event': MarathonRemoveHealthCheckEvent, 'failed_health_check_event': MarathonFailedHealthCheckEvent, 'health_status_changed_event': MarathonHealthStatusChangedEvent, + 'unhealthy_task_kill_event' : MarathonUnhealthyTaskKillEvent, 'group_change_success': MarathonGroupChangeSuccess, 'group_change_failed': MarathonGroupChangeFailed, 'deployment_success': MarathonDeploymentSuccess, From acbdb089ac7a308b65dc4d8cdb1170345aaf3bf2 Mon Sep 17 00:00:00 2001 From: Anatolii Lapytskyi Date: Thu, 18 Aug 2016 15:06:20 +0300 Subject: [PATCH 113/292] Fix pep8 warning --- marathon/models/events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marathon/models/events.py b/marathon/models/events.py index e698674..b371108 100644 --- a/marathon/models/events.py +++ b/marathon/models/events.py @@ -135,7 +135,7 @@ def __init__(self): 'remove_health_check_event': MarathonRemoveHealthCheckEvent, 'failed_health_check_event': MarathonFailedHealthCheckEvent, 'health_status_changed_event': MarathonHealthStatusChangedEvent, - 'unhealthy_task_kill_event' : MarathonUnhealthyTaskKillEvent, + 'unhealthy_task_kill_event': MarathonUnhealthyTaskKillEvent, 'group_change_success': MarathonGroupChangeSuccess, 'group_change_failed': MarathonGroupChangeFailed, 'deployment_success': MarathonDeploymentSuccess, From dfdaafbaa65bfc0ca1cec6b89fd55c7dd5b33bff Mon Sep 17 00:00:00 2001 From: Mark Beacom Date: Mon, 22 Aug 2016 15:04:39 -0400 Subject: [PATCH 114/292] Fix #140 - Resolve gpus TypeError --- marathon/models/app.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/marathon/models/app.py b/marathon/models/app.py index 281ee19..659423b 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -31,6 +31,7 @@ class MarathonApp(MarathonResource): :type deployments: list[:class:`marathon.models.deployment.MarathonDeployment`] :param dict env: env vars :param str executor: executor + :param int gpus: gpus required per instance :param health_checks: health checks :type health_checks: list[:class:`marathon.models.MarathonHealthCheck`] or list[dict] :param str id: app id @@ -67,8 +68,8 @@ class MarathonApp(MarathonResource): UPDATE_OK_ATTRIBUTES = [ 'args', 'backoff_factor', 'backoff_seconds', 'cmd', 'constraints', 'container', 'cpus', 'dependencies', 'disk', - 'env', 'executor', 'health_checks', 'instances', 'labels', 'max_launch_delay_seconds', 'mem', 'ports', 'require_ports', - 'store_urls', 'task_rate_limit', 'upgrade_strategy', 'uris', 'user', 'version' + 'env', 'executor', 'gpus', 'health_checks', 'instances', 'labels', 'max_launch_delay_seconds', 'mem', 'ports', + 'require_ports', 'store_urls', 'task_rate_limit', 'upgrade_strategy', 'uris', 'user', 'version' ] """List of attributes which may be updated/changed after app creation""" @@ -87,7 +88,7 @@ def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, task_kill_grace_period_seconds=None, tasks_unhealthy=None, upgrade_strategy=None, uris=None, user=None, version=None, version_info=None, ip_address=None, fetch=None, task_stats=None, readiness_checks=None, - readiness_check_results=None, secrets=None, port_definitions=None, residency=None,): + readiness_check_results=None, secrets=None, port_definitions=None, residency=None, gpus=None): # self.args = args or [] self.accepted_resource_roles = accepted_resource_roles @@ -114,6 +115,7 @@ def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, self.disk = disk self.env = env self.executor = executor + self.gpus = gpus self.health_checks = health_checks or [] self.health_checks = [ hc if isinstance( From 275407e253582db07ee4af7d72e2911ae7941bfc Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Mon, 22 Aug 2016 20:11:36 +0100 Subject: [PATCH 115/292] fixup tox to run multiple python version --- Makefile | 4 ++-- itests/itest.sh | 8 ++++++++ itests/itest_utils.py | 11 ++--------- tox.ini | 42 +++++++++++++----------------------------- 4 files changed, 25 insertions(+), 40 deletions(-) create mode 100755 itests/itest.sh diff --git a/Makefile b/Makefile index 4ad278f..2bd0527 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ itests: - tox -e itests + tox -e itest-py27,itest-py33 test: - tox + tox -e test-py27,test-py33 clean: rm -rf dist/ build/ diff --git a/itests/itest.sh b/itests/itest.sh new file mode 100755 index 0000000..7a7ba4f --- /dev/null +++ b/itests/itest.sh @@ -0,0 +1,8 @@ +#!/bin/bash +[[ -n $TRAVIS ]] || echo MARATHONVERSION=$MARATHONVERSION > marathon-version +[[ -n $TRAVIS ]] || docker-compose build +[[ -n $TRAVIS ]] || docker-compose pull +[[ -n $TRAVIS ]] || docker-compose up -d +behave "$@" +[[ -n $TRAVIS ]] || docker-compose stop +[[ -n $TRAVIS ]] || docker-compose rm --force diff --git a/itests/itest_utils.py b/itests/itest_utils.py index ed54055..198f112 100644 --- a/itests/itest_utils.py +++ b/itests/itest_utils.py @@ -7,7 +7,7 @@ import time import requests -from compose.cli import command +import compose.cli.command class TimeoutError(Exception): @@ -55,8 +55,7 @@ def wait_for_marathon(): def get_compose_service(service_name): """Returns a compose object for the service""" - cmd = command.Command() - project = cmd.get_project(cmd.get_config_path()) + project = compose.cli.command.get_project(os.path.dirname(os.path.realpath(__file__))) return project.get_service(service_name) @@ -67,12 +66,6 @@ def get_marathon_connection_string(): else: service_port = get_service_internal_port('marathon') local_port = get_compose_service('marathon').get_container().get_local_port(service_port) - - # Check if we're at OSX. Use ip from DOCKER_HOST - if sys.platform == 'darwin': - m = re.match("(.*?)://(.*?):(\d+)", os.environ["DOCKER_HOST"]) - local_port = "{}:{}".format(m.group(2), local_port.split(":")[1]) - return local_port diff --git a/tox.ini b/tox.ini index 1246035..5a0d1bb 100644 --- a/tox.ini +++ b/tox.ini @@ -1,44 +1,28 @@ [tox] passenv = TRAVIS usedevelop=True -basepython = python2.7 -envlist = py,pep8 +envlist={test,itest}-{py27,py33} -[testenv:itests] +[testenv] passenv = TRAVIS MARATHONVERSION DOCKER_HOST DOCKER_TLS_VERIFY DOCKER_CERT_PATH DOCKER_MACHINE_NAME -basepython = python2.7 +basepython = + py27: python2.7 + py33: python3 whitelist_externals=/bin/bash skipsdist=True -changedir=itests/ +changedir = + test: {toxinidir} + itest: {toxinidir}/itests/ deps = requests<2.7 - {[testenv]deps} - docker-compose==1.3.1 + -rrequirements.txt + docker-compose behave + pytest mock commands = - /bin/bash -c "[[ -n $TRAVIS ]] || echo MARATHONVERSION=$MARATHONVERSION > marathon-version" - /bin/bash -c "[[ -n $TRAVIS ]] || docker-compose build" - /bin/bash -c "[[ -n $TRAVIS ]] || docker-compose pull" - /bin/bash -c "[[ -n $TRAVIS ]] || docker-compose up -d" - behave {posargs} - /bin/bash -c "[[ -n $TRAVIS ]] || docker-compose stop" - /bin/bash -c "[[ -n $TRAVIS ]] || docker-compose rm --force" - -[testenv] -usedevelop=True -basepython = python2.7 -deps = -rrequirements.txt - -[testenv:py] -recreate=True -basepython = python2.7 -deps = - {[testenv]deps} - pytest - mock -commands = - py.test -s -vv {posargs:tests} + test: py.test -s -vv {posargs:tests} + itest: ./itest.sh {posargs} [testenv:pep8] deps = flake8 From 0d0a28dfabe5650e50db2313d22dd13bf38fd0b5 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Mon, 22 Aug 2016 20:37:04 +0100 Subject: [PATCH 116/292] python versions in tox not travis --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ec4ac60..d405dab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,6 @@ env: language: python python: - 2.7 - - 3.4 install: - pip install tox script: From 74cfa6a7ccc55521cee99566c9db6357c62ab0b7 Mon Sep 17 00:00:00 2001 From: Miguel E dos Santos Date: Mon, 22 Aug 2016 16:59:34 -0300 Subject: [PATCH 117/292] Removed sseclient dependency + major enhancements on event_stream() --- marathon/client.py | 33 ++++++++++++++++++++------------- requirements.txt | 1 - 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index 66e49d9..6f17ba6 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -96,24 +96,26 @@ def _do_request(self, method, path, params=None, data=None): return response def _do_sse_request(self, path, params=None, data=None): - from sseclient import SSEClient - headers = {'Accept': 'text/event-stream'} messages = None servers = list(self.servers) + while servers and messages is None: server = servers.pop(0) url = ''.join([server.rstrip('/'), path]) try: - messages = SSEClient(url, params=params, data=data, headers=headers, - auth=self.auth) + response = requests.get( + url, + stream=True, + headers={'Accept': 'text/event-stream'} + ) except Exception as e: marathon.log.error('Error while calling %s: %s', url, e.message) - if messages is None: + if not response.ok: raise MarathonError('No remaining Marathon servers to try') - return messages + return response.iter_lines() def list_endpoints(self): """List the current endpoints for all applications @@ -730,11 +732,16 @@ def event_stream(self): :rtype: iterator """ - messages = self._do_sse_request('/v2/events') - ef = EventFactory() - for message in messages: - if not message.data: - continue - data = json.loads(message.data) - yield ef.process(data) + + for raw_message in self._do_sse_request('/v2/events'): + try: + _data = raw_message.decode('utf8').split(':', 1) + + if _data[0] == 'data': + event_data = json.loads(_data[1].strip()) + if not 'eventType' in event_data: + raise MarathonError('Invalid event data received.') + yield ef.process(event_data) + except ValueError as e: + raise MarathonError('Invalid event data received.') diff --git a/requirements.txt b/requirements.txt index cb1d49e..40e7bdf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1 @@ requests-mock -sseclient From 181b3cf11e32e19413fac4b17115e9d30451a525 Mon Sep 17 00:00:00 2001 From: Miguel E dos Santos Date: Mon, 22 Aug 2016 17:11:07 -0300 Subject: [PATCH 118/292] Minor changes to pass all tests --- marathon/client.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index 6f17ba6..1f096da 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -96,8 +96,6 @@ def _do_request(self, method, path, params=None, data=None): return response def _do_sse_request(self, path, params=None, data=None): - headers = {'Accept': 'text/event-stream'} - messages = None servers = list(self.servers) while servers and messages is None: @@ -743,5 +741,5 @@ def event_stream(self): if not 'eventType' in event_data: raise MarathonError('Invalid event data received.') yield ef.process(event_data) - except ValueError as e: + except ValueError: raise MarathonError('Invalid event data received.') From 8fea13f3eda8f608040b7c6ff33f996ef6c97c4f Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Mon, 22 Aug 2016 21:52:11 +0100 Subject: [PATCH 119/292] fix requests-mock --- itests/itest_utils.py | 4 +- requirements.txt | 2 +- tests/test_api.py | 232 +++++++++++++++++++++--------------------- tox.ini | 2 +- 4 files changed, 120 insertions(+), 120 deletions(-) diff --git a/itests/itest_utils.py b/itests/itest_utils.py index 198f112..6faba02 100644 --- a/itests/itest_utils.py +++ b/itests/itest_utils.py @@ -38,7 +38,7 @@ def wait_for_marathon(): """Blocks until marathon is up""" marathon_service = get_marathon_connection_string() while True: - print 'Connecting to marathon on %s' % marathon_service + print('Connecting to marathon on %s' % marathon_service) try: response = requests.get( 'http://%s/ping' % marathon_service, timeout=2) @@ -49,7 +49,7 @@ def wait_for_marathon(): time.sleep(2) continue if response.status_code == 200: - print "Marathon is up and running!" + print("Marathon is up and running!") break diff --git a/requirements.txt b/requirements.txt index cb1d49e..da03096 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -requests-mock +requests==2.11.1 sseclient diff --git a/tests/test_api.py b/tests/test_api.py index 72f6b4a..aa809d6 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -3,8 +3,7 @@ from marathon import models -@requests_mock.mock() -def test_get_deployments_pre_1_0(m): +def test_get_deployments_pre_1_0(): fake_response = """[ { "affectedApps": [ @@ -30,26 +29,26 @@ def test_get_deployments_pre_1_0(m): "totalSteps": 1 } ]""" - m.get('http://fake_server/v2/deployments', text=fake_response) - mock_client = MarathonClient(servers='http://fake_server') - actual_deployments = mock_client.list_deployments() - expected_deployments = [models.MarathonDeployment( - id=u"fakeid", - steps=[ - [models.MarathonDeploymentAction( - action="ScaleApplication", app="/test")]], - current_actions=[models.MarathonDeploymentAction( - action="ScaleApplication", app="/test")], - current_step=1, - total_steps=1, - affected_apps=[u"/test"], - version=u"fakeversion" - )] - assert expected_deployments == actual_deployments + with requests_mock.mock() as m: + m.get('http://fake_server/v2/deployments', text=fake_response) + mock_client = MarathonClient(servers='http://fake_server') + actual_deployments = mock_client.list_deployments() + expected_deployments = [models.MarathonDeployment( + id=u"fakeid", + steps=[ + [models.MarathonDeploymentAction( + action="ScaleApplication", app="/test")]], + current_actions=[models.MarathonDeploymentAction( + action="ScaleApplication", app="/test")], + current_step=1, + total_steps=1, + affected_apps=[u"/test"], + version=u"fakeversion" + )] + assert expected_deployments == actual_deployments -@requests_mock.mock() -def test_get_deployments_post_1_0(m): +def test_get_deployments_post_1_0(): fake_response = """[ { "id": "4d2ff4d8-fbe5-4239-a886-f0831ed68d20", @@ -86,36 +85,36 @@ def test_get_deployments_post_1_0(m): "totalSteps": 2 } ]""" - m.get('http://fake_server/v2/deployments', text=fake_response) - mock_client = MarathonClient(servers='http://fake_server') - actual_deployments = mock_client.list_deployments() - expected_deployments = [models.MarathonDeployment( - id=u"4d2ff4d8-fbe5-4239-a886-f0831ed68d20", - steps=[ - models.MarathonDeploymentStep( - actions=[models.MarathonDeploymentAction( - type="StartApplication", app="/test-trivial-app")], - ), - models.MarathonDeploymentStep( - actions=[models.MarathonDeploymentAction( - type="ScaleApplication", app="/test-trivial-app")], - ), - ], - current_actions=[models.MarathonDeploymentAction( - action="ScaleApplication", app="/test-trivial-app", readiness_check_results=[]) - ], - current_step=2, - total_steps=2, - affected_apps=[u"/test-trivial-app"], - version=u"2016-04-20T18:00:20.084Z" - )] - # Helpful for tox to see the diff - assert expected_deployments[0].__dict__ == actual_deployments[0].__dict__ - assert expected_deployments == actual_deployments + with requests_mock.mock() as m: + m.get('http://fake_server/v2/deployments', text=fake_response) + mock_client = MarathonClient(servers='http://fake_server') + actual_deployments = mock_client.list_deployments() + expected_deployments = [models.MarathonDeployment( + id=u"4d2ff4d8-fbe5-4239-a886-f0831ed68d20", + steps=[ + models.MarathonDeploymentStep( + actions=[models.MarathonDeploymentAction( + type="StartApplication", app="/test-trivial-app")], + ), + models.MarathonDeploymentStep( + actions=[models.MarathonDeploymentAction( + type="ScaleApplication", app="/test-trivial-app")], + ), + ], + current_actions=[models.MarathonDeploymentAction( + action="ScaleApplication", app="/test-trivial-app", readiness_check_results=[]) + ], + current_step=2, + total_steps=2, + affected_apps=[u"/test-trivial-app"], + version=u"2016-04-20T18:00:20.084Z" + )] + # Helpful for tox to see the diff + assert expected_deployments[0].__dict__ == actual_deployments[0].__dict__ + assert expected_deployments == actual_deployments -@requests_mock.mock() -def test_list_tasks_with_app_id(m): +def test_list_tasks_with_app_id(): fake_response = '{ "tasks": [ { "appId": "/anapp", "healthCheckResults": ' \ '[ { "alive": true, "consecutiveFailures": 0, "firstSuccess": "2014-10-03T22:57:02.246Z", ' \ '"lastFailure": null, "lastSuccess": "2014-10-03T22:57:41.643Z", "taskId": "bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799" } ],' \ @@ -127,53 +126,11 @@ def test_list_tasks_with_app_id(m): '"host": "10.141.141.10", "id": "bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799", "ports": [ 31001 ], ' \ '"servicePorts": [ 9000 ], "stagedAt": "2014-10-03T22:16:33.814Z", "startedAt": "2014-10-03T22:57:41.593Z", ' \ '"version": "2014-10-03T22:16:23.634Z" } ] }' - m.get('http://fake_server/v2/tasks', text=fake_response) - mock_client = MarathonClient(servers='http://fake_server') - actual_deployments = mock_client.list_tasks(app_id='/anapp') - expected_deployments = [models.task.MarathonTask( - app_id="/anapp", - health_check_results=[ - models.task.MarathonHealthCheckResult( - alive=True, - consecutive_failures=0, - first_success="2014-10-03T22:57:02.246Z", - last_failure=None, - last_success="2014-10-03T22:57:41.643Z", - task_id="bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799" - ) - ], - host="10.141.141.10", - id="bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799", - ports=[ - 31000 - ], - service_ports=[ - 9000 - ], - staged_at="2014-10-03T22:16:27.811Z", - started_at="2014-10-03T22:57:41.587Z", - version="2014-10-03T22:16:23.634Z" - )] - assert actual_deployments == expected_deployments - - -@requests_mock.mock() -def test_list_tasks_without_app_id(m): - fake_response = '{ "tasks": [ { "appId": "/anapp", "healthCheckResults": ' \ - '[ { "alive": true, "consecutiveFailures": 0, "firstSuccess": "2014-10-03T22:57:02.246Z", "lastFailure": null, ' \ - '"lastSuccess": "2014-10-03T22:57:41.643Z", "taskId": "bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799" } ],' \ - ' "host": "10.141.141.10", "id": "bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799", "ports": [ 31000 ], ' \ - '"servicePorts": [ 9000 ], "stagedAt": "2014-10-03T22:16:27.811Z", "startedAt": "2014-10-03T22:57:41.587Z", ' \ - '"version": "2014-10-03T22:16:23.634Z" }, { "appId": "/anotherapp", ' \ - '"healthCheckResults": [ { "alive": true, "consecutiveFailures": 0, "firstSuccess": "2014-10-03T22:57:02.246Z", ' \ - '"lastFailure": null, "lastSuccess": "2014-10-03T22:57:41.649Z", "taskId": "bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799" } ], ' \ - '"host": "10.141.141.10", "id": "bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799", "ports": [ 31001 ], "servicePorts": [ 9000 ], ' \ - '"stagedAt": "2014-10-03T22:16:33.814Z", "startedAt": "2014-10-03T22:57:41.593Z", "version": "2014-10-03T22:16:23.634Z" } ] }' - m.get('http://fake_server/v2/tasks', text=fake_response) - mock_client = MarathonClient(servers='http://fake_server') - actual_deployments = mock_client.list_tasks() - expected_deployments = [ - models.task.MarathonTask( + with requests_mock.mock() as m: + m.get('http://fake_server/v2/tasks', text=fake_response) + mock_client = MarathonClient(servers='http://fake_server') + actual_deployments = mock_client.list_tasks(app_id='/anapp') + expected_deployments = [models.task.MarathonTask( app_id="/anapp", health_check_results=[ models.task.MarathonHealthCheckResult( @@ -196,25 +153,68 @@ def test_list_tasks_without_app_id(m): staged_at="2014-10-03T22:16:27.811Z", started_at="2014-10-03T22:57:41.587Z", version="2014-10-03T22:16:23.634Z" - ), - models.task.MarathonTask( - app_id="/anotherapp", - health_check_results=[ - models.task.MarathonHealthCheckResult( - alive=True, - consecutive_failures=0, - first_success="2014-10-03T22:57:02.246Z", - last_failure=None, - last_success="2014-10-03T22:57:41.649Z", - task_id="bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799" - ) - ], - host="10.141.141.10", - id="bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799", - ports=[31001], - service_ports=[9000], - staged_at="2014-10-03T22:16:33.814Z", - started_at="2014-10-03T22:57:41.593Z", - version="2014-10-03T22:16:23.634Z" )] - assert actual_deployments == expected_deployments + assert actual_deployments == expected_deployments + + +def test_list_tasks_without_app_id(): + fake_response = '{ "tasks": [ { "appId": "/anapp", "healthCheckResults": ' \ + '[ { "alive": true, "consecutiveFailures": 0, "firstSuccess": "2014-10-03T22:57:02.246Z", "lastFailure": null, ' \ + '"lastSuccess": "2014-10-03T22:57:41.643Z", "taskId": "bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799" } ],' \ + ' "host": "10.141.141.10", "id": "bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799", "ports": [ 31000 ], ' \ + '"servicePorts": [ 9000 ], "stagedAt": "2014-10-03T22:16:27.811Z", "startedAt": "2014-10-03T22:57:41.587Z", ' \ + '"version": "2014-10-03T22:16:23.634Z" }, { "appId": "/anotherapp", ' \ + '"healthCheckResults": [ { "alive": true, "consecutiveFailures": 0, "firstSuccess": "2014-10-03T22:57:02.246Z", ' \ + '"lastFailure": null, "lastSuccess": "2014-10-03T22:57:41.649Z", "taskId": "bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799" } ], ' \ + '"host": "10.141.141.10", "id": "bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799", "ports": [ 31001 ], "servicePorts": [ 9000 ], ' \ + '"stagedAt": "2014-10-03T22:16:33.814Z", "startedAt": "2014-10-03T22:57:41.593Z", "version": "2014-10-03T22:16:23.634Z" } ] }' + with requests_mock.mock() as m: + m.get('http://fake_server/v2/tasks', text=fake_response) + mock_client = MarathonClient(servers='http://fake_server') + actual_deployments = mock_client.list_tasks() + expected_deployments = [ + models.task.MarathonTask( + app_id="/anapp", + health_check_results=[ + models.task.MarathonHealthCheckResult( + alive=True, + consecutive_failures=0, + first_success="2014-10-03T22:57:02.246Z", + last_failure=None, + last_success="2014-10-03T22:57:41.643Z", + task_id="bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799" + ) + ], + host="10.141.141.10", + id="bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799", + ports=[ + 31000 + ], + service_ports=[ + 9000 + ], + staged_at="2014-10-03T22:16:27.811Z", + started_at="2014-10-03T22:57:41.587Z", + version="2014-10-03T22:16:23.634Z" + ), + models.task.MarathonTask( + app_id="/anotherapp", + health_check_results=[ + models.task.MarathonHealthCheckResult( + alive=True, + consecutive_failures=0, + first_success="2014-10-03T22:57:02.246Z", + last_failure=None, + last_success="2014-10-03T22:57:41.649Z", + task_id="bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799" + ) + ], + host="10.141.141.10", + id="bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799", + ports=[31001], + service_ports=[9000], + staged_at="2014-10-03T22:16:33.814Z", + started_at="2014-10-03T22:57:41.593Z", + version="2014-10-03T22:16:23.634Z" + )] + assert actual_deployments == expected_deployments diff --git a/tox.ini b/tox.ini index 5a0d1bb..001e3ff 100644 --- a/tox.ini +++ b/tox.ini @@ -14,8 +14,8 @@ changedir = test: {toxinidir} itest: {toxinidir}/itests/ deps = - requests<2.7 -rrequirements.txt + requests-mock==1.0.0 docker-compose behave pytest From df0240bba4cecd838f22b66234c49ea791bcc50b Mon Sep 17 00:00:00 2001 From: Miguel Elias dos Santos Date: Mon, 22 Aug 2016 21:56:20 -0300 Subject: [PATCH 120/292] PEP8 fix --- marathon/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marathon/client.py b/marathon/client.py index 1f096da..c8749a3 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -738,7 +738,7 @@ def event_stream(self): if _data[0] == 'data': event_data = json.loads(_data[1].strip()) - if not 'eventType' in event_data: + if 'eventType' not in event_data: raise MarathonError('Invalid event data received.') yield ef.process(event_data) except ValueError: From f350bfbc47d659b0b7005d6c296bc67f22affda8 Mon Sep 17 00:00:00 2001 From: Miguel Elias dos Santos Date: Mon, 22 Aug 2016 22:04:12 -0300 Subject: [PATCH 121/292] PEP8 fix --- marathon/client.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index c8749a3..420881d 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -95,17 +95,16 @@ def _do_request(self, method, path, params=None, data=None): return response - def _do_sse_request(self, path, params=None, data=None): - servers = list(self.servers) - - while servers and messages is None: + def _do_sse_request(self, path): + while list(self.servers) is None: server = servers.pop(0) url = ''.join([server.rstrip('/'), path]) try: response = requests.get( url, stream=True, - headers={'Accept': 'text/event-stream'} + headers={'Accept': 'text/event-stream'}, + auth=self.auth ) except Exception as e: marathon.log.error('Error while calling %s: %s', url, e.message) From 0eb4e4f6fe0ed8bf7aa1a4679bba02446b13b04f Mon Sep 17 00:00:00 2001 From: Miguel Elias dos Santos Date: Mon, 22 Aug 2016 22:08:02 -0300 Subject: [PATCH 122/292] Code cleanup --- marathon/client.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index 420881d..36a799e 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -96,8 +96,7 @@ def _do_request(self, method, path, params=None, data=None): return response def _do_sse_request(self, path): - while list(self.servers) is None: - server = servers.pop(0) + for server in list(self.servers): url = ''.join([server.rstrip('/'), path]) try: response = requests.get( From dedd6ba598f329974c105fe2ae4929c5fe2f2113 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Tue, 23 Aug 2016 09:29:43 +0100 Subject: [PATCH 123/292] run against mesos 0.28 to remove errors --- itests/install-marathon.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/itests/install-marathon.sh b/itests/install-marathon.sh index 8425b66..05bd432 100755 --- a/itests/install-marathon.sh +++ b/itests/install-marathon.sh @@ -21,7 +21,7 @@ sudo apt-get -y purge oracle-java7-installer sudo update-java-alternatives -s java-8-oracle sudo DEBIAN_FRONTEND=noninteractive apt-get install oracle-java8-set-default -sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes install mesos=1.0.* marathon=$MARATHONVERSION* +sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes install mesos=0.28.* marathon=$MARATHONVERSION* # WTF MARATHON? # Why does the precise version have java7 hardcoded if it requires java8? From 4e880cde9a91e3398ebde9e97ad01a2351ed3ba4 Mon Sep 17 00:00:00 2001 From: Guanglu Guo Date: Tue, 23 Aug 2016 16:34:11 +0800 Subject: [PATCH 124/292] Add NONE as valid docker network mode --- marathon/models/container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marathon/models/container.py b/marathon/models/container.py index 3ef4291..0ee0f70 100644 --- a/marathon/models/container.py +++ b/marathon/models/container.py @@ -46,7 +46,7 @@ class MarathonDockerContainer(MarathonObject): :param bool force_pull_image: Force a docker pull before launching """ - NETWORK_MODES = ['BRIDGE', 'HOST'] + NETWORK_MODES = ['BRIDGE', 'HOST', 'NONE'] """Valid network modes""" def __init__(self, image=None, network='HOST', port_mappings=None, parameters=None, privileged=None, From bf211a0d16b0d8fbf8461e452fff634c21f0f52a Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Tue, 23 Aug 2016 18:12:30 +0100 Subject: [PATCH 125/292] add pep8 step --- Makefile | 2 +- itests/itest_utils.py | 2 -- tox.ini | 3 ++- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 2bd0527..89f45ed 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ itests: tox -e itest-py27,itest-py33 test: - tox -e test-py27,test-py33 + tox -e pep8,test-py27,test-py33 clean: rm -rf dist/ build/ diff --git a/itests/itest_utils.py b/itests/itest_utils.py index 6faba02..6b4d17c 100644 --- a/itests/itest_utils.py +++ b/itests/itest_utils.py @@ -2,8 +2,6 @@ from functools import wraps import os import signal -import sys -import re import time import requests diff --git a/tox.ini b/tox.ini index 001e3ff..7841c6e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] passenv = TRAVIS usedevelop=True -envlist={test,itest}-{py27,py33} +envlist={test,itest}-{py27,py33},pep8 [testenv] passenv = TRAVIS MARATHONVERSION DOCKER_HOST DOCKER_TLS_VERIFY DOCKER_CERT_PATH DOCKER_MACHINE_NAME @@ -25,6 +25,7 @@ commands = itest: ./itest.sh {posargs} [testenv:pep8] +basepython = python2.7 deps = flake8 commands = flake8 . From 724119520806645c369a2371b7085d7146cd661b Mon Sep 17 00:00:00 2001 From: Miguel E dos Santos Date: Thu, 25 Aug 2016 12:18:52 -0300 Subject: [PATCH 126/292] Fixing _do_sse_request change of behaviour --- marathon/client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index 36a799e..0a96c6c 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -108,10 +108,10 @@ def _do_sse_request(self, path): except Exception as e: marathon.log.error('Error while calling %s: %s', url, e.message) - if not response.ok: - raise MarathonError('No remaining Marathon servers to try') + if esponse.ok: + return response.iter_lines() - return response.iter_lines() + raise MarathonError('No remaining Marathon servers to try') def list_endpoints(self): """List the current endpoints for all applications From a9851b0ae882e409f78e161694f25040e7f1eb44 Mon Sep 17 00:00:00 2001 From: Miguel E dos Santos Date: Thu, 25 Aug 2016 12:25:23 -0300 Subject: [PATCH 127/292] Fixed variable typo --- marathon/client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/marathon/client.py b/marathon/client.py index 0a96c6c..dc855ad 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -96,6 +96,7 @@ def _do_request(self, method, path, params=None, data=None): return response def _do_sse_request(self, path): + """Query Marathon server for events.""" for server in list(self.servers): url = ''.join([server.rstrip('/'), path]) try: @@ -108,7 +109,7 @@ def _do_sse_request(self, path): except Exception as e: marathon.log.error('Error while calling %s: %s', url, e.message) - if esponse.ok: + if response.ok: return response.iter_lines() raise MarathonError('No remaining Marathon servers to try') From 52c48fa518b5029428d69264a576614b5961859e Mon Sep 17 00:00:00 2001 From: Miguel E dos Santos Date: Mon, 22 Aug 2016 16:59:34 -0300 Subject: [PATCH 128/292] Removed sseclient dependency + major enhancements on event_stream() --- marathon/client.py | 33 ++++++++++++++++++++------------- requirements.txt | 1 - 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index 66e49d9..6f17ba6 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -96,24 +96,26 @@ def _do_request(self, method, path, params=None, data=None): return response def _do_sse_request(self, path, params=None, data=None): - from sseclient import SSEClient - headers = {'Accept': 'text/event-stream'} messages = None servers = list(self.servers) + while servers and messages is None: server = servers.pop(0) url = ''.join([server.rstrip('/'), path]) try: - messages = SSEClient(url, params=params, data=data, headers=headers, - auth=self.auth) + response = requests.get( + url, + stream=True, + headers={'Accept': 'text/event-stream'} + ) except Exception as e: marathon.log.error('Error while calling %s: %s', url, e.message) - if messages is None: + if not response.ok: raise MarathonError('No remaining Marathon servers to try') - return messages + return response.iter_lines() def list_endpoints(self): """List the current endpoints for all applications @@ -730,11 +732,16 @@ def event_stream(self): :rtype: iterator """ - messages = self._do_sse_request('/v2/events') - ef = EventFactory() - for message in messages: - if not message.data: - continue - data = json.loads(message.data) - yield ef.process(data) + + for raw_message in self._do_sse_request('/v2/events'): + try: + _data = raw_message.decode('utf8').split(':', 1) + + if _data[0] == 'data': + event_data = json.loads(_data[1].strip()) + if not 'eventType' in event_data: + raise MarathonError('Invalid event data received.') + yield ef.process(event_data) + except ValueError as e: + raise MarathonError('Invalid event data received.') diff --git a/requirements.txt b/requirements.txt index cb1d49e..40e7bdf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1 @@ requests-mock -sseclient From 03682f87cd2e60757c96e358c900fa2f09ee065b Mon Sep 17 00:00:00 2001 From: Miguel E dos Santos Date: Mon, 22 Aug 2016 17:11:07 -0300 Subject: [PATCH 129/292] Minor changes to pass all tests --- marathon/client.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index 6f17ba6..1f096da 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -96,8 +96,6 @@ def _do_request(self, method, path, params=None, data=None): return response def _do_sse_request(self, path, params=None, data=None): - headers = {'Accept': 'text/event-stream'} - messages = None servers = list(self.servers) while servers and messages is None: @@ -743,5 +741,5 @@ def event_stream(self): if not 'eventType' in event_data: raise MarathonError('Invalid event data received.') yield ef.process(event_data) - except ValueError as e: + except ValueError: raise MarathonError('Invalid event data received.') From f0354e363d7d6c476ac89b93233547a1689f04bc Mon Sep 17 00:00:00 2001 From: Miguel Elias dos Santos Date: Mon, 22 Aug 2016 21:56:20 -0300 Subject: [PATCH 130/292] PEP8 fix --- marathon/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marathon/client.py b/marathon/client.py index 1f096da..c8749a3 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -738,7 +738,7 @@ def event_stream(self): if _data[0] == 'data': event_data = json.loads(_data[1].strip()) - if not 'eventType' in event_data: + if 'eventType' not in event_data: raise MarathonError('Invalid event data received.') yield ef.process(event_data) except ValueError: From c2ca3229ad6dc1a33558373201a4892a2ef34018 Mon Sep 17 00:00:00 2001 From: Miguel Elias dos Santos Date: Mon, 22 Aug 2016 22:04:12 -0300 Subject: [PATCH 131/292] PEP8 fix --- marathon/client.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index c8749a3..420881d 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -95,17 +95,16 @@ def _do_request(self, method, path, params=None, data=None): return response - def _do_sse_request(self, path, params=None, data=None): - servers = list(self.servers) - - while servers and messages is None: + def _do_sse_request(self, path): + while list(self.servers) is None: server = servers.pop(0) url = ''.join([server.rstrip('/'), path]) try: response = requests.get( url, stream=True, - headers={'Accept': 'text/event-stream'} + headers={'Accept': 'text/event-stream'}, + auth=self.auth ) except Exception as e: marathon.log.error('Error while calling %s: %s', url, e.message) From e43f61e7ddfd39e0fca9d2e71cdddc2b4c14c1a7 Mon Sep 17 00:00:00 2001 From: Miguel Elias dos Santos Date: Mon, 22 Aug 2016 22:08:02 -0300 Subject: [PATCH 132/292] Code cleanup --- marathon/client.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index 420881d..36a799e 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -96,8 +96,7 @@ def _do_request(self, method, path, params=None, data=None): return response def _do_sse_request(self, path): - while list(self.servers) is None: - server = servers.pop(0) + for server in list(self.servers): url = ''.join([server.rstrip('/'), path]) try: response = requests.get( From 1143b7b4b0cb6726950f4e32ff1fab506ff78970 Mon Sep 17 00:00:00 2001 From: Miguel E dos Santos Date: Thu, 25 Aug 2016 12:18:52 -0300 Subject: [PATCH 133/292] Fixing _do_sse_request change of behaviour --- marathon/client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index 36a799e..0a96c6c 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -108,10 +108,10 @@ def _do_sse_request(self, path): except Exception as e: marathon.log.error('Error while calling %s: %s', url, e.message) - if not response.ok: - raise MarathonError('No remaining Marathon servers to try') + if esponse.ok: + return response.iter_lines() - return response.iter_lines() + raise MarathonError('No remaining Marathon servers to try') def list_endpoints(self): """List the current endpoints for all applications From 7f9fe9caccc7ea407f41d43c4e04a536ea06e2fb Mon Sep 17 00:00:00 2001 From: Miguel E dos Santos Date: Thu, 25 Aug 2016 12:25:23 -0300 Subject: [PATCH 134/292] Fixed variable typo --- marathon/client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/marathon/client.py b/marathon/client.py index 0a96c6c..dc855ad 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -96,6 +96,7 @@ def _do_request(self, method, path, params=None, data=None): return response def _do_sse_request(self, path): + """Query Marathon server for events.""" for server in list(self.servers): url = ''.join([server.rstrip('/'), path]) try: @@ -108,7 +109,7 @@ def _do_sse_request(self, path): except Exception as e: marathon.log.error('Error while calling %s: %s', url, e.message) - if esponse.ok: + if response.ok: return response.iter_lines() raise MarathonError('No remaining Marathon servers to try') From aa1108497c23a1c9540e41aedfb785da89851505 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Mon, 29 Aug 2016 09:47:46 -0700 Subject: [PATCH 135/292] Release 0.8.6 --- CHANGELOG.md | 17 +++++++++++++++++ setup.py | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7a24f6..52c4e75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ # Change Log +## [0.8.6](https://github.com/thefactory/marathon-python/tree/0.8.6) (2016-08-29) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.5...0.8.6) + +**Closed issues:** + +- Unexpected keyword argument: gpus [\#140](https://github.com/thefactory/marathon-python/issues/140) +- \[Profiling\] Humongous CPU with event\_stream [\#139](https://github.com/thefactory/marathon-python/issues/139) +- Python 3 test not running [\#80](https://github.com/thefactory/marathon-python/issues/80) + +**Merged pull requests:** + +- Add NONE as valid docker network mode [\#144](https://github.com/thefactory/marathon-python/pull/144) ([fengyehong](https://github.com/fengyehong)) +- Removed sseclient dependency + major enhancements on event\_stream\(\) [\#143](https://github.com/thefactory/marathon-python/pull/143) ([migueleliasweb](https://github.com/migueleliasweb)) +- run tox against multiple python versions [\#142](https://github.com/thefactory/marathon-python/pull/142) ([Rob-Johnson](https://github.com/Rob-Johnson)) +- Fix \#140 - Resolve gpus TypeError [\#141](https://github.com/thefactory/marathon-python/pull/141) ([mbeacom](https://github.com/mbeacom)) +- Add support for unhealthy\_task\_kill\_event [\#137](https://github.com/thefactory/marathon-python/pull/137) ([nuclon](https://github.com/nuclon)) + ## [0.8.5](https://github.com/thefactory/marathon-python/tree/0.8.5) (2016-08-10) [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.4...0.8.5) diff --git a/setup.py b/setup.py index 3bcd8b2..1ace3de 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='marathon', - version='0.8.5', + version='0.8.6', description='Marathon Client Library', long_description="""Python interface to the Mesos Marathon REST API.""", author='Mike Babineau', From 50b0c7794698734b22e649f2b830d4a7a32cf93a Mon Sep 17 00:00:00 2001 From: Drew Robb Date: Thu, 15 Sep 2016 16:07:57 -0700 Subject: [PATCH 136/292] Add external volume support --- marathon/models/container.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/marathon/models/container.py b/marathon/models/container.py index 0ee0f70..286730e 100644 --- a/marathon/models/container.py +++ b/marathon/models/container.py @@ -104,14 +104,16 @@ class MarathonContainerVolume(MarathonObject): :param str host_path: host path :param str mode: one of ['RO', 'RW'] :param object persistent: persistent volume options, should be of the form {'size': 1000} + :param object external: external volume options """ MODES = ['RO', 'RW'] - def __init__(self, container_path=None, host_path=None, mode='RW', persistent=None): + def __init__(self, container_path=None, host_path=None, mode='RW', persistent=None, external=None): self.container_path = container_path self.host_path = host_path if mode not in self.MODES: raise InvalidChoiceError('mode', mode, self.MODES) self.mode = mode self.persistent = persistent + self.external = external From 6c8f71af105b85dc59e325c04354cc0d6a75c19e Mon Sep 17 00:00:00 2001 From: Nathan Handler Date: Mon, 19 Sep 2016 14:10:05 -0700 Subject: [PATCH 137/292] Add support for marathon 1.3.0, which requires mesos 1.0.0 --- .travis.yml | 1 + itests/install-marathon.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d405dab..6ffcd00 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ env: - MARATHONVERSION: 0.14.1 - MARATHONVERSION: 0.15.3 - MARATHONVERSION: 1.1.2 + - MARATHONVERSION: 1.3.0 language: python python: diff --git a/itests/install-marathon.sh b/itests/install-marathon.sh index 05bd432..8425b66 100755 --- a/itests/install-marathon.sh +++ b/itests/install-marathon.sh @@ -21,7 +21,7 @@ sudo apt-get -y purge oracle-java7-installer sudo update-java-alternatives -s java-8-oracle sudo DEBIAN_FRONTEND=noninteractive apt-get install oracle-java8-set-default -sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes install mesos=0.28.* marathon=$MARATHONVERSION* +sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes install mesos=1.0.* marathon=$MARATHONVERSION* # WTF MARATHON? # Why does the precise version have java7 hardcoded if it requires java8? From e79821ef0bdb8cc79046b1fcb9cb109643ca2585 Mon Sep 17 00:00:00 2001 From: Nathan Handler Date: Mon, 19 Sep 2016 15:34:54 -0700 Subject: [PATCH 138/292] Test newest marathon first in travis --- .travis.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6ffcd00..5a8060b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,11 @@ env: - - MARATHONVERSION: 0.10.1 - - MARATHONVERSION: 0.11.1 - - MARATHONVERSION: 0.13.1 - - MARATHONVERSION: 0.14.1 - - MARATHONVERSION: 0.15.3 - - MARATHONVERSION: 1.1.2 - MARATHONVERSION: 1.3.0 + - MARATHONVERSION: 1.1.2 + - MARATHONVERSION: 0.15.3 + - MARATHONVERSION: 0.14.1 + - MARATHONVERSION: 0.13.1 + - MARATHONVERSION: 0.11.1 + - MARATHONVERSION: 0.10.1 language: python python: From 25b0b488469df04c5de4bc9c84bb3870830f9041 Mon Sep 17 00:00:00 2001 From: Nathan Handler Date: Mon, 19 Sep 2016 15:35:23 -0700 Subject: [PATCH 139/292] Make Makefile run tox environments independently --- Makefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 89f45ed..6c3bbe8 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,11 @@ itests: - tox -e itest-py27,itest-py33 + tox -e itest-py27 + tox -e itest-py33 test: - tox -e pep8,test-py27,test-py33 + tox -e pep8 + tox -e test-py27 + tox -e test-py33 clean: rm -rf dist/ build/ From ff402ae3e9d2e8ce0dae0428ca82c3edff63d5ef Mon Sep 17 00:00:00 2001 From: Nathan Handler Date: Mon, 19 Sep 2016 15:51:59 -0700 Subject: [PATCH 140/292] set -e in itest.sh to make it return an error on test failure --- itests/itest.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/itests/itest.sh b/itests/itest.sh index 7a7ba4f..bbc8b81 100755 --- a/itests/itest.sh +++ b/itests/itest.sh @@ -1,4 +1,7 @@ #!/bin/bash + +set -e + [[ -n $TRAVIS ]] || echo MARATHONVERSION=$MARATHONVERSION > marathon-version [[ -n $TRAVIS ]] || docker-compose build [[ -n $TRAVIS ]] || docker-compose pull From 376e8cda46acaaf22b8c7b4df8ce8a64a97d0f4b Mon Sep 17 00:00:00 2001 From: Nathan Handler Date: Mon, 19 Sep 2016 17:07:29 -0700 Subject: [PATCH 141/292] Support buildref in /v2/info: https://github.com/mesosphere/marathon/pull/4020 --- marathon/models/info.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/marathon/models/info.py b/marathon/models/info.py index 872f8f2..7795585 100644 --- a/marathon/models/info.py +++ b/marathon/models/info.py @@ -21,10 +21,11 @@ class MarathonInfo(MarathonResource): :param event_subscriber: :type event_subscriber: :class`marathon.models.info.MarathonEventSubscriber` or dict :param bool elected: + :param str buildref: """ def __init__(self, event_subscriber=None, framework_id=None, http_config=None, leader=None, marathon_config=None, - name=None, version=None, elected=None, zookeeper_config=None): + name=None, version=None, elected=None, zookeeper_config=None, buildref=None): if isinstance(event_subscriber, MarathonEventSubscriber): self.event_subscriber = event_subscriber elif event_subscriber is not None: @@ -43,6 +44,7 @@ def __init__(self, event_subscriber=None, framework_id=None, http_config=None, l self.elected = elected self.zookeeper_config = zookeeper_config if isinstance(zookeeper_config, MarathonZooKeeperConfig) \ else MarathonZooKeeperConfig().from_json(zookeeper_config) + self.buildref = buildref class MarathonConfig(MarathonObject): From 3050a2afff9e4415d34e44ec5fb648c0233e3bed Mon Sep 17 00:00:00 2001 From: Nathan Handler Date: Mon, 19 Sep 2016 17:25:02 -0700 Subject: [PATCH 142/292] Relax itest --- itests/steps/marathon_steps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/itests/steps/marathon_steps.py b/itests/steps/marathon_steps.py index 9c460a2..355bd9b 100644 --- a/itests/steps/marathon_steps.py +++ b/itests/steps/marathon_steps.py @@ -158,7 +158,7 @@ def stop_listening_stream(context): # and 2 deployment_step_success events filtered_events = [e for e in context.events if e.event_type == "deployment_success"] - assert len(filtered_events) == 2 + assert len(filtered_events) >= 2 @then('we should be able to see a deployment') From e70a264f90228042b9a5b230514709fbc8d3730e Mon Sep 17 00:00:00 2001 From: Nathan Handler Date: Tue, 20 Sep 2016 10:43:54 -0700 Subject: [PATCH 143/292] Revert "Fix test" This reverts commit efc8802f4fd2f7b697aab2936afd91dc25cd0f4e. --- itests/steps/marathon_steps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/itests/steps/marathon_steps.py b/itests/steps/marathon_steps.py index 355bd9b..e2bfdb8 100644 --- a/itests/steps/marathon_steps.py +++ b/itests/steps/marathon_steps.py @@ -158,7 +158,7 @@ def stop_listening_stream(context): # and 2 deployment_step_success events filtered_events = [e for e in context.events if e.event_type == "deployment_success"] - assert len(filtered_events) >= 2 + assert len(filtered_events) == 1 @then('we should be able to see a deployment') From cf7fa8c54061e13c9267c31327b252954e02adae Mon Sep 17 00:00:00 2001 From: Nathan Handler Date: Tue, 20 Sep 2016 11:35:22 -0700 Subject: [PATCH 144/292] Add some debug logic to the assert --- itests/steps/marathon_steps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/itests/steps/marathon_steps.py b/itests/steps/marathon_steps.py index e2bfdb8..13f0964 100644 --- a/itests/steps/marathon_steps.py +++ b/itests/steps/marathon_steps.py @@ -158,7 +158,7 @@ def stop_listening_stream(context): # and 2 deployment_step_success events filtered_events = [e for e in context.events if e.event_type == "deployment_success"] - assert len(filtered_events) == 1 + assert len(filtered_events) == 1, "We had %d filtered_events: %s" % (len(filtered_events), filtered_events) @then('we should be able to see a deployment') From 2714c7b99fb3f9543a887f76f746245d6add5b93 Mon Sep 17 00:00:00 2001 From: Nathan Handler Date: Tue, 20 Sep 2016 12:48:51 -0700 Subject: [PATCH 145/292] Simply test that we got >= 1 raw event to avoid flake --- itests/steps/marathon_steps.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/itests/steps/marathon_steps.py b/itests/steps/marathon_steps.py index 13f0964..59ab187 100644 --- a/itests/steps/marathon_steps.py +++ b/itests/steps/marathon_steps.py @@ -145,20 +145,7 @@ def start_listening_stream(context): def stop_listening_stream(context): time.sleep(10) context.p.terminate() - - print(context.events) - - # event list should contain 5 status_update_event with taskStatus == TASK_RUNNING - filtered_events = [e for e in context.events if e.event_type == "status_update_event" and e.task_status == "TASK_RUNNING"] - assert len(filtered_events) == 5 - - # and 1 status_update_event with taskStatus == TASK_KILLED - filtered_events = [e for e in context.events if e.event_type == "status_update_event" and e.task_status == "TASK_KILLED"] - assert len(filtered_events) == 1 - - # and 2 deployment_step_success events - filtered_events = [e for e in context.events if e.event_type == "deployment_success"] - assert len(filtered_events) == 1, "We had %d filtered_events: %s" % (len(filtered_events), filtered_events) + assert len(context.events) >= 1, "We had %d events: %s" % (len(context.events), context.events) @then('we should be able to see a deployment') From bca3778ba51e94ddd74a119c2e6a82eb0990b413 Mon Sep 17 00:00:00 2001 From: Nathan Handler Date: Tue, 20 Sep 2016 12:55:47 -0700 Subject: [PATCH 146/292] More debugging --- itests/steps/marathon_steps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/itests/steps/marathon_steps.py b/itests/steps/marathon_steps.py index 59ab187..cab50ee 100644 --- a/itests/steps/marathon_steps.py +++ b/itests/steps/marathon_steps.py @@ -150,4 +150,4 @@ def stop_listening_stream(context): @then('we should be able to see a deployment') def see_a_deployment(context): - assert len(context.client.list_deployments()) == 1 + assert len(context.client.list_deployments()) == 1, "We had %d deployments: %s" % (len(context.client.list_deployments()), context.client.list_deployments()) From e1671ebb12a8f8cb50e6a67ab7bf77a7d79bde1d Mon Sep 17 00:00:00 2001 From: Nathan Handler Date: Tue, 20 Sep 2016 13:05:36 -0700 Subject: [PATCH 147/292] Shorten line for pep8 --- itests/steps/marathon_steps.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/itests/steps/marathon_steps.py b/itests/steps/marathon_steps.py index cab50ee..331b72c 100644 --- a/itests/steps/marathon_steps.py +++ b/itests/steps/marathon_steps.py @@ -150,4 +150,5 @@ def stop_listening_stream(context): @then('we should be able to see a deployment') def see_a_deployment(context): - assert len(context.client.list_deployments()) == 1, "We had %d deployments: %s" % (len(context.client.list_deployments()), context.client.list_deployments()) + deployments = context.client.list_deployments() + assert len(deployments) == 1, "We had %d deployments: %s" % (len(deployments), deployments) From 659995f473a4f744fa4e5c59ac2145306e1d2bfb Mon Sep 17 00:00:00 2001 From: Greg Hill Date: Thu, 22 Sep 2016 11:05:55 -0500 Subject: [PATCH 148/292] Add support for token-based Auth DCOS has an `adminrouter` to control public access to the Marathon API. This requires token-based authentication. This simple change lets a user pass in an `auth_token` rather than `username` and `password` for this use-case. Fixes #148 --- marathon/client.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/marathon/client.py b/marathon/client.py index dc855ad..d47505e 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -20,7 +20,8 @@ class MarathonClient(object): """Client interface for the Marathon REST API.""" - def __init__(self, servers, username=None, password=None, timeout=10, session=None): + def __init__(self, servers, username=None, password=None, timeout=10, session=None, + auth_token=None): """Create a MarathonClient instance. If multiple servers are specified, each will be tried in succession until a non-"Connection Error"-type @@ -32,6 +33,7 @@ def __init__(self, servers, username=None, password=None, timeout=10, session=No :param str username: Basic auth username :param str password: Basic auth password :param int timeout: Timeout (in seconds) for requests to Marathon + :param str auth_token: Token-based auth token, used with DCOS + Oauth """ if session is None: self.session = requests.Session() @@ -41,6 +43,11 @@ def __init__(self, servers, username=None, password=None, timeout=10, session=No self.auth = (username, password) if username and password else None self.timeout = timeout + self.auth_token = auth_token + if self.auth and self.auth_token: + raise ValueError("Can't specify both auth token and username/password. Must select " + "one type of authentication.") + def __repr__(self): return 'Connection:%s' % self.servers @@ -58,6 +65,10 @@ def _do_request(self, method, path, params=None, data=None): """Query Marathon server.""" headers = { 'Content-Type': 'application/json', 'Accept': 'application/json'} + + if self.auth_token: + headers['Authorization'] = "token={}".format(self.auth_token) + response = None servers = list(self.servers) while servers and response is None: From 2873c53b8ed318dc3f5f3d9fdc810581daa7a735 Mon Sep 17 00:00:00 2001 From: Matthieu Melcot Date: Wed, 12 Oct 2016 17:01:19 +0200 Subject: [PATCH 149/292] Add missing 'timeout_seconds' parameter in ReadinessCheck class --- marathon/models/app.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/marathon/models/app.py b/marathon/models/app.py index 659423b..d1f5896 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -391,7 +391,7 @@ class ReadinessCheck(MarathonObject): """ def __init__(self, name=None, protocol=None, path=None, port_name=None, interval_seconds=None, - http_status_codes_for_ready=None, preserve_last_response=None): + http_status_codes_for_ready=None, preserve_last_response=None, timeout_seconds=None): self.name = name self.protocol = protocol self.path = path @@ -399,6 +399,7 @@ def __init__(self, name=None, protocol=None, path=None, port_name=None, interval self.interval_seconds = interval_seconds self.http_status_codes_for_ready = http_status_codes_for_ready self.preserve_last_response = preserve_last_response + self.timeout_seconds = timeout_seconds class PortDefinition(MarathonObject): From 5edf51e9973778a532e0258a48bbdfeb43291dac Mon Sep 17 00:00:00 2001 From: Tim Anderegg Date: Wed, 19 Oct 2016 13:41:47 -0400 Subject: [PATCH 150/292] Add support for MESOS container type --- marathon/models/container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marathon/models/container.py b/marathon/models/container.py index 286730e..ad254db 100644 --- a/marathon/models/container.py +++ b/marathon/models/container.py @@ -15,7 +15,7 @@ class MarathonContainer(MarathonObject): :type volumes: list[:class:`marathon.models.container.MarathonContainerVolume`] or list[dict] """ - TYPES = ['DOCKER'] + TYPES = ['DOCKER', 'MESOS'] """Valid container types""" def __init__(self, docker=None, type='DOCKER', volumes=None): From e91ff0999f4cc54deb79ae52328f68f95e82705b Mon Sep 17 00:00:00 2001 From: Tim Anderegg Date: Wed, 19 Oct 2016 13:52:25 -0400 Subject: [PATCH 151/292] Check for docker==None --- marathon/models/container.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/marathon/models/container.py b/marathon/models/container.py index ad254db..17411e7 100644 --- a/marathon/models/container.py +++ b/marathon/models/container.py @@ -22,8 +22,11 @@ def __init__(self, docker=None, type='DOCKER', volumes=None): if type not in self.TYPES: raise InvalidChoiceError('type', type, self.TYPES) self.type = type - self.docker = docker if isinstance(docker, MarathonDockerContainer) \ - else MarathonDockerContainer().from_json(docker) + + if docker: + self.docker = docker if isinstance(docker, MarathonDockerContainer) \ + else MarathonDockerContainer().from_json(docker) + self.volumes = [ v if isinstance( v, MarathonContainerVolume) else MarathonContainerVolume().from_json(v) From 9f2479dfe0ad97d84ebab3cdac492f08bdfd357e Mon Sep 17 00:00:00 2001 From: Tim Anderegg Date: Wed, 19 Oct 2016 15:27:20 -0400 Subject: [PATCH 152/292] Removed whitespace to comply with pep8 --- marathon/models/container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marathon/models/container.py b/marathon/models/container.py index 17411e7..bb9520b 100644 --- a/marathon/models/container.py +++ b/marathon/models/container.py @@ -22,7 +22,7 @@ def __init__(self, docker=None, type='DOCKER', volumes=None): if type not in self.TYPES: raise InvalidChoiceError('type', type, self.TYPES) self.type = type - + if docker: self.docker = docker if isinstance(docker, MarathonDockerContainer) \ else MarathonDockerContainer().from_json(docker) From 5a3eaf1a5916acce3d01af6632065baf01d04a5f Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Fri, 21 Oct 2016 16:08:17 -0700 Subject: [PATCH 153/292] Added 1.4 to the travis matrix --- .travis.yml | 1 + itests/install-marathon.sh | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5a8060b..5eccbc9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,5 @@ env: + - MARATHONVERSION: 1.4.0 - MARATHONVERSION: 1.3.0 - MARATHONVERSION: 1.1.2 - MARATHONVERSION: 0.15.3 diff --git a/itests/install-marathon.sh b/itests/install-marathon.sh index 8425b66..ad0c608 100755 --- a/itests/install-marathon.sh +++ b/itests/install-marathon.sh @@ -11,9 +11,10 @@ DISTRO=$(lsb_release -is | tr '[:upper:]' '[:lower:]') CODENAME=$(lsb_release -cs) # Add the repository -echo "deb http://repos.mesosphere.com/${DISTRO} ${CODENAME} main" | - sudo tee /etc/apt/sources.list.d/mesosphere.list -sudo apt-get update +echo "deb http://repos.mesosphere.com/${DISTRO} ${CODENAME} main" | sudo tee /etc/apt/sources.list.d/mesosphere.list +# Temporary repo to get marathon 1.4 +echo "deb https://dl.bintray.com/yelp/paasta trusty main" | sudo tee /etc/apt/sources.list.d/paasta.list +sudo apt-get -y install apt-transport-https && sudo apt-get update # Install packages sudo DEBIAN_FRONTEND=noninteractive apt-get -y install oracle-java8-installer From 50b777ad95daf0567c12200f2f6238b0e639f6eb Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Fri, 21 Oct 2016 16:53:02 -0700 Subject: [PATCH 154/292] Added preliminary Marathon 1.4 support --- itests/install-marathon.sh | 2 +- itests/steps/marathon_steps.py | 4 ++-- marathon/models/deployment.py | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/itests/install-marathon.sh b/itests/install-marathon.sh index ad0c608..d1532d6 100755 --- a/itests/install-marathon.sh +++ b/itests/install-marathon.sh @@ -3,7 +3,7 @@ set -vxeu # Default version of marathon to test against if not set by the user [[ -f /root/marathon-version ]] && source /root/marathon-version -MARATHONVERSION="${MARATHONVERSION:-0.8.2}" +MARATHONVERSION="${MARATHONVERSION:-1.4.0}" # Setup sudo apt-key adv --keyserver keyserver.ubuntu.com --recv 81026D0004C44CF7EF55ADF8DF7D54CBE56151BF diff --git a/itests/steps/marathon_steps.py b/itests/steps/marathon_steps.py index 331b72c..afc4c12 100644 --- a/itests/steps/marathon_steps.py +++ b/itests/steps/marathon_steps.py @@ -1,7 +1,7 @@ import sys import time import multiprocessing -from distutils.version import StrictVersion +from distutils.version import LooseVersion import marathon from behave import given, when, then @@ -126,7 +126,7 @@ def listen_for_events(client, events): @when(u'marathon version is greater than {version}') def marathon_version_chech(context, version): info = context.client.get_info() - if StrictVersion(info.version) < StrictVersion(version): + if LooseVersion(info.version) < LooseVersion(version): context.scenario.skip(reason='Marathon version is too low for this scenario') diff --git a/marathon/models/deployment.py b/marathon/models/deployment.py index b93bf40..929847b 100644 --- a/marathon/models/deployment.py +++ b/marathon/models/deployment.py @@ -17,10 +17,11 @@ class MarathonDeployment(MarathonResource): :type steps: list[:class:`marathon.models.deployment.MarathonDeploymentAction`] or list[dict] :param int total_steps: total number of steps :param str version: version id + :param str affected_pods: list of strings """ def __init__(self, affected_apps=None, current_actions=None, current_step=None, id=None, steps=None, - total_steps=None, version=None): + total_steps=None, version=None, affected_pods=None): self.affected_apps = affected_apps self.current_actions = [ a if isinstance( @@ -32,6 +33,7 @@ def __init__(self, affected_apps=None, current_actions=None, current_step=None, self.steps = [self.parse_deployment_step(step) for step in (steps or [])] self.total_steps = total_steps self.version = version + self.affected_pods = affected_pods def parse_deployment_step(self, step): if step.__class__ == dict: From 776c0264017d47fc52b4befe50406789d2b69df9 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Mon, 24 Oct 2016 11:21:51 -0700 Subject: [PATCH 155/292] Release 0.8.7 --- CHANGELOG.md | 17 +++++++++++++++++ README.md | 3 +-- setup.py | 2 +- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 52c4e75..6f55543 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ # Change Log +## [0.8.7](https://github.com/thefactory/marathon-python/tree/0.8.7) (2016-10-24) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.6...0.8.7) + +**Closed issues:** + +- Need to support Oauth tokens for use with DCOS + adminrouter [\#148](https://github.com/thefactory/marathon-python/issues/148) +- Not yet compatible with marathon 1.1.1 external volumes [\#98](https://github.com/thefactory/marathon-python/issues/98) + +**Merged pull requests:** + +- Preliminary Marathon 1.4 Support [\#155](https://github.com/thefactory/marathon-python/pull/155) ([solarkennedy](https://github.com/solarkennedy)) +- Mesos container support [\#154](https://github.com/thefactory/marathon-python/pull/154) ([tanderegg](https://github.com/tanderegg)) +- Add missing 'timeout\_seconds' parameter in ReadinessCheck class [\#152](https://github.com/thefactory/marathon-python/pull/152) ([mmelcot](https://github.com/mmelcot)) +- Add support for token-based Auth [\#149](https://github.com/thefactory/marathon-python/pull/149) ([jimbobhickville](https://github.com/jimbobhickville)) +- \[WIP\] Add support for marathon 1.3.0 [\#147](https://github.com/thefactory/marathon-python/pull/147) ([nhandler](https://github.com/nhandler)) +- Add external volume support [\#146](https://github.com/thefactory/marathon-python/pull/146) ([drewrobb](https://github.com/drewrobb)) + ## [0.8.6](https://github.com/thefactory/marathon-python/tree/0.8.6) (2016-08-29) [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.5...0.8.6) diff --git a/README.md b/README.md index 3470cf7..4b99c96 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ This is a Python library for interfacing with [Marathon](https://github.com/meso #### Compatibility +* For Marathon 1.4.0, use at least 0.8.7 * For Marathon 1.1.1 and 0.15.x, use at least 0.8.1 * For Marathon 0.14.x, use at least 0.7.6 * For Marathon 0.8.x-0.11.x, use at least marathon-python 0.7.5 @@ -13,8 +14,6 @@ This is a Python library for interfacing with [Marathon](https://github.com/meso * For Marathon 0.7.x, use at least marathon-python 0.6.10 * For all version changes, please see `CHANGELOG.md` -Note: Not all versions of Python are tested against every version of Marathon. - If you find a feature that is broken, please submit a PR that adds a test for it so it will be fixed and will continue to stay fixed as Marathon changes over time. diff --git a/setup.py b/setup.py index 1ace3de..f91239d 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='marathon', - version='0.8.6', + version='0.8.7', description='Marathon Client Library', long_description="""Python interface to the Mesos Marathon REST API.""", author='Mike Babineau', From 2bfb89cbc5184456f4f8d73d5932697d15dd48f1 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Mon, 31 Oct 2016 20:15:31 -0700 Subject: [PATCH 156/292] Added a MANIFEST file to include the LICENSE file. Fixes #156 --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..1aba38f --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include LICENSE From c37174833529f11922cb07d1e7a4b835a0abe4cb Mon Sep 17 00:00:00 2001 From: Kevin Mooney Date: Tue, 15 Nov 2016 13:14:52 -0600 Subject: [PATCH 157/292] Expose error details from response object MarathonHttpError --- marathon/exceptions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/marathon/exceptions.py b/marathon/exceptions.py index e5cf597..4dcff8d 100644 --- a/marathon/exceptions.py +++ b/marathon/exceptions.py @@ -12,6 +12,7 @@ def __init__(self, response): if response.content: content = response.json() self.error_message = content.get('message', self.error_message) + self.error_details = content.get('details') self.status_code = response.status_code super(MarathonHttpError, self).__init__(self.__str__()) From cbdca5b2507ff54a638222ca04ae9e3d253664f2 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Tue, 15 Nov 2016 18:01:21 -0800 Subject: [PATCH 158/292] Minor pep8 compliance fixes --- marathon/models/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/marathon/models/base.py b/marathon/models/base.py index 719b35d..b4abd6e 100644 --- a/marathon/models/base.py +++ b/marathon/models/base.py @@ -63,6 +63,7 @@ def __eq__(self, other): def __str__(self): return "{clazz}::".format(clazz=self.__class__.__name__) + str(self.__dict__) + # See: # https://github.com/mesosphere/marathon/blob/2a9d1d20ec2f1cfcc49fbb1c0e7348b26418ef38/src/main/scala/mesosphere/marathon/api/ModelValidation.scala#L224 ID_PATTERN = re.compile( From 722145a6dc0fd777557f49562b05fab0c0a5d8cc Mon Sep 17 00:00:00 2001 From: Djailla Date: Tue, 6 Dec 2016 17:52:42 +0100 Subject: [PATCH 159/292] Allow to disable SSL certificate validation Add an option to disable SSL certificate verification. When certificate are auto signed it will create some issues. --- marathon/client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index d47505e..a57256d 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -21,7 +21,7 @@ class MarathonClient(object): """Client interface for the Marathon REST API.""" def __init__(self, servers, username=None, password=None, timeout=10, session=None, - auth_token=None): + auth_token=None, verify=True): """Create a MarathonClient instance. If multiple servers are specified, each will be tried in succession until a non-"Connection Error"-type @@ -34,6 +34,7 @@ def __init__(self, servers, username=None, password=None, timeout=10, session=No :param str password: Basic auth password :param int timeout: Timeout (in seconds) for requests to Marathon :param str auth_token: Token-based auth token, used with DCOS + Oauth + :param bool verify: Enable SSL certificate verification """ if session is None: self.session = requests.Session() @@ -41,6 +42,7 @@ def __init__(self, servers, username=None, password=None, timeout=10, session=No self.session = session self.servers = servers if isinstance(servers, list) else [servers] self.auth = (username, password) if username and password else None + self.verify = verify self.timeout = timeout self.auth_token = auth_token @@ -77,7 +79,7 @@ def _do_request(self, method, path, params=None, data=None): try: response = self.session.request( method, url, params=params, data=data, headers=headers, - auth=self.auth, timeout=self.timeout) + auth=self.auth, timeout=self.timeout, verify=self.verify) marathon.log.info('Got response from %s', server) except requests.exceptions.RequestException as e: marathon.log.error( From fc51c852e6eb539f8c7ac1dceb2a7508f9a44599 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Fri, 9 Dec 2016 07:55:58 -0800 Subject: [PATCH 160/292] Release 0.8.8 --- CHANGELOG.md | 12 ++++++++++++ setup.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f55543..c408740 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Change Log +## [0.8.8](https://github.com/thefactory/marathon-python/tree/0.8.8) (2016-12-09) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.7...0.8.8) + +**Closed issues:** + +- Can we include the license file in the source release tarball ? [\#156](https://github.com/thefactory/marathon-python/issues/156) + +**Merged pull requests:** + +- Allow to disable SSL certificate validation [\#159](https://github.com/thefactory/marathon-python/pull/159) ([Djailla](https://github.com/Djailla)) +- Expose error details from response object MarathonHttpError [\#157](https://github.com/thefactory/marathon-python/pull/157) ([moonkev](https://github.com/moonkev)) + ## [0.8.7](https://github.com/thefactory/marathon-python/tree/0.8.7) (2016-10-24) [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.6...0.8.7) diff --git a/setup.py b/setup.py index f91239d..caa3c2c 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='marathon', - version='0.8.7', + version='0.8.8', description='Marathon Client Library', long_description="""Python interface to the Mesos Marathon REST API.""", author='Mike Babineau', From 5bd7fcaf667c19e30cde6f07b4a3441246330a80 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Wed, 14 Dec 2016 14:37:11 -0800 Subject: [PATCH 161/292] Added more unimplemented Marathon 1.4 API keywords --- marathon/models/app.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/marathon/models/app.py b/marathon/models/app.py index d1f5896..c949e4c 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -64,6 +64,8 @@ class MarathonApp(MarathonResource): :type readiness_checks: list[:class:`marathon.models.app.ReadinessCheck`] or list[dict] :type residency: :class:`marathon.models.app.Residency` or dict :param int task_kill_grace_period_seconds: Configures the termination signal escalation behavior of executors when stopping tasks. + :param str kill_selection + :param dict unreachable_strategy """ UPDATE_OK_ATTRIBUTES = [ @@ -88,7 +90,8 @@ def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, task_kill_grace_period_seconds=None, tasks_unhealthy=None, upgrade_strategy=None, uris=None, user=None, version=None, version_info=None, ip_address=None, fetch=None, task_stats=None, readiness_checks=None, - readiness_check_results=None, secrets=None, port_definitions=None, residency=None, gpus=None): + readiness_check_results=None, secrets=None, port_definitions=None, residency=None, gpus=None, + kill_selection=None, unreachable_strategy=None): # self.args = args or [] self.accepted_resource_roles = accepted_resource_roles @@ -124,6 +127,7 @@ def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, ] self.id = assert_valid_path(id) self.instances = instances + self.kill_selection = kill_selection self.labels = labels or {} self.last_task_failure = last_task_failure if (isinstance(last_task_failure, MarathonTaskFailure) or last_task_failure is None) \ else MarathonTaskFailure.from_json(last_task_failure) @@ -165,6 +169,7 @@ def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, self.uris = uris or [] self.fetch = fetch or [] self.user = user + self.unreachable_strategy = unreachable_strategy self.version = version self.version_info = version_info if (isinstance(version_info, MarathonAppVersionInfo) or version_info is None) \ else MarathonAppVersionInfo.from_json(version_info) From 86f99fd9cb9be140f5ea7daad2dea82de12c3a8e Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Wed, 14 Dec 2016 18:03:25 -0800 Subject: [PATCH 162/292] Release 0.8.9 --- CHANGELOG.md | 11 +++++++++++ README.md | 2 +- setup.py | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c408740..f76dd91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Change Log +## [0.8.9](https://github.com/thefactory/marathon-python/tree/0.8.9) (2016-12-14) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.8...0.8.9) + +**Closed issues:** + +- 0.8.8 on PyPi [\#160](https://github.com/thefactory/marathon-python/issues/160) + +**Merged pull requests:** + +- Added more unimplemented Marathon 1.4 API keywords [\#161](https://github.com/thefactory/marathon-python/pull/161) ([solarkennedy](https://github.com/solarkennedy)) + ## [0.8.8](https://github.com/thefactory/marathon-python/tree/0.8.8) (2016-12-09) [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.7...0.8.8) diff --git a/README.md b/README.md index 4b99c96..6d9b168 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ This is a Python library for interfacing with [Marathon](https://github.com/meso #### Compatibility -* For Marathon 1.4.0, use at least 0.8.7 +* For Marathon 1.4.0, use at least 0.8.9 * For Marathon 1.1.1 and 0.15.x, use at least 0.8.1 * For Marathon 0.14.x, use at least 0.7.6 * For Marathon 0.8.x-0.11.x, use at least marathon-python 0.7.5 diff --git a/setup.py b/setup.py index caa3c2c..c95fccc 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='marathon', - version='0.8.8', + version='0.8.9', description='Marathon Client Library', long_description="""Python interface to the Mesos Marathon REST API.""", author='Mike Babineau', From f6af600e2ea9f87b1877470a1e54eb96c6d4c0ca Mon Sep 17 00:00:00 2001 From: stj Date: Thu, 15 Dec 2016 16:47:56 -0800 Subject: [PATCH 163/292] Add new Marathon 1.4 API keywords --- marathon/models/app.py | 59 ++++++++++++++++++++++++++++++++-------- marathon/models/group.py | 10 ++++++- marathon/models/task.py | 5 +++- 3 files changed, 60 insertions(+), 14 deletions(-) diff --git a/marathon/models/app.py b/marathon/models/app.py index c949e4c..0b336c1 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -1,5 +1,6 @@ from datetime import datetime +from ..exceptions import InvalidChoiceError from .base import MarathonResource, MarathonObject, assert_valid_path from .constraint import MarathonConstraint from .container import MarathonContainer @@ -64,14 +65,15 @@ class MarathonApp(MarathonResource): :type readiness_checks: list[:class:`marathon.models.app.ReadinessCheck`] or list[dict] :type residency: :class:`marathon.models.app.Residency` or dict :param int task_kill_grace_period_seconds: Configures the termination signal escalation behavior of executors when stopping tasks. - :param str kill_selection - :param dict unreachable_strategy + :param list[dict] unreachable_strategy: Handling for unreachable instances. + :param str kill_selection: Defines which instance should be killed first in case of e.g. rescaling. """ UPDATE_OK_ATTRIBUTES = [ 'args', 'backoff_factor', 'backoff_seconds', 'cmd', 'constraints', 'container', 'cpus', 'dependencies', 'disk', - 'env', 'executor', 'gpus', 'health_checks', 'instances', 'labels', 'max_launch_delay_seconds', 'mem', 'ports', - 'require_ports', 'store_urls', 'task_rate_limit', 'upgrade_strategy', 'uris', 'user', 'version' + 'env', 'executor', 'gpus', 'health_checks', 'instances', 'kill_selection', 'labels', 'max_launch_delay_seconds', + 'mem', 'ports', 'require_ports', 'store_urls', 'task_rate_limit', 'upgrade_strategy', 'unreachable_strategy', + 'uris', 'user', 'version' ] """List of attributes which may be updated/changed after app creation""" @@ -82,16 +84,17 @@ class MarathonApp(MarathonResource): 'deployments', 'tasks', 'tasks_running', 'tasks_staged', 'tasks_healthy', 'tasks_unhealthy'] """List of read-only attributes""" + KILL_SELECTIONS = ["YoungestFirst", "OldestFirst"] + def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, backoff_seconds=None, cmd=None, constraints=None, container=None, cpus=None, dependencies=None, deployments=None, disk=None, env=None, - executor=None, health_checks=None, id=None, instances=None, labels=None, last_task_failure=None, - max_launch_delay_seconds=None, mem=None, ports=None, require_ports=None, store_urls=None, - task_rate_limit=None, tasks=None, tasks_running=None, tasks_staged=None, tasks_healthy=None, - task_kill_grace_period_seconds=None, tasks_unhealthy=None, upgrade_strategy=None, - uris=None, user=None, version=None, version_info=None, + executor=None, health_checks=None, id=None, instances=None, kill_selection=None, labels=None, + last_task_failure=None, max_launch_delay_seconds=None, mem=None, ports=None, require_ports=None, + store_urls=None, task_rate_limit=None, tasks=None, tasks_running=None, tasks_staged=None, + tasks_healthy=None, task_kill_grace_period_seconds=None, tasks_unhealthy=None, upgrade_strategy=None, + unreachable_strategy=None, uris=None, user=None, version=None, version_info=None, ip_address=None, fetch=None, task_stats=None, readiness_checks=None, - readiness_check_results=None, secrets=None, port_definitions=None, residency=None, gpus=None, - kill_selection=None, unreachable_strategy=None): + readiness_check_results=None, secrets=None, port_definitions=None, residency=None, gpus=None): # self.args = args or [] self.accepted_resource_roles = accepted_resource_roles @@ -127,6 +130,9 @@ def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, ] self.id = assert_valid_path(id) self.instances = instances + if kill_selection and kill_selection not in self.KILL_SELECTIONS: + raise InvalidChoiceError( + 'kill_selection', kill_selection, self.KILL_SELECTIONS) self.kill_selection = kill_selection self.labels = labels or {} self.last_task_failure = last_task_failure if (isinstance(last_task_failure, MarathonTaskFailure) or last_task_failure is None) \ @@ -166,10 +172,13 @@ def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, self.tasks_unhealthy = tasks_unhealthy self.upgrade_strategy = upgrade_strategy if (isinstance(upgrade_strategy, MarathonUpgradeStrategy) or upgrade_strategy is None) \ else MarathonUpgradeStrategy.from_json(upgrade_strategy) + self.unreachable_strategy = unreachable_strategy \ + if (isinstance(unreachable_strategy, MarathonUnreachableStrategy) + or unreachable_strategy is None) \ + else MarathonUnreachableStrategy.from_json(unreachable_strategy) self.uris = uris or [] self.fetch = fetch or [] self.user = user - self.unreachable_strategy = unreachable_strategy self.version = version self.version_info = version_info if (isinstance(version_info, MarathonAppVersionInfo) or version_info is None) \ else MarathonAppVersionInfo.from_json(version_info) @@ -255,6 +264,32 @@ def __init__(self, maximum_over_capacity=None, self.minimum_health_capacity = minimum_health_capacity +class MarathonUnreachableStrategy(MarathonObject): + + """Marathon unreachable Strategy. + + Define handling for unreachable instances. Given + `unreachable_inactive_after_seconds = 60` and + `unreachable_expunge_after = 120`, an instance will be expunged if it has + been unreachable for more than 120 seconds or a second instance is started + if it has been unreachable for more than 60 seconds.", + + See https://mesosphere.github.io/marathon/docs/? + + :param int unreachable_inactive_after_seconds: time an instance is + unreachable for in seconds before marked as inactive. + :param int unreachable_expunge_after_seconds: time an instance is + unreachable for in seconds before expunged. + """ + + def __init__(self, unreachable_inactive_after_seconds=None, + unreachable_expunge_after_seconds=None): + self.unreachable_inactive_after_seconds = \ + unreachable_inactive_after_seconds + self.unreachable_expunge_after_seconds = \ + unreachable_expunge_after_seconds + + class MarathonAppVersionInfo(MarathonObject): """Marathon App version info. diff --git a/marathon/models/group.py b/marathon/models/group.py index 40cf6bf..c7c0339 100644 --- a/marathon/models/group.py +++ b/marathon/models/group.py @@ -14,11 +14,13 @@ class MarathonGroup(MarathonResource): :param groups: :type groups: list[:class:`marathon.models.group.MarathonGroup`] or list[dict] :param str id: + :param pods: + :type pods: list[:class:`marathon.models.pod.MarathonPod`] or list[dict] :param str version: """ def __init__(self, apps=None, dependencies=None, - groups=None, id=None, version=None): + groups=None, id=None, pods=None, version=None): self.apps = [ a if isinstance(a, MarathonApp) else MarathonApp().from_json(a) for a in (apps or []) @@ -28,5 +30,11 @@ def __init__(self, apps=None, dependencies=None, g if isinstance(g, MarathonGroup) else MarathonGroup().from_json(g) for g in (groups or []) ] + self.pods = [] + # ToDo: Create class MarathonPod + # self.pods = [ + # p if isinstance(p, MarathonPod) else MarathonPod().from_json(p) + # for p in (pods or []) + # ] self.id = assert_valid_id(id) self.version = version diff --git a/marathon/models/task.py b/marathon/models/task.py index fad8808..6b81f74 100644 --- a/marathon/models/task.py +++ b/marathon/models/task.py @@ -73,12 +73,14 @@ class MarathonHealthCheckResult(MarathonObject): :param str last_failure_cause: cause for last failure :param str last_success: last time when which healthcheck succeeded :param str task_id: task id + :param str instance_id: instance id """ DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ' def __init__(self, alive=None, consecutive_failures=None, first_success=None, - last_failure=None, last_success=None, task_id=None, last_failure_cause=None): + last_failure=None, last_success=None, task_id=None, + last_failure_cause=None, instance_id=None): self.alive = alive self.consecutive_failures = consecutive_failures self.first_success = first_success if (first_success is None or isinstance(first_success, datetime)) \ @@ -89,3 +91,4 @@ def __init__(self, alive=None, consecutive_failures=None, first_success=None, else datetime.strptime(last_success, self.DATETIME_FORMAT) self.task_id = task_id self.last_failure_cause = last_failure_cause + self.instance_id = instance_id From e0599b9df118710bf70220f67697cbefafd398ee Mon Sep 17 00:00:00 2001 From: Miguel E dos Santos Date: Wed, 4 Jan 2017 18:57:50 -0200 Subject: [PATCH 164/292] Removed unused sseclient depencency --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c95fccc..c0c9be3 100755 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ long_description="""Python interface to the Mesos Marathon REST API.""", author='Mike Babineau', author_email='michael.babineau@gmail.com', - install_requires=['requests>=2.0.0', 'sseclient'], + install_requires=['requests>=2.0.0'], url='https://github.com/thefactory/marathon-python', packages=['marathon', 'marathon.models'], license='MIT', From a45d9c7b6542e411caa08eac43eb47759d0e1f8a Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Fri, 6 Jan 2017 12:52:38 -0800 Subject: [PATCH 165/292] Use instance_id in task health failures for marathon 1.4 --- marathon/models/app.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/marathon/models/app.py b/marathon/models/app.py index 0b336c1..fb5737f 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -228,6 +228,7 @@ class MarathonTaskFailure(MarathonObject): :param str host: mesos slave running the task :param str message: error message :param str task_id: task id + :param str instance_id: instance id :param str state: task state :param timestamp: when this task failed :type timestamp: datetime or str @@ -236,12 +237,13 @@ class MarathonTaskFailure(MarathonObject): DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ' - def __init__(self, app_id=None, host=None, message=None, task_id=None, + def __init__(self, app_id=None, host=None, message=None, task_id=None, instance_id=None, slave_id=None, state=None, timestamp=None, version=None): self.app_id = app_id self.host = host self.message = message self.task_id = task_id + self.instance_id = instance_id self.slave_id = slave_id self.state = state self.timestamp = timestamp if (timestamp is None or isinstance(timestamp, datetime)) \ From 410dfa93718fcf643ca3563ef5417fc050670779 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Fri, 6 Jan 2017 15:15:29 -0800 Subject: [PATCH 166/292] Added more API settings for MarathonUnreachableStrategy new in marathon 1.4 --- itests/Dockerfile | 2 +- marathon/models/app.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/itests/Dockerfile b/itests/Dockerfile index 8077318..c43b783 100644 --- a/itests/Dockerfile +++ b/itests/Dockerfile @@ -5,7 +5,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install \ RUN add-apt-repository ppa:webupd8team/java RUN echo "debconf shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections RUN echo "debconf shared/accepted-oracle-license-v1-1 seen true" | debconf-set-selections -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install \ +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y -q install \ lsb-release \ oracle-java8-installer diff --git a/marathon/models/app.py b/marathon/models/app.py index fb5737f..366e958 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -282,14 +282,17 @@ class MarathonUnreachableStrategy(MarathonObject): unreachable for in seconds before marked as inactive. :param int unreachable_expunge_after_seconds: time an instance is unreachable for in seconds before expunged. + :param int inactive_after_seconds + :param int expunge_after_seconds """ def __init__(self, unreachable_inactive_after_seconds=None, - unreachable_expunge_after_seconds=None): - self.unreachable_inactive_after_seconds = \ - unreachable_inactive_after_seconds - self.unreachable_expunge_after_seconds = \ - unreachable_expunge_after_seconds + unreachable_expunge_after_seconds=None, + inactive_after_seconds=None, expunge_after_seconds=None): + self.unreachable_inactive_after_seconds = unreachable_inactive_after_seconds + self.unreachable_expunge_after_seconds = unreachable_expunge_after_seconds + self.inactive_after_seconds = inactive_after_seconds + self.expunge_after_seconds = expunge_after_seconds class MarathonAppVersionInfo(MarathonObject): From fe482fcdc1c9e870c8f2d8af0ced2e78304b6236 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Fri, 6 Jan 2017 16:34:54 -0800 Subject: [PATCH 167/292] Release 0.8.10 --- CHANGELOG.md | 15 ++++++++++++++- setup.py | 2 +- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f76dd91..5a1aa46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,19 @@ # Change Log -## [0.8.9](https://github.com/thefactory/marathon-python/tree/0.8.9) (2016-12-14) +## [0.8.10](https://github.com/thefactory/marathon-python/tree/0.8.10) (2017-01-06) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.9...0.8.10) + +**Closed issues:** + +- InvalidChoiceError when container type is "MESOS" [\#153](https://github.com/thefactory/marathon-python/issues/153) + +**Merged pull requests:** + +- Marathon 1.4 instance [\#165](https://github.com/thefactory/marathon-python/pull/165) ([solarkennedy](https://github.com/solarkennedy)) +- Removed unused sseclient depencency [\#164](https://github.com/thefactory/marathon-python/pull/164) ([migueleliasweb](https://github.com/migueleliasweb)) +- Add new Marathon 1.4 API keywords [\#162](https://github.com/thefactory/marathon-python/pull/162) ([stj](https://github.com/stj)) + +## [0.8.9](https://github.com/thefactory/marathon-python/tree/0.8.9) (2016-12-15) [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.8...0.8.9) **Closed issues:** diff --git a/setup.py b/setup.py index c0c9be3..1369714 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='marathon', - version='0.8.9', + version='0.8.10', description='Marathon Client Library', long_description="""Python interface to the Mesos Marathon REST API.""", author='Mike Babineau', From 9cc3639691399c7991aa73d91d1565faa8caf6c0 Mon Sep 17 00:00:00 2001 From: Dalton Barreto Date: Wed, 25 Jan 2017 18:06:13 -0200 Subject: [PATCH 168/292] Adds MarathonApp.add_env() method --- marathon/models/app.py | 5 ++++- tests/test_model_app.py | 26 ++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 tests/test_model_app.py diff --git a/marathon/models/app.py b/marathon/models/app.py index 366e958..3053f55 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -119,7 +119,7 @@ def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, for d in (deployments or []) ] self.disk = disk - self.env = env + self.env = env or dict() self.executor = executor self.gpus = gpus self.health_checks = health_checks or [] @@ -185,6 +185,9 @@ def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, self.task_stats = task_stats if (isinstance(task_stats, MarathonTaskStats) or task_stats is None) \ else MarathonTaskStats.from_json(task_stats) + def add_env(self, key, value): + self.env[key] = value + class MarathonHealthCheck(MarathonObject): diff --git a/tests/test_model_app.py b/tests/test_model_app.py new file mode 100644 index 0000000..adddb4d --- /dev/null +++ b/tests/test_model_app.py @@ -0,0 +1,26 @@ +# encoding: utf-8 + +from marathon.models.app import MarathonApp +import unittest + + +class MarathonAppTest(unittest.TestCase): + + def test_env_defaults_to_empty_dict(self): + """ + é testé + """ + app = MarathonApp() + self.assertEquals(app.env, {}) + + def test_add_env_empty_dict(self): + app = MarathonApp() + app.add_env("MY_ENV", "my-value") + self.assertDictEqual({"MY_ENV": "my-value"}, app.env) + + def test_add_env_non_empty_dict(self): + env_data = {"OTHER_ENV": "other-value"} + app = MarathonApp(env=env_data) + + app.add_env("MY_ENV", "my-value") + self.assertDictEqual({"MY_ENV": "my-value", "OTHER_ENV": "other-value"}, app.env) From 9bad9a57ecbc7d3481e9860af78078948ed83f8f Mon Sep 17 00:00:00 2001 From: Shuya Tsukamoto Date: Fri, 17 Feb 2017 11:29:20 +0900 Subject: [PATCH 169/292] Change location --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 408cf6e..934fbf9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -9,7 +9,7 @@ marathon-python documentation Python library for interfacing with `Marathon`_ servers via Marathon's `REST API`_. .. _Marathon: https://github.com/mesosphere/marathon -.. _REST API: https://github.com/mesosphere/marathon/blob/master/REST.md +.. _REST API: https://github.com/mesosphere/marathon/blob/master/docs/docs/rest-api.md Project home: https://github.com/thefactory/marathon-python From 8e41c6c62ba68e3df5e3d7242f13b02e00e51a43 Mon Sep 17 00:00:00 2001 From: Nathan Handler Date: Sat, 18 Feb 2017 17:59:35 -0800 Subject: [PATCH 170/292] Update Kill_SELECTIONS https://github.com/mesosphere/marathon/issues/4882 --- marathon/models/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marathon/models/app.py b/marathon/models/app.py index 3053f55..6ca7d84 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -84,7 +84,7 @@ class MarathonApp(MarathonResource): 'deployments', 'tasks', 'tasks_running', 'tasks_staged', 'tasks_healthy', 'tasks_unhealthy'] """List of read-only attributes""" - KILL_SELECTIONS = ["YoungestFirst", "OldestFirst"] + KILL_SELECTIONS = ["YOUNGEST_FIRST", "OLDEST_FIRST"] def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, backoff_seconds=None, cmd=None, constraints=None, container=None, cpus=None, dependencies=None, deployments=None, disk=None, env=None, From 38d745a48dbc8ef2c4d90961924cb0e901b88e8d Mon Sep 17 00:00:00 2001 From: Nathan Handler Date: Sat, 18 Feb 2017 18:06:59 -0800 Subject: [PATCH 171/292] We no longer need the yelp/paasta apt repository --- itests/install-marathon.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/itests/install-marathon.sh b/itests/install-marathon.sh index d1532d6..ce48a2d 100755 --- a/itests/install-marathon.sh +++ b/itests/install-marathon.sh @@ -12,9 +12,7 @@ CODENAME=$(lsb_release -cs) # Add the repository echo "deb http://repos.mesosphere.com/${DISTRO} ${CODENAME} main" | sudo tee /etc/apt/sources.list.d/mesosphere.list -# Temporary repo to get marathon 1.4 -echo "deb https://dl.bintray.com/yelp/paasta trusty main" | sudo tee /etc/apt/sources.list.d/paasta.list -sudo apt-get -y install apt-transport-https && sudo apt-get update +sudo apt-get update # Install packages sudo DEBIAN_FRONTEND=noninteractive apt-get -y install oracle-java8-installer From b7ed965075dc955a3f0fa292d66f006434a182ba Mon Sep 17 00:00:00 2001 From: Nathan Handler Date: Tue, 21 Feb 2017 10:37:09 -0800 Subject: [PATCH 172/292] Marathon 1.4.0 now requires mesos 1.1.0 --- itests/install-marathon.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/itests/install-marathon.sh b/itests/install-marathon.sh index ce48a2d..3ba2a2b 100755 --- a/itests/install-marathon.sh +++ b/itests/install-marathon.sh @@ -20,7 +20,7 @@ sudo apt-get -y purge oracle-java7-installer sudo update-java-alternatives -s java-8-oracle sudo DEBIAN_FRONTEND=noninteractive apt-get install oracle-java8-set-default -sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes install mesos=1.0.* marathon=$MARATHONVERSION* +sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes install mesos=1.1.* marathon=$MARATHONVERSION* # WTF MARATHON? # Why does the precise version have java7 hardcoded if it requires java8? From 902e2ed3af5ecbeb10438799902ad4e2f94d4158 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Wed, 22 Feb 2017 13:16:16 -0800 Subject: [PATCH 173/292] Release 0.8.11 --- README.md | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6d9b168..1528641 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ This is a Python library for interfacing with [Marathon](https://github.com/meso #### Compatibility -* For Marathon 1.4.0, use at least 0.8.9 +* For Marathon 1.4.0, use at least 0.8.11 * For Marathon 1.1.1 and 0.15.x, use at least 0.8.1 * For Marathon 0.14.x, use at least 0.7.6 * For Marathon 0.8.x-0.11.x, use at least marathon-python 0.7.5 diff --git a/setup.py b/setup.py index 1369714..57f95b0 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='marathon', - version='0.8.10', + version='0.8.11', description='Marathon Client Library', long_description="""Python interface to the Mesos Marathon REST API.""", author='Mike Babineau', From 460072b69ad8874fa65656afccb1a59755cf0bcd Mon Sep 17 00:00:00 2001 From: Jbrownstone Date: Thu, 23 Feb 2017 10:05:35 +0100 Subject: [PATCH 174/292] Updated event.py to handle app_terminated_event. As of now, when the event stream gets a app_terminated_event, it raises an exeption : MarathonError: Unknown event_type: app_terminated_event, data: {u'eventType': u'app_terminated_event', u'timestamp': u'2017-02-23T08:54:44.470Z', u'appId': u'/longtest/ms/beds-node'} This fixs it. --- marathon/models/events.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/marathon/models/events.py b/marathon/models/events.py index b371108..1c407cd 100644 --- a/marathon/models/events.py +++ b/marathon/models/events.py @@ -114,6 +114,8 @@ class MarathonEventStreamDetached(MarathonEvent): class MarathonUnhealthyTaskKillEvent(MarathonEvent): KNOWN_ATTRIBUTES = ['app_id', 'task_id', 'version', 'reason'] +class MarathonAppTerminatedEvent(MarathonEvent): + KNOWN_ATTRIBUTES = ['app_id'] class EventFactory: @@ -145,6 +147,7 @@ def __init__(self): 'deployment_step_failure': MarathonDeploymentStepFailure, 'event_stream_attached': MarathonEventStreamAttached, 'event_stream_detached': MarathonEventStreamDetached, + 'app_terminated_event': MarathonAppTerminatedEvent, } def process(self, event): From 324a497a0de8b353aec9e44a277f3b4c2c35961f Mon Sep 17 00:00:00 2001 From: Jbrownstone Date: Thu, 23 Feb 2017 10:34:31 +0100 Subject: [PATCH 175/292] Updated to pass test I fixed the indentation that blocked the tests from passing, had only one carryage return instead of 2 --- marathon/models/events.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/marathon/models/events.py b/marathon/models/events.py index 1c407cd..3b51a3e 100644 --- a/marathon/models/events.py +++ b/marathon/models/events.py @@ -114,9 +114,11 @@ class MarathonEventStreamDetached(MarathonEvent): class MarathonUnhealthyTaskKillEvent(MarathonEvent): KNOWN_ATTRIBUTES = ['app_id', 'task_id', 'version', 'reason'] + class MarathonAppTerminatedEvent(MarathonEvent): KNOWN_ATTRIBUTES = ['app_id'] + class EventFactory: """ From ccd4ef6729353e0c201169ebeb2cef0bc4131fa4 Mon Sep 17 00:00:00 2001 From: Bao Pham Date: Mon, 27 Feb 2017 19:58:58 +0800 Subject: [PATCH 176/292] Update list_apps docs for param app_id --- marathon/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marathon/client.py b/marathon/client.py index a57256d..43d5c20 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -166,7 +166,7 @@ def list_apps(self, cmd=None, embed_tasks=False, embed_counts=False, :param bool embed_last_task_failure: embeds the last task failure :param bool embed_failures: shorthand for embed_last_task_failure :param bool embed_task_stats: embed task stats in result - :param bool app_id: if passed, only show apps with with an 'id' that matches or contains this value + :param str app_id: if passed, only show apps with an 'id' that matches or contains this value :param kwargs: arbitrary search filters :returns: list of applications From 2d6f60d36a27526aa7681ec2f1b26f6c3abf9970 Mon Sep 17 00:00:00 2001 From: Mateusz Moneta Date: Thu, 9 Mar 2017 12:08:44 +0100 Subject: [PATCH 177/292] Use Marathon /v2/apps//tasks endpoint to get tasks by id. --- marathon/client.py | 7 ++----- tests/test_api.py | 9 ++------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index 43d5c20..910af2b 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -477,13 +477,10 @@ def list_tasks(self, app_id=None, **kwargs): :returns: list of tasks :rtype: list[:class:`marathon.models.task.MarathonTask`] """ - response = self._do_request('GET', '/v2/tasks') + response = self._do_request( + 'GET', '/v2/apps/%s/tasks' % app_id if app_id else '/v2/tasks') tasks = self._parse_response( response, MarathonTask, is_list=True, resource_name='tasks') - if app_id: - tasks = [ - task for task in tasks if task.app_id.lstrip('/') == app_id.lstrip('/')] - [setattr(t, 'app_id', app_id) for t in tasks if app_id and t.app_id is None] for k, v in kwargs.items(): diff --git a/tests/test_api.py b/tests/test_api.py index aa809d6..cee709e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -120,14 +120,9 @@ def test_list_tasks_with_app_id(): '"lastFailure": null, "lastSuccess": "2014-10-03T22:57:41.643Z", "taskId": "bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799" } ],' \ ' "host": "10.141.141.10", "id": "bridged-webapp.eb76c51f-4b4a-11e4-ae49-56847afe9799", "ports": [ 31000 ], ' \ '"servicePorts": [ 9000 ], "stagedAt": "2014-10-03T22:16:27.811Z", "startedAt": "2014-10-03T22:57:41.587Z", ' \ - '"version": "2014-10-03T22:16:23.634Z" }, { "appId": "/anotherapp", ' \ - '"healthCheckResults": [ { "alive": true, "consecutiveFailures": 0, "firstSuccess": "2014-10-03T22:57:02.246Z", "lastFailure": null, ' \ - '"lastSuccess": "2014-10-03T22:57:41.649Z", "taskId": "bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799" } ], ' \ - '"host": "10.141.141.10", "id": "bridged-webapp.ef0b5d91-4b4a-11e4-ae49-56847afe9799", "ports": [ 31001 ], ' \ - '"servicePorts": [ 9000 ], "stagedAt": "2014-10-03T22:16:33.814Z", "startedAt": "2014-10-03T22:57:41.593Z", ' \ - '"version": "2014-10-03T22:16:23.634Z" } ] }' + '"version": "2014-10-03T22:16:23.634Z" }]}' with requests_mock.mock() as m: - m.get('http://fake_server/v2/tasks', text=fake_response) + m.get('http://fake_server/v2/apps//anapp/tasks', text=fake_response) mock_client = MarathonClient(servers='http://fake_server') actual_deployments = mock_client.list_tasks(app_id='/anapp') expected_deployments = [models.task.MarathonTask( From ebbfe62196cba82086420cb92007d2768e5db122 Mon Sep 17 00:00:00 2001 From: Nathan Handler Date: Thu, 16 Mar 2017 17:01:02 -0700 Subject: [PATCH 178/292] Add support for 'since' in /v2/queue There appears to be a new attribute called `since` that looks like this: `"since":"2017-03-16T22:37:26.758Z"` --- marathon/models/queue.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/marathon/models/queue.py b/marathon/models/queue.py index 5f14af8..ca212ae 100644 --- a/marathon/models/queue.py +++ b/marathon/models/queue.py @@ -25,13 +25,14 @@ class MarathonQueueItem(MarathonResource): :param bool overdue: """ - def __init__(self, app=None, overdue=None, count=None, delay=None): + def __init__(self, app=None, overdue=None, count=None, delay=None, since=None): self.app = app if isinstance( app, MarathonApp) else MarathonApp().from_json(app) self.overdue = overdue self.count = count self.delay = delay if isinstance( delay, MarathonQueueItemDelay) else MarathonQueueItemDelay().from_json(delay) + self.since = since class MarathonQueueItemDelay(MarathonResource): From aaa04e522c5c1c5f2193308ad5c37e5c02e9dd76 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Thu, 16 Mar 2017 18:08:17 -0700 Subject: [PATCH 179/292] Release 0.8.12 --- README.md | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1528641..e967f40 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ This is a Python library for interfacing with [Marathon](https://github.com/meso #### Compatibility -* For Marathon 1.4.0, use at least 0.8.11 +* For Marathon 1.4.1, use at least 0.8.12 * For Marathon 1.1.1 and 0.15.x, use at least 0.8.1 * For Marathon 0.14.x, use at least 0.7.6 * For Marathon 0.8.x-0.11.x, use at least marathon-python 0.7.5 diff --git a/setup.py b/setup.py index 57f95b0..23b9bb9 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='marathon', - version='0.8.11', + version='0.8.12', description='Marathon Client Library', long_description="""Python interface to the Mesos Marathon REST API.""", author='Mike Babineau', From 7e79cafc6a979b1b0fde904bfb648bdadf4aa1d5 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Thu, 16 Mar 2017 19:51:32 -0700 Subject: [PATCH 180/292] update changelog for 0.8.12 --- CHANGELOG.md | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a1aa46..5e2777e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,29 @@ # Change Log -## [0.8.10](https://github.com/thefactory/marathon-python/tree/0.8.10) (2017-01-06) +## [0.8.12](https://github.com/thefactory/marathon-python/tree/0.8.12) (2017-03-17) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.11...0.8.12) + +**Closed issues:** + +- Unknown event\_type app\_terminated\_event [\#151](https://github.com/thefactory/marathon-python/issues/151) + +**Merged pull requests:** + +- Add support for 'since' in /v2/queue [\#176](https://github.com/thefactory/marathon-python/pull/176) ([nhandler](https://github.com/nhandler)) +- Use Marathon /v2/apps/\/tasks endpoint to get tasks by id. [\#175](https://github.com/thefactory/marathon-python/pull/175) ([nihn](https://github.com/nihn)) +- Update list\_apps docs for param app\_id [\#172](https://github.com/thefactory/marathon-python/pull/172) ([baopham](https://github.com/baopham)) +- Updated event.py to handle app\_terminated\_event. [\#171](https://github.com/thefactory/marathon-python/pull/171) ([Jbrownstone](https://github.com/Jbrownstone)) + +## [0.8.11](https://github.com/thefactory/marathon-python/tree/0.8.11) (2017-02-22) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.10...0.8.11) + +**Merged pull requests:** + +- Update to work with Marathon 1.4.0 [\#169](https://github.com/thefactory/marathon-python/pull/169) ([nhandler](https://github.com/nhandler)) +- Change location [\#168](https://github.com/thefactory/marathon-python/pull/168) ([tsukaby](https://github.com/tsukaby)) +- Adds MarathonApp.add\_env\(\) method [\#166](https://github.com/thefactory/marathon-python/pull/166) ([daltonmatos](https://github.com/daltonmatos)) + +## [0.8.10](https://github.com/thefactory/marathon-python/tree/0.8.10) (2017-01-07) [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.9...0.8.10) **Closed issues:** From 3257b089ce80aabd6f1e1c3453c9dfa37cc1a514 Mon Sep 17 00:00:00 2001 From: Nathan Handler Date: Thu, 16 Mar 2017 20:42:34 -0700 Subject: [PATCH 181/292] /v2/queue also added a processed_offers_summary attribute --- marathon/models/queue.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/marathon/models/queue.py b/marathon/models/queue.py index ca212ae..33c0228 100644 --- a/marathon/models/queue.py +++ b/marathon/models/queue.py @@ -25,7 +25,8 @@ class MarathonQueueItem(MarathonResource): :param bool overdue: """ - def __init__(self, app=None, overdue=None, count=None, delay=None, since=None): + def __init__(self, app=None, overdue=None, count=None, delay=None, since=None, + processed_offers_summary=None): self.app = app if isinstance( app, MarathonApp) else MarathonApp().from_json(app) self.overdue = overdue @@ -33,6 +34,7 @@ def __init__(self, app=None, overdue=None, count=None, delay=None, since=None): self.delay = delay if isinstance( delay, MarathonQueueItemDelay) else MarathonQueueItemDelay().from_json(delay) self.since = since + self.processed_offers_summary = processed_offers_summary class MarathonQueueItemDelay(MarathonResource): From e70307e74dccee0958c06a8431fcae1774cd2835 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Fri, 17 Mar 2017 10:33:52 -0700 Subject: [PATCH 182/292] Release 0.8.13 --- README.md | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e967f40..0822d97 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ This is a Python library for interfacing with [Marathon](https://github.com/meso #### Compatibility -* For Marathon 1.4.1, use at least 0.8.12 +* For Marathon 1.4.1, use at least 0.8.13 * For Marathon 1.1.1 and 0.15.x, use at least 0.8.1 * For Marathon 0.14.x, use at least 0.7.6 * For Marathon 0.8.x-0.11.x, use at least marathon-python 0.7.5 diff --git a/setup.py b/setup.py index 23b9bb9..1358612 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='marathon', - version='0.8.12', + version='0.8.13', description='Marathon Client Library', long_description="""Python interface to the Mesos Marathon REST API.""", author='Mike Babineau', From 8822aef1ddf61cd758a4418f05e4af14aa16536e Mon Sep 17 00:00:00 2001 From: Hugues Lerebours Date: Wed, 22 Mar 2017 09:49:45 +0100 Subject: [PATCH 183/292] [fix] broken build: glibc++ not found --- .travis.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.travis.yml b/.travis.yml index 5eccbc9..575df5d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,3 +23,10 @@ script: # Work around travis-ci/travis-ci#5227 addons: hostname: localhost + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - libstdc++6-4.7-dev + +sudo: required # make it explicit: it was by default only because this repo was set up before 2015 (new forks need it) From faebe66e8fb0c7a43a86b6c3c14380d856ab0e88 Mon Sep 17 00:00:00 2001 From: Hugues Lerebours Date: Fri, 17 Mar 2017 13:19:44 +0100 Subject: [PATCH 184/292] [fix] Handle non-JSON errors from Marathon --- marathon/exceptions.py | 2 +- tests/test_exceptions.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 tests/test_exceptions.py diff --git a/marathon/exceptions.py b/marathon/exceptions.py index 4dcff8d..8d2d249 100644 --- a/marathon/exceptions.py +++ b/marathon/exceptions.py @@ -9,7 +9,7 @@ def __init__(self, response): :param :class:`requests.Response` response: HTTP response """ self.error_message = response.reason or '' - if response.content: + if response.content and 'application/json' in response.headers.get('content-type', ''): content = response.json() self.error_message = content.get('message', self.error_message) self.error_details = content.get('details') diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py new file mode 100644 index 0000000..906c119 --- /dev/null +++ b/tests/test_exceptions.py @@ -0,0 +1,38 @@ +import json + +import requests + +from marathon.exceptions import MarathonHttpError, InternalServerError + + +def test_400_error(): + fake_response = requests.Response() + fake_message = "Invalid JSON" + fake_details = [{"path": "/taskKillGracePeriodSeconds", "errors": ["error.expected.jsnumber"]}] + fake_response._content = json.dumps({"message": fake_message, "details": fake_details}).encode() + fake_response.status_code = 400 + fake_response.headers['Content-Type'] = 'application/json' + + exc = MarathonHttpError(fake_response) + assert exc.status_code == 400 + assert exc.error_message == fake_message + assert exc.error_details == fake_details + + +def test_503_error(): + fake_response = requests.Response() + fake_response._content = """ + +Error 503 + + +

HTTP ERROR: 503

+ +""" + fake_response.reason = "reason" + fake_response.status_code = 503 + + exc = InternalServerError(fake_response) + assert exc.status_code == 503 + assert exc.error_message == "reason" + assert not hasattr(exc, 'error_details') From e5ac47c6b21c86d34487c8713979a30e84ee25d3 Mon Sep 17 00:00:00 2001 From: Hugues Lerebours Date: Wed, 22 Mar 2017 16:11:54 +0100 Subject: [PATCH 185/292] [fix] util.to_camel_case doesn't handle digits it was converting "ignore_http1xx" into "ignoreHttp1Xx" instead of "ignoreHttp1xx" (2 lower x at the end) --- marathon/util.py | 2 +- tests/test_util.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 tests/test_util.py diff --git a/marathon/util.py b/marathon/util.py index 917c0b8..e999333 100644 --- a/marathon/util.py +++ b/marathon/util.py @@ -56,7 +56,7 @@ def default(self, obj): def to_camel_case(snake_str): words = snake_str.split('_') - return words[0] + ''.join(w.title() for w in words[1:]) + return words[0] + ''.join(w.capitalize() for w in words[1:]) def to_snake_case(camel_str): diff --git a/tests/test_util.py b/tests/test_util.py new file mode 100644 index 0000000..8956051 --- /dev/null +++ b/tests/test_util.py @@ -0,0 +1,29 @@ +from marathon.util import to_camel_case, to_snake_case + + +def _apply_on_pairs(f): + # this strategy is used to have the assertion stack trace + # point to the right pair of strings in case of test failure + f('foo', 'foo') + f('foo42', 'foo42') + f('fooBar', 'foo_bar') + f('f0o42Bar', 'f0o42_bar') + f('fooBarBaz', 'foo_bar_baz') + f('ignoreHttp1xx', 'ignore_http1xx') + f('whereAmI', 'where_am_i') + f('iSee', 'i_see') + f('doISee', 'do_i_see') + + +def test_to_camel_case(): + def test(camel, snake): + assert to_camel_case(snake) == camel + + _apply_on_pairs(test) + + +def test_to_snake_case(): + def test(camel, snake): + assert to_snake_case(camel) == snake + + _apply_on_pairs(test) From 247865b95f481e2ae883823f7e471ceb5b949ca2 Mon Sep 17 00:00:00 2001 From: Mateusz Moneta Date: Mon, 20 Mar 2017 10:33:41 +0100 Subject: [PATCH 186/292] Support for "disabled" unreachableStrategy. --- marathon/models/app.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/marathon/models/app.py b/marathon/models/app.py index 6ca7d84..469584f 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -288,6 +288,7 @@ class MarathonUnreachableStrategy(MarathonObject): :param int inactive_after_seconds :param int expunge_after_seconds """ + DISABLED = 'disabled' def __init__(self, unreachable_inactive_after_seconds=None, unreachable_expunge_after_seconds=None, @@ -297,6 +298,12 @@ def __init__(self, unreachable_inactive_after_seconds=None, self.inactive_after_seconds = inactive_after_seconds self.expunge_after_seconds = expunge_after_seconds + @classmethod + def from_json(cls, attributes): + if attributes == cls.DISABLED: + return cls.DISABLED + return super(MarathonUnreachableStrategy, cls).from_json(attributes) + class MarathonAppVersionInfo(MarathonObject): From ff2bd95bd98722212d2963b26f0d9cae746f5b41 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Fri, 24 Mar 2017 09:33:01 -0700 Subject: [PATCH 187/292] Release 0.8.14 --- CHANGELOG.md | 24 ++++++++++++++++++++++++ setup.py | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e2777e..0ca1d50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,29 @@ # Change Log +## [0.8.14](https://github.com/thefactory/marathon-python/tree/0.8.14) (2017-03-24) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.13...0.8.14) + +**Closed issues:** + +- Pypi need update to marathon 0.8.13 [\#185](https://github.com/thefactory/marathon-python/issues/185) +- Tests fail on master branch [\#180](https://github.com/thefactory/marathon-python/issues/180) +- ignoreHttp1xx or ignoreHttp1Xx [\#125](https://github.com/thefactory/marathon-python/issues/125) +- ValueError when 401 Unauthorized is received [\#22](https://github.com/thefactory/marathon-python/issues/22) + +**Merged pull requests:** + +- \[fix\] util.to\_camel\_case doesn't handle digits [\#184](https://github.com/thefactory/marathon-python/pull/184) ([hlerebours](https://github.com/hlerebours)) +- \[fix\] broken build: glibc++ not found [\#183](https://github.com/thefactory/marathon-python/pull/183) ([hlerebours](https://github.com/hlerebours)) +- Support for "disabled" unreachableStrategy. [\#182](https://github.com/thefactory/marathon-python/pull/182) ([nihn](https://github.com/nihn)) +- \[fix\] Handle non-JSON errors from Marathon [\#178](https://github.com/thefactory/marathon-python/pull/178) ([hlerebours](https://github.com/hlerebours)) + +## [0.8.13](https://github.com/thefactory/marathon-python/tree/0.8.13) (2017-03-17) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.12...0.8.13) + +**Merged pull requests:** + +- Support processed\_offers\_summary attribute [\#177](https://github.com/thefactory/marathon-python/pull/177) ([nhandler](https://github.com/nhandler)) + ## [0.8.12](https://github.com/thefactory/marathon-python/tree/0.8.12) (2017-03-17) [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.11...0.8.12) diff --git a/setup.py b/setup.py index 1358612..c9a7248 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='marathon', - version='0.8.13', + version='0.8.14', description='Marathon Client Library', long_description="""Python interface to the Mesos Marathon REST API.""", author='Mike Babineau', From 1ba16e6add3fe40dd11e47220d3ff4b329be288b Mon Sep 17 00:00:00 2001 From: Nathan Handler Date: Mon, 3 Apr 2017 13:40:33 -0700 Subject: [PATCH 188/292] Run travis tests against 1.4.2 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 575df5d..a478dbf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ env: - - MARATHONVERSION: 1.4.0 + - MARATHONVERSION: 1.4.2 - MARATHONVERSION: 1.3.0 - MARATHONVERSION: 1.1.2 - MARATHONVERSION: 0.15.3 From 8a9eda9907ac47335229dea41cb995646e91acd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20GERMAIN?= Date: Tue, 11 Apr 2017 14:08:03 +0000 Subject: [PATCH 189/292] handle case when non-ascii char are logged --- marathon/client.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index 910af2b..f6fcc94 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -90,21 +90,21 @@ def _do_request(self, method, path, params=None, data=None): if response.status_code >= 500: marathon.log.error('Got HTTP {code}: {body}'.format( - code=response.status_code, body=response.text)) + code=response.status_code, body=response.text.encode('utf-8'))) raise InternalServerError(response) elif response.status_code >= 400: marathon.log.error('Got HTTP {code}: {body}'.format( - code=response.status_code, body=response.text)) + code=response.status_code, body=response.text.encode('utf-8'))) if response.status_code == 404: raise NotFoundError(response) else: raise MarathonHttpError(response) elif response.status_code >= 300: marathon.log.warn('Got HTTP {code}: {body}'.format( - code=response.status_code, body=response.text)) + code=response.status_code, body=response.text.encode('utf-8'))) else: marathon.log.debug('Got HTTP {code}: {body}'.format( - code=response.status_code, body=response.text)) + code=response.status_code, body=response.text.encode('utf-8'))) return response @@ -721,7 +721,7 @@ def ping(self): :rtype: str """ response = self._do_request('GET', '/ping') - return response.text + return response.text.encode('utf-8') def get_metrics(self): """Get server metrics From b499e0431b59684d32fccc3a7e5dec2652cc0e68 Mon Sep 17 00:00:00 2001 From: "Trevor Joynson (trevorj)" Date: Tue, 11 Apr 2017 21:23:06 -0700 Subject: [PATCH 190/292] The client should not enforce what the API accepts. --- marathon/models/constraint.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/marathon/models/constraint.py b/marathon/models/constraint.py index 184d9f8..9b23c18 100644 --- a/marathon/models/constraint.py +++ b/marathon/models/constraint.py @@ -16,12 +16,9 @@ class MarathonConstraint(MarathonObject): :type value: str, int, or None """ - OPERATORS = ['UNIQUE', 'CLUSTER', 'GROUP_BY', 'LIKE', 'UNLIKE'] """Valid operators""" def __init__(self, field, operator, value=None): - if operator not in self.OPERATORS: - raise InvalidChoiceError('operator', operator, self.OPERATORS) self.field = field self.operator = operator self.value = value From 10a2e0b9fb42e3a79716738af2bfefa38581bd40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Leite?= Date: Sun, 16 Apr 2017 23:26:39 -0300 Subject: [PATCH 191/292] Adding the key "networks" in the JSON received of marathon --- marathon/models/app.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/marathon/models/app.py b/marathon/models/app.py index 469584f..4c486a0 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -94,7 +94,7 @@ def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, tasks_healthy=None, task_kill_grace_period_seconds=None, tasks_unhealthy=None, upgrade_strategy=None, unreachable_strategy=None, uris=None, user=None, version=None, version_info=None, ip_address=None, fetch=None, task_stats=None, readiness_checks=None, - readiness_check_results=None, secrets=None, port_definitions=None, residency=None, gpus=None): + readiness_check_results=None, secrets=None, port_definitions=None, residency=None, gpus=None, networks=None): # self.args = args or [] self.accepted_resource_roles = accepted_resource_roles @@ -184,6 +184,8 @@ def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, else MarathonAppVersionInfo.from_json(version_info) self.task_stats = task_stats if (isinstance(task_stats, MarathonTaskStats) or task_stats is None) \ else MarathonTaskStats.from_json(task_stats) + self.networks = networks + def add_env(self, key, value): self.env[key] = value From fadbc9b0e155fcf73ca34c2542458a4f3ac53302 Mon Sep 17 00:00:00 2001 From: Guanglu Guo Date: Mon, 17 Apr 2017 10:35:02 +0800 Subject: [PATCH 192/292] Fix pep8 unused import --- marathon/models/constraint.py | 1 - 1 file changed, 1 deletion(-) diff --git a/marathon/models/constraint.py b/marathon/models/constraint.py index 9b23c18..7b14bf3 100644 --- a/marathon/models/constraint.py +++ b/marathon/models/constraint.py @@ -1,4 +1,3 @@ -from ..exceptions import InvalidChoiceError from .base import MarathonObject From 8fbd7a08cf068f4e3f9206dbc47c9711b71f25f5 Mon Sep 17 00:00:00 2001 From: Guanglu Guo Date: Tue, 18 Apr 2017 11:48:08 +0800 Subject: [PATCH 193/292] Add raw option for event_stream method When handling events asynchronously, you may just want to put event data in a message queue, so getting raw data here is a small optimization that saves parsing and serializing event data. --- marathon/client.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index f6fcc94..48aa4bc 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -732,9 +732,10 @@ def get_metrics(self): response = self._do_request('GET', '/metrics') return response.json() - def event_stream(self): + def event_stream(self, raw=False): """Polls event bus using /v2/events + :param bool raw: if true, yield raw event text, else yield MarathonEvent object :returns: iterator with events :rtype: iterator """ @@ -746,9 +747,12 @@ def event_stream(self): _data = raw_message.decode('utf8').split(':', 1) if _data[0] == 'data': - event_data = json.loads(_data[1].strip()) - if 'eventType' not in event_data: - raise MarathonError('Invalid event data received.') - yield ef.process(event_data) + if raw: + yield _data[1] + else: + event_data = json.loads(_data[1].strip()) + if 'eventType' not in event_data: + raise MarathonError('Invalid event data received.') + yield ef.process(event_data) except ValueError: raise MarathonError('Invalid event data received.') From 612e516c512d5af83af812942de1d794aeccf996 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Wed, 19 Apr 2017 14:57:34 -0700 Subject: [PATCH 194/292] Updated changelog --- CHANGELOG.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ca1d50..eff7bbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ # Change Log +## [Unreleased](https://github.com/thefactory/marathon-python/tree/HEAD) + +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.14...HEAD) + +**Closed issues:** + +- Adding the key "networks" in the JSON received of marathon \( list apps \) [\#192](https://github.com/thefactory/marathon-python/issues/192) +- logs cause exception with non-ascii characters [\#187](https://github.com/thefactory/marathon-python/issues/187) +- Add new Yelp Contributors [\#181](https://github.com/thefactory/marathon-python/issues/181) + +**Merged pull requests:** + +- Adding the key "networks" in the JSON received of marathon [\#193](https://github.com/thefactory/marathon-python/pull/193) ([joaoleite](https://github.com/joaoleite)) +- Remove out of date constraint validation of operator. [\#190](https://github.com/thefactory/marathon-python/pull/190) ([akatrevorjay](https://github.com/akatrevorjay)) +- Add raw\_data option for event\_stream method [\#189](https://github.com/thefactory/marathon-python/pull/189) ([fengyehong](https://github.com/fengyehong)) +- handle case when non-ascii char are logged [\#188](https://github.com/thefactory/marathon-python/pull/188) ([tgermain](https://github.com/tgermain)) + ## [0.8.14](https://github.com/thefactory/marathon-python/tree/0.8.14) (2017-03-24) [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.13...0.8.14) From 181a89c470265af419dd80d4ab7a1328180fa5ec Mon Sep 17 00:00:00 2001 From: Jorge Gallegos Date: Fri, 21 Apr 2017 15:42:18 -0700 Subject: [PATCH 195/292] There are cases where this check stack traces For example, if you're trying to compare a MarathonObject with ``None`` or ``str()`` those won't have the ``__dict__`` attribute. Another approach was to test if the ``__dict__`` attribute was part of ``other`` but it introspection feels a bit more off than this. --- marathon/models/base.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/marathon/models/base.py b/marathon/models/base.py index b4abd6e..f381368 100644 --- a/marathon/models/base.py +++ b/marathon/models/base.py @@ -12,7 +12,10 @@ def __repr__(self): return "{clazz}::{obj}".format(clazz=self.__class__.__name__, obj=self.to_json(minimal=False)) def __eq__(self, other): - return self.__dict__ == other.__dict__ + try: + return self.__dict__ == other.__dict__ + except: + return False def json_repr(self, minimal=False): """Construct a JSON-friendly representation of the object. @@ -58,7 +61,10 @@ def __repr__(self): return "{clazz}::{obj}".format(clazz=self.__class__.__name__, obj=self.to_json()) def __eq__(self, other): - return self.__dict__ == other.__dict__ + try: + return self.__dict__ == other.__dict__ + except: + return False def __str__(self): return "{clazz}::".format(clazz=self.__class__.__name__) + str(self.__dict__) From 86fc98d262cd8f6c2228353a04f0fc134074a678 Mon Sep 17 00:00:00 2001 From: Bekir Dogan Date: Sun, 23 Apr 2017 06:23:49 +0100 Subject: [PATCH 196/292] add new marathon event_stream events --- marathon/models/deployment.py | 9 ++++++--- marathon/models/events.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/marathon/models/deployment.py b/marathon/models/deployment.py index 929847b..97e73ec 100644 --- a/marathon/models/deployment.py +++ b/marathon/models/deployment.py @@ -58,10 +58,11 @@ class MarathonDeploymentAction(MarathonObject): :param type readiness_check_results: Undocumented """ - def __init__(self, action=None, app=None, apps=None, type=None, readiness_check_results=None): + def __init__(self, action=None, app=None, apps=None, type=None, readiness_check_results=None, pod=None): self.action = action self.app = app self.apps = apps + self.pod = pod self.type = type # TODO: Remove builtin shadow self.readiness_check_results = readiness_check_results # TODO: The docs say this is called just "readinessChecks?" @@ -86,20 +87,22 @@ def __init__(self, actions=None): class MarathonDeploymentOriginalState(MarathonObject): def __init__(self, dependencies=None, - apps=None, id=None, version=None, groups=None): + apps=None, id=None, version=None, groups=None, pods=None): self.apps = apps self.groups = groups self.id = id self.version = version self.dependencies = dependencies + self.pods = pods class MarathonDeploymentTargetState(MarathonObject): def __init__(self, groups=None, apps=None, - dependencies=None, id=None, version=None): + dependencies=None, id=None, version=None, pods=None): self.apps = apps self.groups = groups self.id = id self.version = version self.dependencies = dependencies + self.pods = pods diff --git a/marathon/models/events.py b/marathon/models/events.py index 3b51a3e..3860f8e 100644 --- a/marathon/models/events.py +++ b/marathon/models/events.py @@ -119,6 +119,30 @@ class MarathonAppTerminatedEvent(MarathonEvent): KNOWN_ATTRIBUTES = ['app_id'] +class MarathonInstanceChangedEvent(MarathonEvent): + KNOWN_ATTRIBUTES = ['instance_id', 'slave_id', 'condition', 'host', 'run_spec_id', 'run_spec_version'] + + +class MarathonUnknownInstanceTerminated(MarathonEvent): + KNOWN_ATTRIBUTES = ['instance_id', 'run_spec_id', 'condition'] + + +class MarathonInstanceHealthChanged(MarathonEvent): + KNOWN_ATTRIBUTES = ['instance_id', 'run_spec_id', 'run_spec_version', 'healthy'] + + +class MarathonPodCreatedEvent(MarathonEvent): + KNOWN_ATTRIBUTES = ['client_ip', 'uri'] + + +class MarathonPodUpdatedEvent(MarathonEvent): + KNOWN_ATTRIBUTES = ['client_ip', 'uri'] + + +class MarathonPodDeletedEvent(MarathonEvent): + KNOWN_ATTRIBUTES = ['client_ip', 'uri'] + + class EventFactory: """ @@ -150,6 +174,12 @@ def __init__(self): 'event_stream_attached': MarathonEventStreamAttached, 'event_stream_detached': MarathonEventStreamDetached, 'app_terminated_event': MarathonAppTerminatedEvent, + 'instance_changed_event': MarathonInstanceChangedEvent, + 'unknown_instance_terminated_event': MarathonUnknownInstanceTerminated, + 'instance_health_changed_event': MarathonInstanceChangedEvent, + 'pod_created_event': MarathonPodCreatedEvent, + 'pod_updated_event': MarathonPodUpdatedEvent, + 'pod_deleted_event': MarathonPodDeletedEvent, } def process(self, event): From 99688817301849146352c1b6e3c55136e3e70da7 Mon Sep 17 00:00:00 2001 From: Bekir Dogan Date: Mon, 24 Apr 2017 12:52:56 +0100 Subject: [PATCH 197/292] add missing event: unhealthy_instance_kill_event --- marathon/models/events.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/marathon/models/events.py b/marathon/models/events.py index 3860f8e..f2948ee 100644 --- a/marathon/models/events.py +++ b/marathon/models/events.py @@ -143,6 +143,10 @@ class MarathonPodDeletedEvent(MarathonEvent): KNOWN_ATTRIBUTES = ['client_ip', 'uri'] +class MarathonUnhealthyInstanceKillEvent(MarathonEvent): + KNOWN_ATTRIBUTES = ['app_id', 'task_id', 'instance_id', 'version', 'reason', 'host', 'slave_id'] + + class EventFactory: """ @@ -176,6 +180,7 @@ def __init__(self): 'app_terminated_event': MarathonAppTerminatedEvent, 'instance_changed_event': MarathonInstanceChangedEvent, 'unknown_instance_terminated_event': MarathonUnknownInstanceTerminated, + 'unhealthy_instance_kill_event': MarathonUnhealthyInstanceKillEvent, 'instance_health_changed_event': MarathonInstanceChangedEvent, 'pod_created_event': MarathonPodCreatedEvent, 'pod_updated_event': MarathonPodUpdatedEvent, From 1e79eede42d1446158f3eb8cfde110212010824e Mon Sep 17 00:00:00 2001 From: Guanglu Guo Date: Fri, 28 Apr 2017 18:12:10 +0800 Subject: [PATCH 198/292] Fix local variable reference error --- marathon/client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index 48aa4bc..5961ccd 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -121,9 +121,9 @@ def _do_sse_request(self, path): ) except Exception as e: marathon.log.error('Error while calling %s: %s', url, e.message) - - if response.ok: - return response.iter_lines() + else: + if response.ok: + return response.iter_lines() raise MarathonError('No remaining Marathon servers to try') From 7b4c3697ebddb2de9b64160ef580b49ee3e64f83 Mon Sep 17 00:00:00 2001 From: Guanglu Guo Date: Fri, 28 Apr 2017 19:49:01 +0800 Subject: [PATCH 199/292] Pep8 fix --- marathon/models/app.py | 1 - 1 file changed, 1 deletion(-) diff --git a/marathon/models/app.py b/marathon/models/app.py index 4c486a0..0a8eafc 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -185,7 +185,6 @@ def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, self.task_stats = task_stats if (isinstance(task_stats, MarathonTaskStats) or task_stats is None) \ else MarathonTaskStats.from_json(task_stats) self.networks = networks - def add_env(self, key, value): self.env[key] = value From 87e377650fbdc6c1b6d02334e615627ff2ab7585 Mon Sep 17 00:00:00 2001 From: wujiaxing Date: Tue, 6 Jun 2017 20:36:47 +0800 Subject: [PATCH 200/292] Add requests session param tip. --- marathon/client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/marathon/client.py b/marathon/client.py index 5961ccd..ffc3f8a 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -32,6 +32,7 @@ def __init__(self, servers, username=None, password=None, timeout=10, session=No :type servers: str or list[str] :param str username: Basic auth username :param str password: Basic auth password + :param requests.session session: requests.session for reusing the connections :param int timeout: Timeout (in seconds) for requests to Marathon :param str auth_token: Token-based auth token, used with DCOS + Oauth :param bool verify: Enable SSL certificate verification From b307df3b2b45e5ab003903b8ed5cf341506965fd Mon Sep 17 00:00:00 2001 From: Joseph Lynch Date: Tue, 20 Jun 2017 18:47:09 -0700 Subject: [PATCH 201/292] Add a regression test showing hashing error --- tests/test_model_object.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 tests/test_model_object.py diff --git a/tests/test_model_object.py b/tests/test_model_object.py new file mode 100644 index 0000000..9f37191 --- /dev/null +++ b/tests/test_model_object.py @@ -0,0 +1,21 @@ +# encoding: utf-8 + +from marathon.models.base import MarathonObject +import unittest + + +class MarathonObjectTest(unittest.TestCase): + + def test_hashable(self): + """ + Regression test for issue #203 + + MarathonObject defined __eq__ but not __hash__, meaning that in + in Python2.7 MarathonObjects are hashable, but in Python3 they're not, + + This test ensures that we are hashable in all versions of python + """ + obj = MarathonObject() + collection = {} + collection[obj] = True + assert collection[obj] From fd34b1cd6b828d1e17c8f033bc9629e646a8d7c3 Mon Sep 17 00:00:00 2001 From: Joseph Lynch Date: Tue, 20 Jun 2017 21:16:43 -0700 Subject: [PATCH 202/292] Define __hash__ on MarathonObject --- marathon/models/base.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/marathon/models/base.py b/marathon/models/base.py index f381368..93a499f 100644 --- a/marathon/models/base.py +++ b/marathon/models/base.py @@ -5,7 +5,6 @@ class MarathonObject(object): - """Base Marathon object.""" def __repr__(self): @@ -17,6 +16,12 @@ def __eq__(self, other): except: return False + def __hash__(self): + # Technically this class shouldn't be hashable because it often + # contains mutable fields, but in practice this class is used more + # like a record or namedtuple. + return hash(self.to_json()) + def json_repr(self, minimal=False): """Construct a JSON-friendly representation of the object. From b52edbcdb3b1849e012fbf9456ac686ee383007d Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Wed, 21 Jun 2017 09:51:48 -0700 Subject: [PATCH 203/292] Release 0.9.0 --- CHANGELOG.md | 15 ++++++++++++--- setup.py | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eff7bbe..49f24ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,17 +1,26 @@ # Change Log -## [Unreleased](https://github.com/thefactory/marathon-python/tree/HEAD) - -[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.14...HEAD) +## [0.9.0](https://github.com/thefactory/marathon-python/tree/0.9.0) (2017-06-21) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.14...0.9.0) **Closed issues:** +- MarathonObject is not hashable in Python3 [\#203](https://github.com/thefactory/marathon-python/issues/203) +- Missing pods attribute in MarathonDeploymentOriginalState [\#196](https://github.com/thefactory/marathon-python/issues/196) +- Travis tests have stopped automatically running [\#194](https://github.com/thefactory/marathon-python/issues/194) - Adding the key "networks" in the JSON received of marathon \( list apps \) [\#192](https://github.com/thefactory/marathon-python/issues/192) +- Unknown event\_type: instance\_changed\_event [\#191](https://github.com/thefactory/marathon-python/issues/191) - logs cause exception with non-ascii characters [\#187](https://github.com/thefactory/marathon-python/issues/187) - Add new Yelp Contributors [\#181](https://github.com/thefactory/marathon-python/issues/181) **Merged pull requests:** +- Add hash to marathon object [\#204](https://github.com/thefactory/marathon-python/pull/204) ([jolynch](https://github.com/jolynch)) +- Add requests session param tip. [\#201](https://github.com/thefactory/marathon-python/pull/201) ([Colstuwjx](https://github.com/Colstuwjx)) +- Fix variable [\#199](https://github.com/thefactory/marathon-python/pull/199) ([fengyehong](https://github.com/fengyehong)) +- add new marathon event\_stream events [\#198](https://github.com/thefactory/marathon-python/pull/198) ([bergerx](https://github.com/bergerx)) +- There are cases where this check stack traces [\#197](https://github.com/thefactory/marathon-python/pull/197) ([thekad](https://github.com/thekad)) +- Updated changelog [\#195](https://github.com/thefactory/marathon-python/pull/195) ([solarkennedy](https://github.com/solarkennedy)) - Adding the key "networks" in the JSON received of marathon [\#193](https://github.com/thefactory/marathon-python/pull/193) ([joaoleite](https://github.com/joaoleite)) - Remove out of date constraint validation of operator. [\#190](https://github.com/thefactory/marathon-python/pull/190) ([akatrevorjay](https://github.com/akatrevorjay)) - Add raw\_data option for event\_stream method [\#189](https://github.com/thefactory/marathon-python/pull/189) ([fengyehong](https://github.com/fengyehong)) diff --git a/setup.py b/setup.py index c9a7248..dd8b4a6 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='marathon', - version='0.8.14', + version='0.9.0', description='Marathon Client Library', long_description="""Python interface to the Mesos Marathon REST API.""", author='Mike Babineau', From 0519824c537a96474e0501e1ac45f7a626391a31 Mon Sep 17 00:00:00 2001 From: Joseph Lynch Date: Mon, 26 Jun 2017 22:08:13 -0700 Subject: [PATCH 204/292] Add regression test for MarathonResource --- tests/test_model_object.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_model_object.py b/tests/test_model_object.py index 9f37191..89795fb 100644 --- a/tests/test_model_object.py +++ b/tests/test_model_object.py @@ -1,6 +1,7 @@ # encoding: utf-8 from marathon.models.base import MarathonObject +from marathon.models.base import MarathonResource import unittest @@ -19,3 +20,21 @@ def test_hashable(self): collection = {} collection[obj] = True assert collection[obj] + + +class MarathonResourceHashable(unittest.TestCase): + + def test_hashable(self): + """ + Regression test for issue #203 + + MarathonResource defined __eq__ but not __hash__, meaning that in + in Python2.7 MarathonResources are hashable, but in Python3 they're + not + + This test ensures that we are hashable in all versions of python + """ + obj = MarathonResource() + collection = {} + collection[obj] = True + assert collection[obj] From a614abfd1afc9c79213e2b95623ecca8e09125db Mon Sep 17 00:00:00 2001 From: Joseph Lynch Date: Mon, 26 Jun 2017 22:09:03 -0700 Subject: [PATCH 205/292] Define __hash__ on MarathonResource too --- marathon/models/base.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/marathon/models/base.py b/marathon/models/base.py index 93a499f..ecc8030 100644 --- a/marathon/models/base.py +++ b/marathon/models/base.py @@ -71,6 +71,12 @@ def __eq__(self, other): except: return False + def __hash__(self): + # Technically this class shouldn't be hashable because it often + # contains mutable fields, but in practice this class is used more + # like a record or namedtuple. + return hash(self.to_json()) + def __str__(self): return "{clazz}::".format(clazz=self.__class__.__name__) + str(self.__dict__) From ddea2f5532524226222330c9ecd3ee08ea43e5cd Mon Sep 17 00:00:00 2001 From: Guanglu Guo Date: Thu, 29 Jun 2017 11:18:34 +0800 Subject: [PATCH 206/292] Allow event type filter on event stream This if supported from marathon 1.4. --- itests/steps/marathon_steps.py | 2 +- marathon/client.py | 18 ++++++++++++++---- marathon/models/events.py | 2 ++ 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/itests/steps/marathon_steps.py b/itests/steps/marathon_steps.py index afc4c12..434bc04 100644 --- a/itests/steps/marathon_steps.py +++ b/itests/steps/marathon_steps.py @@ -115,7 +115,7 @@ def kill_tasks(context, to_kill, which): def list_tasks(context, which): app = context.client.get_app('test-%s-app' % which) tasks = context.client.list_tasks('test-%s-app' % which) - assert len(tasks) == app.instances + assert len(tasks) == app.instances, "we defined %s tasks, got %s tasks" % (app.instances, len(tasks)) def listen_for_events(client, events): diff --git a/marathon/client.py b/marathon/client.py index ffc3f8a..8c72042 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -12,7 +12,7 @@ import marathon from .models import MarathonApp, MarathonDeployment, MarathonGroup, MarathonInfo, MarathonTask, MarathonEndpoint, MarathonQueueItem from .exceptions import InternalServerError, NotFoundError, MarathonHttpError, MarathonError -from .models.events import EventFactory +from .models.events import EventFactory, MarathonEvent from .util import MarathonJsonEncoder, MarathonMinimalJsonEncoder @@ -109,13 +109,14 @@ def _do_request(self, method, path, params=None, data=None): return response - def _do_sse_request(self, path): + def _do_sse_request(self, path, params=None): """Query Marathon server for events.""" for server in list(self.servers): url = ''.join([server.rstrip('/'), path]) try: response = requests.get( url, + params=params, stream=True, headers={'Accept': 'text/event-stream'}, auth=self.auth @@ -733,17 +734,26 @@ def get_metrics(self): response = self._do_request('GET', '/metrics') return response.json() - def event_stream(self, raw=False): + def event_stream(self, raw=False, event_types=None): """Polls event bus using /v2/events :param bool raw: if true, yield raw event text, else yield MarathonEvent object + :param event_types: a list of event types to consume + :type event_types: list[type] or list[str] :returns: iterator with events :rtype: iterator """ ef = EventFactory() - for raw_message in self._do_sse_request('/v2/events'): + params = { + 'event_type': [ + EventFactory.class_to_event[et] if isinstance(et, type) and issubclass(et, MarathonEvent) else et + for et in event_types or [] + ] + } + + for raw_message in self._do_sse_request('/v2/events', params=params): try: _data = raw_message.decode('utf8').split(':', 1) diff --git a/marathon/models/events.py b/marathon/models/events.py index f2948ee..7274e32 100644 --- a/marathon/models/events.py +++ b/marathon/models/events.py @@ -187,6 +187,8 @@ def __init__(self): 'pod_deleted_event': MarathonPodDeletedEvent, } + class_to_event = dict((v, k) for k, v in event_to_class.iteritems()) + def process(self, event): event_type = event['eventType'] if event_type in self.event_to_class: From f6b70c7072a74a18461034688f3b9f9124d53a6c Mon Sep 17 00:00:00 2001 From: Alexis Tacnet Date: Thu, 6 Jul 2017 13:46:19 +0200 Subject: [PATCH 207/292] Add udp,tcp to protocols --- marathon/models/container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marathon/models/container.py b/marathon/models/container.py index bb9520b..a910b41 100644 --- a/marathon/models/container.py +++ b/marathon/models/container.py @@ -83,7 +83,7 @@ class MarathonContainerPortMapping(MarathonObject): :param object labels: """ - PROTOCOLS = ['tcp', 'udp'] + PROTOCOLS = ['tcp', 'udp', 'udp,tcp'] """Valid protocols""" def __init__(self, name=None, container_port=None, host_port=0, service_port=None, protocol='tcp', labels=None): From 7a83f3bd46cc9f70d2418bf3854fc50b93ddd7f9 Mon Sep 17 00:00:00 2001 From: Guanglu Guo Date: Fri, 21 Jul 2017 14:46:38 +0800 Subject: [PATCH 208/292] Enable TCP keepalive for sse requests With TCP keepalive client can get exception if server fail silently or network partition. --- marathon/client.py | 13 +++++++++++-- setup.py | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index 8c72042..6a5269e 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -8,6 +8,7 @@ import requests import requests.exceptions +from requests_toolbelt.adapters import socket_options import marathon from .models import MarathonApp, MarathonDeployment, MarathonGroup, MarathonInfo, MarathonTask, MarathonEndpoint, MarathonQueueItem @@ -21,7 +22,7 @@ class MarathonClient(object): """Client interface for the Marathon REST API.""" def __init__(self, servers, username=None, password=None, timeout=10, session=None, - auth_token=None, verify=True): + auth_token=None, verify=True, sse_session=None): """Create a MarathonClient instance. If multiple servers are specified, each will be tried in succession until a non-"Connection Error"-type @@ -36,11 +37,19 @@ def __init__(self, servers, username=None, password=None, timeout=10, session=No :param int timeout: Timeout (in seconds) for requests to Marathon :param str auth_token: Token-based auth token, used with DCOS + Oauth :param bool verify: Enable SSL certificate verification + :param requests.session sse_session: requests.session for event stream connections, which by default enables tcp keepalive """ if session is None: self.session = requests.Session() else: self.session = session + if sse_session is None: + self.sse_session = requests.Session() + keep_alive = socket_options.TCPKeepAliveAdapter() + self.sse_session.mount('http://', keep_alive) + self.sse_session.mount('https://', keep_alive) + else: + self.sse_session = sse_session self.servers = servers if isinstance(servers, list) else [servers] self.auth = (username, password) if username and password else None self.verify = verify @@ -114,7 +123,7 @@ def _do_sse_request(self, path, params=None): for server in list(self.servers): url = ''.join([server.rstrip('/'), path]) try: - response = requests.get( + response = self.sse_session.get( url, params=params, stream=True, diff --git a/setup.py b/setup.py index dd8b4a6..67775fe 100755 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ long_description="""Python interface to the Mesos Marathon REST API.""", author='Mike Babineau', author_email='michael.babineau@gmail.com', - install_requires=['requests>=2.0.0'], + install_requires=['requests>=2.4.0', 'requests-toolbelt>=0.4.0'], url='https://github.com/thefactory/marathon-python', packages=['marathon', 'marathon.models'], license='MIT', From 367930f0bdb5b4cddb7be9a7613ed9b2e1f4df7b Mon Sep 17 00:00:00 2001 From: Robert Johnson Date: Fri, 28 Jul 2017 04:09:05 -0700 Subject: [PATCH 209/292] add embed option for /v2/queue --- marathon/client.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index 6a5269e..79dc540 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -668,13 +668,17 @@ def list_deployments(self): response = self._do_request('GET', '/v2/deployments') return self._parse_response(response, MarathonDeployment, is_list=True) - def list_queue(self): + def list_queue(self, embed_last_unused_offers=False): """List all the tasks queued up or waiting to be scheduled. :returns: list of queue items :rtype: list[:class:`marathon.models.queue.MarathonQueueItem`] """ - response = self._do_request('GET', '/v2/queue') + if embed_last_unused_offers: + params = {'embed': 'lastUnusedOffers'} + else: + params = {} + response = self._do_request('GET', '/v2/queue', params=params) return self._parse_response(response, MarathonQueueItem, is_list=True, resource_name='queue') def delete_deployment(self, deployment_id, force=False): From f904de7169a16deaeacbbed8758c81a085dc695c Mon Sep 17 00:00:00 2001 From: iandyh Date: Thu, 3 Aug 2017 09:49:38 +0900 Subject: [PATCH 210/292] enable filter applications by labels --- marathon/client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/marathon/client.py b/marathon/client.py index 79dc540..28b3ea3 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -166,7 +166,7 @@ def create_app(self, app_id, app): def list_apps(self, cmd=None, embed_tasks=False, embed_counts=False, embed_deployments=False, embed_readiness=False, embed_last_task_failure=False, embed_failures=False, - embed_task_stats=False, app_id=None, **kwargs): + embed_task_stats=False, app_id=None, label=None, **kwargs): """List all apps. :param str cmd: if passed, only show apps with a matching `cmd` @@ -178,6 +178,7 @@ def list_apps(self, cmd=None, embed_tasks=False, embed_counts=False, :param bool embed_failures: shorthand for embed_last_task_failure :param bool embed_task_stats: embed task stats in result :param str app_id: if passed, only show apps with an 'id' that matches or contains this value + :param str label: if passed, only show apps with the selected labels :param kwargs: arbitrary search filters :returns: list of applications @@ -188,6 +189,8 @@ def list_apps(self, cmd=None, embed_tasks=False, embed_counts=False, params['cmd'] = cmd if app_id: params['id'] = app_id + if label: + params['label'] = label embed_params = { 'app.tasks': embed_tasks, From b2e6a14aaa42396c3dadcc2de1a996c988721791 Mon Sep 17 00:00:00 2001 From: iandyh Date: Thu, 3 Aug 2017 09:51:45 +0900 Subject: [PATCH 211/292] pep8 --- marathon/client.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index 28b3ea3..2616711 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -131,7 +131,8 @@ def _do_sse_request(self, path, params=None): auth=self.auth ) except Exception as e: - marathon.log.error('Error while calling %s: %s', url, e.message) + marathon.log.error( + 'Error while calling %s: %s', url, e.message) else: if response.ok: return response.iter_lines() @@ -450,7 +451,8 @@ def rollback_group(self, group_id, version, force=False): params = {'force': force} response = self._do_request( 'PUT', - '/v2/groups/{group_id}/versions/{version}'.format(group_id=group_id, version=version), + '/v2/groups/{group_id}/versions/{version}'.format( + group_id=group_id, version=version), params=params) return response.json() @@ -764,7 +766,8 @@ def event_stream(self, raw=False, event_types=None): params = { 'event_type': [ - EventFactory.class_to_event[et] if isinstance(et, type) and issubclass(et, MarathonEvent) else et + EventFactory.class_to_event[et] if isinstance( + et, type) and issubclass(et, MarathonEvent) else et for et in event_types or [] ] } From e8520f99832844c72e75db7d5728792b9f0fb23d Mon Sep 17 00:00:00 2001 From: David Zisky Date: Mon, 28 Aug 2017 11:54:32 +0200 Subject: [PATCH 212/292] Update container.py Added USER network mode (for example calico) --- marathon/models/container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marathon/models/container.py b/marathon/models/container.py index a910b41..afccffe 100644 --- a/marathon/models/container.py +++ b/marathon/models/container.py @@ -49,7 +49,7 @@ class MarathonDockerContainer(MarathonObject): :param bool force_pull_image: Force a docker pull before launching """ - NETWORK_MODES = ['BRIDGE', 'HOST', 'NONE'] + NETWORK_MODES = ['BRIDGE', 'HOST', 'USER', 'NONE'] """Valid network modes""" def __init__(self, image=None, network='HOST', port_mappings=None, parameters=None, privileged=None, From b8684ab9cffb3c09fe0dba271dfa8c768bcdc5c2 Mon Sep 17 00:00:00 2001 From: Jonathan Meyer Date: Fri, 1 Sep 2017 18:15:55 +0000 Subject: [PATCH 213/292] Fix for Marathon 1.5 breaking the /v2/apps API moving portMappings --- marathon/models/container.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/marathon/models/container.py b/marathon/models/container.py index afccffe..586bf40 100644 --- a/marathon/models/container.py +++ b/marathon/models/container.py @@ -11,6 +11,8 @@ class MarathonContainer(MarathonObject): :param docker: docker field (e.g., {"image": "mygroup/myimage"})' :type docker: :class:`marathon.models.container.MarathonDockerContainer` or dict :param str type: + :param port_mappings: New in Marathon v1.5. container.docker.port_mappings moved here. + :type port_mappings: list[:class:`marathon.models.container.MarathonContainerPortMapping`] or list[dict] :param volumes: :type volumes: list[:class:`marathon.models.container.MarathonContainerVolume`] or list[dict] """ @@ -18,11 +20,20 @@ class MarathonContainer(MarathonObject): TYPES = ['DOCKER', 'MESOS'] """Valid container types""" - def __init__(self, docker=None, type='DOCKER', volumes=None): + def __init__(self, docker=None, type='DOCKER', port_mappings=None, volumes=None): if type not in self.TYPES: raise InvalidChoiceError('type', type, self.TYPES) self.type = type + # Marathon v1.5 moved portMappings from within container.docker object directly + # under the container object + if port_mappings: + self.port_mappings = [ + pm if isinstance( + pm, MarathonContainerPortMapping) else MarathonContainerPortMapping().from_json(pm) + for pm in (port_mappings or []) + ] + if docker: self.docker = docker if isinstance(docker, MarathonDockerContainer) \ else MarathonDockerContainer().from_json(docker) From 5ce5adb4d0a9f9a27f808ee7436fd284279dfe8e Mon Sep 17 00:00:00 2001 From: Jonathan Meyer Date: Wed, 6 Sep 2017 12:20:17 -0400 Subject: [PATCH 214/292] Removed zookeeper start from build as it was unneeded and caused failure --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a478dbf..d24d1ec 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,6 @@ install: script: - make test - ./itests/install-marathon.sh - - /etc/init.d/zookeeper start - ./itests/start-marathon.sh & - make itests From b0075f87a18efb78a2d9873d248d5d22f1e390d5 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Wed, 6 Sep 2017 09:55:45 -0700 Subject: [PATCH 215/292] Release 0.9.1 --- CHANGELOG.md | 19 +++++++++++++++++++ setup.py | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 49f24ac..c5eb760 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Change Log +## [0.9.1](https://github.com/thefactory/marathon-python/tree/0.9.1) (2017-09-06) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.9.0...0.9.1) + +**Closed issues:** + +- \_do\_request can raise JSONDecodeError when it means to raise InternalServerError [\#202](https://github.com/thefactory/marathon-python/issues/202) +- marathon.exceptions.InvalidChoiceError: Invalid choice "tcp,udp" for param "protocol". Must be one of \['tcp', 'udp'\] [\#150](https://github.com/thefactory/marathon-python/issues/150) + +**Merged pull requests:** + +- Fix for Marathon 1.5 breaking the /v2/apps API moving portMappings [\#213](https://github.com/thefactory/marathon-python/pull/213) ([gisjedi](https://github.com/gisjedi)) +- Update container.py [\#212](https://github.com/thefactory/marathon-python/pull/212) ([DavidZisky](https://github.com/DavidZisky)) +- Support filtering applications by labels [\#211](https://github.com/thefactory/marathon-python/pull/211) ([iandyh](https://github.com/iandyh)) +- add embed option for /v2/queue [\#210](https://github.com/thefactory/marathon-python/pull/210) ([Rob-Johnson](https://github.com/Rob-Johnson)) +- Enable TCP keepalive for sse requests [\#209](https://github.com/thefactory/marathon-python/pull/209) ([fengyehong](https://github.com/fengyehong)) +- Add "udp,tcp" to authorized protocols for containers [\#208](https://github.com/thefactory/marathon-python/pull/208) ([fuegowolf](https://github.com/fuegowolf)) +- Allow event type filter on event stream [\#207](https://github.com/thefactory/marathon-python/pull/207) ([fengyehong](https://github.com/fengyehong)) +- Fix MarathonResource hash as well [\#205](https://github.com/thefactory/marathon-python/pull/205) ([jolynch](https://github.com/jolynch)) + ## [0.9.0](https://github.com/thefactory/marathon-python/tree/0.9.0) (2017-06-21) [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.14...0.9.0) diff --git a/setup.py b/setup.py index 67775fe..5a1ac65 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='marathon', - version='0.9.0', + version='0.9.1', description='Marathon Client Library', long_description="""Python interface to the Mesos Marathon REST API.""", author='Mike Babineau', From c74a4fbfdd2b75bde394e2c5ad5daab182d8975f Mon Sep 17 00:00:00 2001 From: Guanglu Guo Date: Thu, 7 Sep 2017 12:00:39 +0800 Subject: [PATCH 216/292] Fix events --- marathon/models/events.py | 6 +++--- tests/test_model_event.py | 10 ++++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 tests/test_model_event.py diff --git a/marathon/models/events.py b/marathon/models/events.py index 7274e32..bd64b16 100644 --- a/marathon/models/events.py +++ b/marathon/models/events.py @@ -127,7 +127,7 @@ class MarathonUnknownInstanceTerminated(MarathonEvent): KNOWN_ATTRIBUTES = ['instance_id', 'run_spec_id', 'condition'] -class MarathonInstanceHealthChanged(MarathonEvent): +class MarathonInstanceHealthChangedEvent(MarathonEvent): KNOWN_ATTRIBUTES = ['instance_id', 'run_spec_id', 'run_spec_version', 'healthy'] @@ -181,13 +181,13 @@ def __init__(self): 'instance_changed_event': MarathonInstanceChangedEvent, 'unknown_instance_terminated_event': MarathonUnknownInstanceTerminated, 'unhealthy_instance_kill_event': MarathonUnhealthyInstanceKillEvent, - 'instance_health_changed_event': MarathonInstanceChangedEvent, + 'instance_health_changed_event': MarathonInstanceHealthChangedEvent, 'pod_created_event': MarathonPodCreatedEvent, 'pod_updated_event': MarathonPodUpdatedEvent, 'pod_deleted_event': MarathonPodDeletedEvent, } - class_to_event = dict((v, k) for k, v in event_to_class.iteritems()) + class_to_event = dict((v, k) for k, v in event_to_class.items()) def process(self, event): event_type = event['eventType'] diff --git a/tests/test_model_event.py b/tests/test_model_event.py new file mode 100644 index 0000000..ad1efef --- /dev/null +++ b/tests/test_model_event.py @@ -0,0 +1,10 @@ +# encoding: utf-8 + +from marathon.models.events import EventFactory +import unittest + + +class MarathonEventTest(unittest.TestCase): + + def test_event_factory(self): + self.assertEqual(set(EventFactory.event_to_class.keys()), set(EventFactory.class_to_event.values())) From cca3108bbb2dbfcfc5b6e5f1f16ce1d6c48c68ca Mon Sep 17 00:00:00 2001 From: Nathan Handler Date: Fri, 8 Sep 2017 09:14:32 -0700 Subject: [PATCH 217/292] Test against the latest marathon 1.4 point release (1.4.7) --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d24d1ec..168f551 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ env: - - MARATHONVERSION: 1.4.2 + - MARATHONVERSION: 1.4.7 - MARATHONVERSION: 1.3.0 - MARATHONVERSION: 1.1.2 - MARATHONVERSION: 0.15.3 From 1a6edd5d543ab3f2ac2c31f2ef0b1841131faa18 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Wed, 13 Sep 2017 11:14:53 -0700 Subject: [PATCH 218/292] Release 0.9.2. Fixes #217 --- CHANGELOG.md | 13 +++++++++++++ setup.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5eb760..9b979c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Change Log +## [0.9.2](https://github.com/thefactory/marathon-python/tree/0.9.2) (2017-09-13) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.9.1...0.9.2) + +**Closed issues:** + +- No support for "USER" network mode. [\#173](https://github.com/thefactory/marathon-python/issues/173) +- YAML support for marathon-cli [\#74](https://github.com/thefactory/marathon-python/issues/74) + +**Merged pull requests:** + +- Test against the latest marathon 1.4 point release \(1.4.7\) [\#215](https://github.com/thefactory/marathon-python/pull/215) ([nhandler](https://github.com/nhandler)) +- Fix events [\#214](https://github.com/thefactory/marathon-python/pull/214) ([fengyehong](https://github.com/fengyehong)) + ## [0.9.1](https://github.com/thefactory/marathon-python/tree/0.9.1) (2017-09-06) [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.9.0...0.9.1) diff --git a/setup.py b/setup.py index 5a1ac65..d111518 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='marathon', - version='0.9.1', + version='0.9.2', description='Marathon Client Library', long_description="""Python interface to the Mesos Marathon REST API.""", author='Mike Babineau', From 03a5b92a6b9f7f7a88636b49553b4a89847362ec Mon Sep 17 00:00:00 2001 From: Guanglu Guo Date: Wed, 13 Sep 2017 11:33:58 +0800 Subject: [PATCH 219/292] Make MarathonZooKeeperConfig compatible with maraton 1.5 --- marathon/models/info.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/marathon/models/info.py b/marathon/models/info.py index 7795585..40d4626 100644 --- a/marathon/models/info.py +++ b/marathon/models/info.py @@ -115,16 +115,20 @@ class MarathonZooKeeperConfig(MarathonObject): :param str zk_session_timeout: :param str zk_state: :param int zk_timeout: + :param int zk_connection_timeout: """ def __init__(self, zk=None, zk_future_timeout=None, zk_hosts=None, zk_max_versions=None, zk_path=None, - zk_session_timeout=None, zk_state=None, zk_timeout=None): + zk_session_timeout=None, zk_state=None, zk_timeout=None, zk_connection_timeout=None): self.zk = zk - self.zk_future_timeout = zk_future_timeout self.zk_hosts = zk_hosts self.zk_path = zk_path self.zk_state = zk_state + self.zk_max_versions = zk_max_versions self.zk_timeout = zk_timeout + self.zk_connection_timeout = zk_connection_timeout + self.zk_future_timeout = zk_future_timeout + self.zk_session_timeout = zk_session_timeout class MarathonHttpConfig(MarathonObject): From 08f8b6cf4cf2d707814711e3a3c2d2a10855ff7d Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 10 Oct 2017 11:57:55 +0100 Subject: [PATCH 220/292] Remove default container.docker.network to allow using MESOS container engine and .networks on newer versions of marathon --- marathon/models/container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marathon/models/container.py b/marathon/models/container.py index 586bf40..9cadedd 100644 --- a/marathon/models/container.py +++ b/marathon/models/container.py @@ -63,7 +63,7 @@ class MarathonDockerContainer(MarathonObject): NETWORK_MODES = ['BRIDGE', 'HOST', 'USER', 'NONE'] """Valid network modes""" - def __init__(self, image=None, network='HOST', port_mappings=None, parameters=None, privileged=None, + def __init__(self, image=None, network=None, port_mappings=None, parameters=None, privileged=None, force_pull_image=None, **kwargs): self.image = image if network: From 2cdc850fc667904b2e4680c2c55459e12ac9a8ae Mon Sep 17 00:00:00 2001 From: Dmitriy Samovskiy Date: Tue, 10 Oct 2017 14:34:01 -0700 Subject: [PATCH 221/292] support more datetime formats in MarathonAppVersionInfo --- marathon/models/app.py | 12 ++++++++++-- tests/test_model_app.py | 10 +++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/marathon/models/app.py b/marathon/models/app.py index 0a8eafc..021d580 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -317,7 +317,10 @@ class MarathonAppVersionInfo(MarathonObject): :param str host: mesos slave running the task """ - DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ' + DATETIME_FORMATS = [ + '%Y-%m-%dT%H:%M:%S.%fZ', + '%Y-%m-%dT%H:%M:%SZ', + ] def __init__(self, last_scaling_at=None, last_config_change_at=None): self.last_scaling_at = self._to_datetime(last_scaling_at) @@ -327,7 +330,12 @@ def _to_datetime(self, timestamp): if (timestamp is None or isinstance(timestamp, datetime)): return timestamp else: - return datetime.strptime(timestamp, self.DATETIME_FORMAT) + for fmt in self.DATETIME_FORMATS: + try: + return datetime.strptime(timestamp, fmt) + except ValueError: + pass + raise ValueError('Unrecognized datetime format: {}'.format(timestamp)) class MarathonTaskStats(MarathonObject): diff --git a/tests/test_model_app.py b/tests/test_model_app.py index adddb4d..345aa8c 100644 --- a/tests/test_model_app.py +++ b/tests/test_model_app.py @@ -1,6 +1,7 @@ # encoding: utf-8 -from marathon.models.app import MarathonApp +from marathon.models.app import MarathonApp, MarathonAppVersionInfo +from datetime import datetime import unittest @@ -24,3 +25,10 @@ def test_add_env_non_empty_dict(self): app.add_env("MY_ENV", "my-value") self.assertDictEqual({"MY_ENV": "my-value", "OTHER_ENV": "other-value"}, app.env) + + def test_version_info_datetime(self): + app_ver_info = MarathonAppVersionInfo() + self.assertEquals(app_ver_info._to_datetime("2017-09-28T00:31:55Z"), datetime(2017, 9, 28, 0, 31, 55)) + self.assertEquals(app_ver_info._to_datetime("2017-09-28T00:31:55.4Z"), datetime(2017, 9, 28, 0, 31, 55, 400000)) + self.assertEquals(app_ver_info._to_datetime("2017-09-28T00:31:55.004Z"), datetime(2017, 9, 28, 0, 31, 55, 4000)) + self.assertEquals(app_ver_info._to_datetime("2017-09-28T00:31:55.00042Z"), datetime(2017, 9, 28, 0, 31, 55, 420)) From 89d076a8f12024edf54fff442b5a8ccb49c2b3fc Mon Sep 17 00:00:00 2001 From: Matthew Bentley Date: Thu, 12 Oct 2017 09:18:05 -0700 Subject: [PATCH 222/292] Fix MarathonQueueItem to know about the possible last_unused_offers arg --- marathon/models/queue.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/marathon/models/queue.py b/marathon/models/queue.py index 33c0228..d53fd70 100644 --- a/marathon/models/queue.py +++ b/marathon/models/queue.py @@ -26,7 +26,7 @@ class MarathonQueueItem(MarathonResource): """ def __init__(self, app=None, overdue=None, count=None, delay=None, since=None, - processed_offers_summary=None): + processed_offers_summary=None, last_unused_offers=None): self.app = app if isinstance( app, MarathonApp) else MarathonApp().from_json(app) self.overdue = overdue @@ -35,6 +35,7 @@ def __init__(self, app=None, overdue=None, count=None, delay=None, since=None, delay, MarathonQueueItemDelay) else MarathonQueueItemDelay().from_json(delay) self.since = since self.processed_offers_summary = processed_offers_summary + self.last_unused_offers = last_unused_offers class MarathonQueueItemDelay(MarathonResource): From ee31446d6b1d9610490a846ef434c60992833133 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Mon, 16 Oct 2017 11:35:50 -0700 Subject: [PATCH 223/292] Make travis automatically upload to pypi on new tags --- .travis.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 168f551..b3d19f7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,4 +28,14 @@ addons: packages: - libstdc++6-4.7-dev -sudo: required # make it explicit: it was by default only because this repo was set up before 2015 (new forks need it) +sudo: required + +deploy: + - provider: pypi + user: yelplabs + password: + secure: "Wl8GWxsfPy4KoORYH26N3FllvMeWrifzeCbEx2Af4corcBQl43heeiFRRTlUOcSX0TIasER21PUvQ0R0cAgCjfknDb3SOROcRtcSBe16+cMmvwysfxcAx2OcF1UYBPY8e/qOsGge2Zyzx2PAPNEmJoWKbIT3vUJ4WvlLVeGYdJ0=" + on: + tags: true + condition: MARATHONVERSION == "1.4.7" + repo: thefactory/marathon-python From a55293279a77f2c70cdd978c282ee0cdc28852d4 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Mon, 16 Oct 2017 13:25:37 -0700 Subject: [PATCH 224/292] Release 0.9.3 --- CHANGELOG.md | 16 ++++++++++++++++ setup.py | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b979c5..d5c0a40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,26 @@ # Change Log +## [0.9.3](https://github.com/thefactory/marathon-python/tree/0.9.3) (2017-10-16) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.9.2...0.9.3) + +**Closed issues:** + +- `list\_queue` doesn't like the `embed\_last\_unused\_offers` option [\#220](https://github.com/thefactory/marathon-python/issues/220) + +**Merged pull requests:** + +- Make travis automatically upload to pypi on new tags [\#223](https://github.com/thefactory/marathon-python/pull/223) ([solarkennedy](https://github.com/solarkennedy)) +- Fix MarathonQueueItem to know about the possible last\_unused\_offers arg [\#221](https://github.com/thefactory/marathon-python/pull/221) ([matthewbentley](https://github.com/matthewbentley)) +- support more datetime formats in MarathonAppVersionInfo [\#219](https://github.com/thefactory/marathon-python/pull/219) ([somic](https://github.com/somic)) +- Remove default container.docker.network [\#218](https://github.com/thefactory/marathon-python/pull/218) ([protetore](https://github.com/protetore)) +- Make MarathonZooKeeperConfig compatible with maraton 1.5 [\#216](https://github.com/thefactory/marathon-python/pull/216) ([fengyehong](https://github.com/fengyehong)) + ## [0.9.2](https://github.com/thefactory/marathon-python/tree/0.9.2) (2017-09-13) [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.9.1...0.9.2) **Closed issues:** +- Failed to import marathon in python3 [\#217](https://github.com/thefactory/marathon-python/issues/217) - No support for "USER" network mode. [\#173](https://github.com/thefactory/marathon-python/issues/173) - YAML support for marathon-cli [\#74](https://github.com/thefactory/marathon-python/issues/74) diff --git a/setup.py b/setup.py index d111518..5f0f2d5 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='marathon', - version='0.9.2', + version='0.9.3', description='Marathon Client Library', long_description="""Python interface to the Mesos Marathon REST API.""", author='Mike Babineau', From d0c147df0432c19c22bca8c938d86ac337f9609c Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Mon, 16 Oct 2017 15:15:08 -0700 Subject: [PATCH 225/292] Release 0.9.3 for real --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b3d19f7..fa43944 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,5 +37,5 @@ deploy: secure: "Wl8GWxsfPy4KoORYH26N3FllvMeWrifzeCbEx2Af4corcBQl43heeiFRRTlUOcSX0TIasER21PUvQ0R0cAgCjfknDb3SOROcRtcSBe16+cMmvwysfxcAx2OcF1UYBPY8e/qOsGge2Zyzx2PAPNEmJoWKbIT3vUJ4WvlLVeGYdJ0=" on: tags: true - condition: MARATHONVERSION == "1.4.7" + condition: $MARATHONVERSION == "1.4.7" repo: thefactory/marathon-python From d178cd41a83aef2e12f3e6a4429c50ebffeeb593 Mon Sep 17 00:00:00 2001 From: Dalton Barreto Date: Thu, 26 Oct 2017 15:08:12 -0200 Subject: [PATCH 226/292] Removes id validation from MarathonGroup() This validation was preventig the use of the root group (`/`), both from `MarathonGroup().from_json()` and `MarathonClient().get_group("/")` Fixes issue #227 --- marathon/models/group.py | 2 +- tests/test_model_group.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 tests/test_model_group.py diff --git a/marathon/models/group.py b/marathon/models/group.py index c7c0339..5c946c6 100644 --- a/marathon/models/group.py +++ b/marathon/models/group.py @@ -36,5 +36,5 @@ def __init__(self, apps=None, dependencies=None, # p if isinstance(p, MarathonPod) else MarathonPod().from_json(p) # for p in (pods or []) # ] - self.id = assert_valid_id(id) + self.id = id self.version = version diff --git a/tests/test_model_group.py b/tests/test_model_group.py new file mode 100644 index 0000000..98925b1 --- /dev/null +++ b/tests/test_model_group.py @@ -0,0 +1,20 @@ +# encoding: utf-8 + +from marathon.models.group import MarathonGroup +import unittest + + +class MarathonGroupTest(unittest.TestCase): + + def test_from_json_parses_root_group(self): + data = { + "id": "/", + "groups": [ + {"id": "/foo", "apps": []}, + {"id": "/bla", "apps": []}, + ], + "apps": [] + } + group = MarathonGroup().from_json(data) + self.assertEqual("/", group.id) + From 800f90f334f80fb40c1bd7ce77476615609bdcb6 Mon Sep 17 00:00:00 2001 From: Dalton Barreto Date: Thu, 26 Oct 2017 16:23:07 -0200 Subject: [PATCH 227/292] Fixing flake8 errors --- marathon/models/group.py | 2 +- tests/test_model_group.py | 21 ++++++++++----------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/marathon/models/group.py b/marathon/models/group.py index 5c946c6..bb4f6bc 100644 --- a/marathon/models/group.py +++ b/marathon/models/group.py @@ -1,4 +1,4 @@ -from .base import MarathonResource, assert_valid_id +from .base import MarathonResource from .app import MarathonApp diff --git a/tests/test_model_group.py b/tests/test_model_group.py index 98925b1..fb84c04 100644 --- a/tests/test_model_group.py +++ b/tests/test_model_group.py @@ -7,14 +7,13 @@ class MarathonGroupTest(unittest.TestCase): def test_from_json_parses_root_group(self): - data = { - "id": "/", - "groups": [ - {"id": "/foo", "apps": []}, - {"id": "/bla", "apps": []}, - ], - "apps": [] - } - group = MarathonGroup().from_json(data) - self.assertEqual("/", group.id) - + data = { + "id": "/", + "groups": [ + {"id": "/foo", "apps": []}, + {"id": "/bla", "apps": []}, + ], + "apps": [] + } + group = MarathonGroup().from_json(data) + self.assertEqual("/", group.id) From 015e5014833713bcfd8bea16bd865b41cb0b3954 Mon Sep 17 00:00:00 2001 From: Dalton Barreto Date: Fri, 27 Oct 2017 10:55:36 -0200 Subject: [PATCH 228/292] Fixing E722 flake8 errors E722: do not use bare except --- marathon/models/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/marathon/models/base.py b/marathon/models/base.py index ecc8030..db77076 100644 --- a/marathon/models/base.py +++ b/marathon/models/base.py @@ -13,7 +13,7 @@ def __repr__(self): def __eq__(self, other): try: return self.__dict__ == other.__dict__ - except: + except Exception: return False def __hash__(self): @@ -68,7 +68,7 @@ def __repr__(self): def __eq__(self, other): try: return self.__dict__ == other.__dict__ - except: + except Exception: return False def __hash__(self): From 9043905aa67ce0080d6fdc13144a5990a236669b Mon Sep 17 00:00:00 2001 From: diogommartins Date: Fri, 27 Oct 2017 17:25:07 -0200 Subject: [PATCH 229/292] Adding MarathonConstraint tests --- tests/test_model_constraint.py | 35 ++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 tests/test_model_constraint.py diff --git a/tests/test_model_constraint.py b/tests/test_model_constraint.py new file mode 100644 index 0000000..4b289e4 --- /dev/null +++ b/tests/test_model_constraint.py @@ -0,0 +1,35 @@ +from marathon.models.app import MarathonConstraint +import unittest + + +class MarathonConstraintTests(unittest.TestCase): + def test_repr_with_value(self): + constraint = MarathonConstraint('a_field', 'OPERATOR', 'a_value') + representation = repr(constraint) + self.assertEqual(representation, + "MarathonConstraint::a_field:OPERATOR:a_value") + + def test_repr_without_value(self): + constraint = MarathonConstraint('a_field', 'OPERATOR') + representation = repr(constraint) + self.assertEqual(representation, + "MarathonConstraint::a_field:OPERATOR") + + def test_json_repr_with_value(self): + constraint = MarathonConstraint('a_field', 'OPERATOR', 'a_value') + json_repr = constraint.json_repr() + self.assertEqual(json_repr, ['a_field', 'OPERATOR', 'a_value']) + + def test_json_repr_without_value(self): + constraint = MarathonConstraint('a_field', 'OPERATOR') + json_repr = constraint.json_repr() + self.assertEqual(json_repr, ['a_field', 'OPERATOR']) + + def test_from_json_with_value(self): + constraint = MarathonConstraint.from_json(['a_field', 'OPERATOR', 'a_value']) + self.assertEqual(constraint, + MarathonConstraint('a_field', 'OPERATOR', 'a_value')) + + def test_from_json_without_value(self): + constraint = MarathonConstraint.from_json(['a_field', 'OPERATOR']) + self.assertEqual(constraint, MarathonConstraint('a_field', 'OPERATOR')) From 262c659967d3c2b6f82d3e470c877a905a5518db Mon Sep 17 00:00:00 2001 From: diogommartins Date: Fri, 27 Oct 2017 17:25:27 -0200 Subject: [PATCH 230/292] Adding MarathonConstraint.from_string classmethod tests --- tests/test_model_constraint.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_model_constraint.py b/tests/test_model_constraint.py index 4b289e4..a33417a 100644 --- a/tests/test_model_constraint.py +++ b/tests/test_model_constraint.py @@ -33,3 +33,22 @@ def test_from_json_with_value(self): def test_from_json_without_value(self): constraint = MarathonConstraint.from_json(['a_field', 'OPERATOR']) self.assertEqual(constraint, MarathonConstraint('a_field', 'OPERATOR')) + + def test_from_string_with_value(self): + constraint = MarathonConstraint.from_string('a_field:OPERATOR:a_value') + self.assertEqual(constraint, + MarathonConstraint('a_field', 'OPERATOR', 'a_value')) + + def test_from_string_without_value(self): + constraint = MarathonConstraint.from_string('a_field:OPERATOR') + self.assertEqual(constraint, MarathonConstraint('a_field', 'OPERATOR')) + + def test_from_string_raises_an_error_for_invalid_format(self): + with self.assertRaises(ValueError): + MarathonConstraint.from_string('a_field:OPERATOR:a_value:') + + with self.assertRaises(ValueError): + MarathonConstraint.from_string('a_field') + + with self.assertRaises(ValueError): + MarathonConstraint.from_string('a_field:OPERATOR:a_value:something') From c64e2032c187f69a8717e8dc37d203ab6a7d803e Mon Sep 17 00:00:00 2001 From: diogommartins Date: Fri, 27 Oct 2017 17:25:42 -0200 Subject: [PATCH 231/292] Adding MarathonConstraint.from_string implementation --- marathon/models/constraint.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/marathon/models/constraint.py b/marathon/models/constraint.py index 7b14bf3..83d749a 100644 --- a/marathon/models/constraint.py +++ b/marathon/models/constraint.py @@ -55,3 +55,22 @@ def from_json(cls, obj): if len(obj) > 2: (field, operator, value) = obj return cls(field, operator, value) + + @classmethod + def from_string(cls, constraint): + """ + :param str constraint: The string representation of a constraint + + :rtype: :class:`MarathonConstraint` + """ + parts = constraint.split(':') + + if len(parts) == 2: + (field, operator) = parts + return cls(field, operator) + elif len(parts) > 2: + (field, operator, value) = parts + return cls(field, operator, value) + else: + raise ValueError("Invalid string format. " + "Expected `field:operator:value`") From 2c97a2cbb6c11f8437108d674df5573f97f9e328 Mon Sep 17 00:00:00 2001 From: diogommartins Date: Fri, 27 Oct 2017 18:21:58 -0200 Subject: [PATCH 232/292] Refactoring MarathonConstraint.from_string to use MarathonConstraint.from_json --- marathon/models/constraint.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/marathon/models/constraint.py b/marathon/models/constraint.py index 83d749a..cc6b3f1 100644 --- a/marathon/models/constraint.py +++ b/marathon/models/constraint.py @@ -63,14 +63,11 @@ def from_string(cls, constraint): :rtype: :class:`MarathonConstraint` """ - parts = constraint.split(':') + obj = constraint.split(':') + marathon_constraint = cls.from_json(obj) - if len(parts) == 2: - (field, operator) = parts - return cls(field, operator) - elif len(parts) > 2: - (field, operator, value) = parts - return cls(field, operator, value) - else: - raise ValueError("Invalid string format. " - "Expected `field:operator:value`") + if marathon_constraint: + return marathon_constraint + + raise ValueError("Invalid string format. " + "Expected `field:operator:value`") From d1982b4e2b1efb35bf97ef6c1c94c5110f46a6de Mon Sep 17 00:00:00 2001 From: diogommartins Date: Sat, 28 Oct 2017 13:02:26 -0200 Subject: [PATCH 233/292] Removing whitespaces on blank lines --- marathon/models/constraint.py | 2 +- tests/test_model_constraint.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/marathon/models/constraint.py b/marathon/models/constraint.py index cc6b3f1..cbbde87 100644 --- a/marathon/models/constraint.py +++ b/marathon/models/constraint.py @@ -55,7 +55,7 @@ def from_json(cls, obj): if len(obj) > 2: (field, operator, value) = obj return cls(field, operator, value) - + @classmethod def from_string(cls, constraint): """ diff --git a/tests/test_model_constraint.py b/tests/test_model_constraint.py index a33417a..e8ad1d5 100644 --- a/tests/test_model_constraint.py +++ b/tests/test_model_constraint.py @@ -19,12 +19,12 @@ def test_json_repr_with_value(self): constraint = MarathonConstraint('a_field', 'OPERATOR', 'a_value') json_repr = constraint.json_repr() self.assertEqual(json_repr, ['a_field', 'OPERATOR', 'a_value']) - + def test_json_repr_without_value(self): constraint = MarathonConstraint('a_field', 'OPERATOR') json_repr = constraint.json_repr() self.assertEqual(json_repr, ['a_field', 'OPERATOR']) - + def test_from_json_with_value(self): constraint = MarathonConstraint.from_json(['a_field', 'OPERATOR', 'a_value']) self.assertEqual(constraint, From ab0397bbbf03b411bf06d8a989a4bc492e67a6b4 Mon Sep 17 00:00:00 2001 From: Diego Date: Fri, 10 Nov 2017 10:40:24 +0000 Subject: [PATCH 234/292] Move getLogger to util so it can be used from anywhere without unnecessary imports from marathon/__init__.py --- marathon/__init__.py | 5 ++--- marathon/util.py | 5 +++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/marathon/__init__.py b/marathon/__init__.py index 8d2975c..fdb6e47 100644 --- a/marathon/__init__.py +++ b/marathon/__init__.py @@ -1,7 +1,6 @@ -import logging - from .client import MarathonClient from .models import MarathonResource, MarathonApp, MarathonTask, MarathonConstraint from .exceptions import MarathonError, MarathonHttpError, NotFoundError, InvalidChoiceError +from .util import get_log -log = logging.getLogger(__name__) +log = get_log() diff --git a/marathon/util.py b/marathon/util.py index e999333..571a072 100644 --- a/marathon/util.py +++ b/marathon/util.py @@ -1,5 +1,6 @@ import collections import datetime +import logging try: import json @@ -10,6 +11,10 @@ from ._compat import string_types +def get_log(): + return logging.getLogger(__name__.split('.')[0]) + + def is_stringy(obj): return isinstance(obj, string_types) From c7304bf2b1e79ec0a7f25d938d1bbbe7780697b2 Mon Sep 17 00:00:00 2001 From: Diego Date: Fri, 10 Nov 2017 10:41:48 +0000 Subject: [PATCH 235/292] Create correct command format when receiving a string and issue a deprecation message when receiving a dict --- marathon/models/app.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/marathon/models/app.py b/marathon/models/app.py index 021d580..a142a1e 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -6,6 +6,9 @@ from .container import MarathonContainer from .deployment import MarathonDeployment from .task import MarathonTask +from ..util import is_stringy, get_log + +log = get_log() class MarathonApp(MarathonResource): @@ -210,7 +213,19 @@ class MarathonHealthCheck(MarathonObject): def __init__(self, command=None, grace_period_seconds=None, interval_seconds=None, max_consecutive_failures=None, path=None, port_index=None, protocol=None, timeout_seconds=None, ignore_http1xx=None, **kwargs): - self.command = command + + if is_stringy(command): + self.command = { + "value": command + } + elif type(command) is dict and 'value' in command: + log.warn('Deprecated: Using command as dict instead of string is deprecated') + self.command = { + "value": command['value'] + } + else: + raise ValueError('Invalid command format: {}'.format(command)) + self.grace_period_seconds = grace_period_seconds self.interval_seconds = interval_seconds self.max_consecutive_failures = max_consecutive_failures @@ -318,8 +333,8 @@ class MarathonAppVersionInfo(MarathonObject): """ DATETIME_FORMATS = [ - '%Y-%m-%dT%H:%M:%S.%fZ', - '%Y-%m-%dT%H:%M:%SZ', + '%Y-%m-%dT%H:%M:%S.%fZ', + '%Y-%m-%dT%H:%M:%SZ', ] def __init__(self, last_scaling_at=None, last_config_change_at=None): From 061d485c1fd4e2286f51ed53952baad3679eba45 Mon Sep 17 00:00:00 2001 From: Diego Date: Fri, 10 Nov 2017 14:09:48 +0000 Subject: [PATCH 236/292] Fix healthcheck creation when command is None --- marathon/models/app.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/marathon/models/app.py b/marathon/models/app.py index a142a1e..cadc3d7 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -214,7 +214,9 @@ class MarathonHealthCheck(MarathonObject): def __init__(self, command=None, grace_period_seconds=None, interval_seconds=None, max_consecutive_failures=None, path=None, port_index=None, protocol=None, timeout_seconds=None, ignore_http1xx=None, **kwargs): - if is_stringy(command): + if command is None: + self.command = None + elif is_stringy(command): self.command = { "value": command } From ba6cf7471f31c32e1a20d7be0f7190710b1007d4 Mon Sep 17 00:00:00 2001 From: iandyh Date: Thu, 30 Nov 2017 11:00:07 +0900 Subject: [PATCH 237/292] make models.info compatible with 1.4.9 --- marathon/models/info.py | 66 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 2 deletions(-) diff --git a/marathon/models/info.py b/marathon/models/info.py index 40d4626..ab879f1 100644 --- a/marathon/models/info.py +++ b/marathon/models/info.py @@ -74,13 +74,47 @@ class MarathonConfig(MarathonObject): :param int task_launch_timeout: :param int task_reservation_timeout: :param int marathon_store_timeout: + :param str access_control_allow_origin: + :param int decline_offer_duration: + :param str default_network_name: + :param str env_vars_prefix: + :param int launch_token: + :param int launch_token_refresh_interval: + :param int max_instances_per_offer: + :param int mesos_heartbeat_failure_threshold: + :param int mesos_heartbeat_interval: + :param int min_revive_offers_interval: + :param int offer_matching_timeout: + :param int on_elected_prepare_timeout: + :param bool revive_offers_for_new_apps: + :param int revive_offers_repetitions: + :param int scale_apps_initial_delay: + :param int scale_apps_interval: + :param bool store_cache: + :param int task_launch_confirm_timeout: + :param int task_lost_expunge_initial_delay: + :param int task_lost_expunge_interval: """ def __init__(self, checkpoint=None, executor=None, failover_timeout=None, framework_name=None, ha=None, hostname=None, leader_proxy_connection_timeout_ms=None, leader_proxy_read_timeout_ms=None, local_port_min=None, local_port_max=None, master=None, mesos_leader_ui_url=None, mesos_role=None, mesos_user=None, webui_url=None, reconciliation_initial_delay=None, reconciliation_interval=None, - task_launch_timeout=None, marathon_store_timeout=None, task_reservation_timeout=None, features=None): + task_launch_timeout=None, marathon_store_timeout=None, task_reservation_timeout=None, features=None, + access_control_allow_origin=None, decline_offer_duration=None, + default_network_name=None, env_vars_prefix=None, + launch_token=None, launch_token_refresh_interval=None, + max_instances_per_offer=None, + mesos_heartbeat_failure_threshold=None, + mesos_heartbeat_interval=None, min_revive_offers_interval=None, + offer_matching_timeout=None, on_elected_prepare_timeout=None, + revive_offers_for_new_apps=None, + revive_offers_repetitions=None, scale_apps_initial_delay=None, + scale_apps_interval=None, store_cache=None, + task_launch_confirm_timeout=None, + task_lost_expunge_initial_delay=None, + task_lost_expunge_interval=None + ): self.checkpoint = checkpoint self.executor = executor self.failover_timeout = failover_timeout @@ -99,6 +133,26 @@ def __init__(self, checkpoint=None, executor=None, failover_timeout=None, framew self.task_launch_timeout = task_launch_timeout self.task_reservation_timeout = task_reservation_timeout self.marathon_store_timeout = marathon_store_timeout + self.access_control_allow_origin = access_control_allow_origin + self.decline_offer_duration = decline_offer_duration + self.default_network_name = default_network_name + self.env_vars_prefix = env_vars_prefix + self.launch_token = launch_token + self.launch_token_refresh_interval = launch_token_refresh_interval + self.max_instances_per_offer = max_instances_per_offer + self.mesos_heartbeat_failure_threshold = mesos_heartbeat_failure_threshold + self.mesos_heartbeat_interval = mesos_heartbeat_interval + self.min_revive_offers_interval = min_revive_offers_interval + self.offer_matching_timeout = offer_matching_timeout + self.on_elected_prepare_timeout = on_elected_prepare_timeout + self.revive_offers_for_new_apps = revive_offers_for_new_apps + self.revive_offers_repetitions = revive_offers_repetitions + self.scale_apps_initial_delay = scale_apps_initial_delay + self.scale_apps_interval = scale_apps_interval + self.store_cache = store_cache + self.task_launch_confirm_timeout = task_launch_confirm_timeout + self.task_lost_expunge_initial_delay = task_lost_expunge_initial_delay + self.task_lost_expunge_interval = task_lost_expunge_interval class MarathonZooKeeperConfig(MarathonObject): @@ -116,10 +170,15 @@ class MarathonZooKeeperConfig(MarathonObject): :param str zk_state: :param int zk_timeout: :param int zk_connection_timeout: + :param bool zk_compression: + :param int zk_compression_threshold: + :param int zk_max_node_size: """ def __init__(self, zk=None, zk_future_timeout=None, zk_hosts=None, zk_max_versions=None, zk_path=None, - zk_session_timeout=None, zk_state=None, zk_timeout=None, zk_connection_timeout=None): + zk_session_timeout=None, zk_state=None, zk_timeout=None, zk_connection_timeout=None, + zk_compression=None, zk_compression_threshold=None, + zk_max_node_size=None): self.zk = zk self.zk_hosts = zk_hosts self.zk_path = zk_path @@ -129,6 +188,9 @@ def __init__(self, zk=None, zk_future_timeout=None, zk_hosts=None, zk_max_versio self.zk_connection_timeout = zk_connection_timeout self.zk_future_timeout = zk_future_timeout self.zk_session_timeout = zk_session_timeout + self.zk_compression = zk_compression + self.zk_compression_threshold = zk_compression_threshold + self.zk_max_node_size = zk_max_node_size class MarathonHttpConfig(MarathonObject): From a45eb210cfbe96702b8857283b216abfb5c6153b Mon Sep 17 00:00:00 2001 From: iandyh Date: Thu, 7 Dec 2017 11:07:02 +0900 Subject: [PATCH 238/292] fix pep8 --- marathon/models/info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marathon/models/info.py b/marathon/models/info.py index ab879f1..803a26a 100644 --- a/marathon/models/info.py +++ b/marathon/models/info.py @@ -114,7 +114,7 @@ def __init__(self, checkpoint=None, executor=None, failover_timeout=None, framew task_launch_confirm_timeout=None, task_lost_expunge_initial_delay=None, task_lost_expunge_interval=None - ): + ): self.checkpoint = checkpoint self.executor = executor self.failover_timeout = failover_timeout From 5fbf29659c587e4256b1d7a66360620f3b6a8df6 Mon Sep 17 00:00:00 2001 From: iandyh Date: Mon, 11 Dec 2017 11:12:12 +0900 Subject: [PATCH 239/292] replace 1.4.7 with 1.4.9 in travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fa43944..0cc300c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ env: - - MARATHONVERSION: 1.4.7 + - MARATHONVERSION: 1.4.9 - MARATHONVERSION: 1.3.0 - MARATHONVERSION: 1.1.2 - MARATHONVERSION: 0.15.3 From f3b8cd32a58795554b1a11233e7c19b2c3110ad0 Mon Sep 17 00:00:00 2001 From: Nathan Handler Date: Wed, 3 Jan 2018 13:50:48 -0800 Subject: [PATCH 240/292] Test against 1.4.10 instead of 1.4.9 Let's test against the latest point release in the series. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0cc300c..ad4d4d1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ env: - - MARATHONVERSION: 1.4.9 + - MARATHONVERSION: 1.4.10 - MARATHONVERSION: 1.3.0 - MARATHONVERSION: 1.1.2 - MARATHONVERSION: 0.15.3 From 92ba425dfad5625c26f48c73441cd564547e61b9 Mon Sep 17 00:00:00 2001 From: jack Date: Thu, 11 Jan 2018 11:58:13 +0800 Subject: [PATCH 241/292] fix isuuse-238 --- marathon/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index 2616711..fefc5f7 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -479,9 +479,9 @@ def scale_group(self, group_id, scale_by): :returns: a dict containing the deployment id and version :rtype: dict """ - params = {'scaleBy': scale_by} + data = {'scaleBy': scale_by} response = self._do_request( - 'PUT', '/v2/groups/{group_id}'.format(group_id=group_id), params=params) + 'PUT', '/v2/groups/{group_id}'.format(group_id=group_id), data=json.dumps(data)) return response.json() def list_tasks(self, app_id=None, **kwargs): From fe38bc6046d23bf759cebcdcf2674f28e9e7c242 Mon Sep 17 00:00:00 2001 From: Nathan Handler Date: Wed, 17 Jan 2018 14:56:56 -0800 Subject: [PATCH 242/292] Test against 1.4.11 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ad4d4d1..b034ae5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ env: - - MARATHONVERSION: 1.4.10 + - MARATHONVERSION: 1.4.11 - MARATHONVERSION: 1.3.0 - MARATHONVERSION: 1.1.2 - MARATHONVERSION: 0.15.3 From a7127e80248592ae820947c6c425dd7df866c207 Mon Sep 17 00:00:00 2001 From: "cmg\\mkatica" Date: Wed, 7 Mar 2018 01:38:29 -0500 Subject: [PATCH 243/292] fixes for issue 244 --- marathon/models/container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marathon/models/container.py b/marathon/models/container.py index 9cadedd..0ec267a 100644 --- a/marathon/models/container.py +++ b/marathon/models/container.py @@ -97,7 +97,7 @@ class MarathonContainerPortMapping(MarathonObject): PROTOCOLS = ['tcp', 'udp', 'udp,tcp'] """Valid protocols""" - def __init__(self, name=None, container_port=None, host_port=0, service_port=None, protocol='tcp', labels=None): + def __init__(self, name=None, container_port=None, host_port=None, service_port=None, protocol='tcp', labels=None): self.name = name self.container_port = container_port self.host_port = host_port From 4ef3905a15fcae6df59b8030bd3dcb21c51c7422 Mon Sep 17 00:00:00 2001 From: iandyh Date: Mon, 9 Apr 2018 11:02:31 +0900 Subject: [PATCH 244/292] add reset delay api --- marathon/client.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/marathon/client.py b/marathon/client.py index fefc5f7..590539a 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -707,6 +707,11 @@ def delete_deployment(self, deployment_id, force=False): 'DELETE', '/v2/deployments/{deployment}'.format(deployment=deployment_id)) return response.json() + def reset_delay(self, app_id): + self._do_request( + "DELETE", '/v2/queue/{app_id}/delay'.format(app_id=app_id) + ) + def get_info(self): """Get server configuration information. From 3864a4c3110f53aadac9cc4718ff6969c74e58c0 Mon Sep 17 00:00:00 2001 From: Corentin Chary Date: Fri, 4 May 2018 11:59:49 +0200 Subject: [PATCH 245/292] install-marathon.sh: do not remove oracle-java7-installer oracle-java7-installer is not present on recent travis vms. --- itests/install-marathon.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/itests/install-marathon.sh b/itests/install-marathon.sh index 3ba2a2b..5edab54 100755 --- a/itests/install-marathon.sh +++ b/itests/install-marathon.sh @@ -16,7 +16,6 @@ sudo apt-get update # Install packages sudo DEBIAN_FRONTEND=noninteractive apt-get -y install oracle-java8-installer -sudo apt-get -y purge oracle-java7-installer sudo update-java-alternatives -s java-8-oracle sudo DEBIAN_FRONTEND=noninteractive apt-get install oracle-java8-set-default From cfde0fa24ab680ccbb04c314d1265be3a3a77d2b Mon Sep 17 00:00:00 2001 From: Corentin Chary Date: Fri, 4 May 2018 11:38:21 +0200 Subject: [PATCH 246/292] MarathonClient: set verify when using sse_session See #247 --- marathon/client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/marathon/client.py b/marathon/client.py index 590539a..38f5333 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -128,7 +128,8 @@ def _do_sse_request(self, path, params=None): params=params, stream=True, headers={'Accept': 'text/event-stream'}, - auth=self.auth + auth=self.auth, + verify=self.verify, ) except Exception as e: marathon.log.error( From 027413bc9f951c29655ed7c1b9ba7904f9ddfcc6 Mon Sep 17 00:00:00 2001 From: Corentin Chary Date: Sun, 6 May 2018 19:41:06 +0200 Subject: [PATCH 247/292] events: add a few attributes Found in https://github.com/mesosphere/marathon/blob/master/src/main/scala/mesosphere/marathon/core/event/Events.scala --- marathon/models/events.py | 41 ++++++++++++++++++++++++--------- tests/test_model_event.py | 48 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 76 insertions(+), 13 deletions(-) diff --git a/marathon/models/events.py b/marathon/models/events.py index bd64b16..630e360 100644 --- a/marathon/models/events.py +++ b/marathon/models/events.py @@ -1,10 +1,13 @@ """ This module is used to translate Events from Marathon's EventBus system. -See: https://mesosphere.github.io/marathon/docs/event-bus.html +See: +* https://mesosphere.github.io/marathon/docs/event-bus.html +* https://github.com/mesosphere/marathon/blob/master/src/main/scala/mesosphere/marathon/core/event/Events.scala """ from marathon.models.base import MarathonObject from marathon.models.app import MarathonHealthCheck +from marathon.models.task import MarathonIpAddress from marathon.models.deployment import MarathonDeploymentPlan from marathon.exceptions import MarathonError @@ -19,7 +22,11 @@ class MarathonEvent(MarathonObject): KNOWN_ATTRIBUTES = [] attribute_name_to_marathon_object = { # Allows embedding of MarathonObjects inside events. 'health_check': MarathonHealthCheck, - 'plan': MarathonDeploymentPlan + 'plan': MarathonDeploymentPlan, + 'ip_address': MarathonIpAddress, + } + seq_name_to_singular = { + 'ip_addresses': 'ip_address', } def __init__(self, event_type, timestamp, **kwargs): @@ -28,13 +35,25 @@ def __init__(self, event_type, timestamp, **kwargs): for attribute in self.KNOWN_ATTRIBUTES: self._set(attribute, kwargs.get(attribute)) + def __to_marathon_object(self, attribute_name, attribute): + if attribute_name in self.attribute_name_to_marathon_object: + clazz = self.attribute_name_to_marathon_object[attribute_name] + # If this attribute already has a Marathon object instantiate it. + attribute = clazz.from_json(attribute) + return attribute + def _set(self, attribute_name, attribute): if not attribute: return - if attribute_name in self.attribute_name_to_marathon_object: - clazz = self.attribute_name_to_marathon_object[attribute_name] - attribute = clazz.from_json( - attribute) # If this attribute already has a Marathon object instantiate it. + # Special handling for lists... + if isinstance(attribute, list): + name = self.seq_name_to_singular.get(attribute_name) + attribute = [ + self.__to_marathon_object(name, v) + for v in attribute + ] + else: + attribute = self.__to_marathon_object(attribute_name, attribute) setattr(self, attribute_name, attribute) @@ -44,7 +63,7 @@ class MarathonApiPostEvent(MarathonEvent): class MarathonStatusUpdateEvent(MarathonEvent): KNOWN_ATTRIBUTES = [ - 'slave_id', 'task_id', 'task_status', 'app_id', 'host', 'ports', 'version', 'message'] + 'slave_id', 'task_id', 'task_status', 'app_id', 'host', 'ports', 'version', 'message', 'ip_addresses'] class MarathonFrameworkMessageEvent(MarathonEvent): @@ -68,11 +87,11 @@ class MarathonRemoveHealthCheckEvent(MarathonEvent): class MarathonFailedHealthCheckEvent(MarathonEvent): - KNOWN_ATTRIBUTES = ['app_id', 'health_check', 'task_id'] + KNOWN_ATTRIBUTES = ['app_id', 'health_check', 'task_id', 'instance_id'] class MarathonHealthStatusChangedEvent(MarathonEvent): - KNOWN_ATTRIBUTES = ['app_id', 'health_check', 'task_id', 'alive'] + KNOWN_ATTRIBUTES = ['app_id', 'health_check', 'task_id', 'instance_id', 'alive'] class MarathonGroupChangeSuccess(MarathonEvent): @@ -92,7 +111,7 @@ class MarathonDeploymentFailed(MarathonEvent): class MarathonDeploymentInfo(MarathonEvent): - KNOWN_ATTRIBUTES = ['plan'] + KNOWN_ATTRIBUTES = ['plan', 'current_step'] class MarathonDeploymentStepSuccess(MarathonEvent): @@ -112,7 +131,7 @@ class MarathonEventStreamDetached(MarathonEvent): class MarathonUnhealthyTaskKillEvent(MarathonEvent): - KNOWN_ATTRIBUTES = ['app_id', 'task_id', 'version', 'reason'] + KNOWN_ATTRIBUTES = ['app_id', 'task_id', 'instance_id', 'version', 'reason'] class MarathonAppTerminatedEvent(MarathonEvent): diff --git a/tests/test_model_event.py b/tests/test_model_event.py index ad1efef..e62c0b3 100644 --- a/tests/test_model_event.py +++ b/tests/test_model_event.py @@ -1,10 +1,54 @@ # encoding: utf-8 -from marathon.models.events import EventFactory +from marathon.models.events import EventFactory, MarathonStatusUpdateEvent +from marathon.models.task import MarathonIpAddress import unittest class MarathonEventTest(unittest.TestCase): def test_event_factory(self): - self.assertEqual(set(EventFactory.event_to_class.keys()), set(EventFactory.class_to_event.values())) + self.assertEqual( + set(EventFactory.event_to_class.keys()), + set(EventFactory.class_to_event.values()), + ) + + def test_marathon_event(self): + """Test that we can process at least one kind of event.""" + payload = { + "eventType": "status_update_event", + "slaveId": "slave-01", + "taskId": "task-01", + "taskStatus": "TASK_RUNNING", + "message": "Some message", + "appId": "/foo/bar", + "host": "host-01", + "ipAddresses": [ + {"ip_address": "127.0.0.1", "protocol": "tcp"}, + {"ip_address": "127.0.0.1", "protocol": "udp"}, + ], + "ports": [0, 1], + "version": "1234", + "timestamp": 12345, + } + factory = EventFactory() + event = factory.process(payload) + + expected_event = MarathonStatusUpdateEvent( + event_type="status_update_event", + timestamp=12345, + slave_id="slave-01", + task_id="task-01", + task_status="TASK_RUNNING", + message="Some message", + app_id="/foo/bar", + host="host-01", + ports=[0, 1], + version="1234", + ) + expected_event.ip_addresses = [ + MarathonIpAddress(ip_address="127.0.0.1", protocol="tcp"), + MarathonIpAddress(ip_address="127.0.0.1", protocol="udp"), + ] + + self.assertEqual(event.to_json(), expected_event.to_json()) From 6a2803fe0d18c681ddc189b7bbab30724c51f900 Mon Sep 17 00:00:00 2001 From: Kornel Maleszka Date: Wed, 16 May 2018 12:22:33 +0200 Subject: [PATCH 248/292] Possibility for send the full json object on create --- marathon/client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index 38f5333..dcb6d06 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -148,17 +148,18 @@ def list_endpoints(self): """ return MarathonEndpoint.from_tasks(self.list_tasks()) - def create_app(self, app_id, app): + def create_app(self, app_id, app, minimal=True): """Create and start an app. :param str app_id: application ID :param :class:`marathon.models.app.MarathonApp` app: the application to create + :param bool minimal: ignore nulls and empty collections :returns: the created app (on success) :rtype: :class:`marathon.models.app.MarathonApp` or False """ app.id = app_id - data = app.to_json() + data = app.to_json(minimal=minimal) response = self._do_request('POST', '/v2/apps', data=data) if response.status_code == 201: return self._parse_response(response, MarathonApp) From 941269af413e9f384ba3e31ea83889be4dc97f6e Mon Sep 17 00:00:00 2001 From: Joris De Winne Date: Fri, 3 Aug 2018 14:03:59 -0700 Subject: [PATCH 249/292] Testing with 1.6 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index b034ae5..4c96d9b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,5 @@ env: + - MARATHONVERSION: 1.6.322 - MARATHONVERSION: 1.4.11 - MARATHONVERSION: 1.3.0 - MARATHONVERSION: 1.1.2 From 2e968e5674191eb590dc66fb444ca5cc2f715f23 Mon Sep 17 00:00:00 2001 From: Joris De Winne Date: Fri, 3 Aug 2018 14:11:03 -0700 Subject: [PATCH 250/292] Depending on mesos 1.11.* --- itests/install-marathon.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/itests/install-marathon.sh b/itests/install-marathon.sh index 5edab54..b86ba98 100755 --- a/itests/install-marathon.sh +++ b/itests/install-marathon.sh @@ -19,7 +19,7 @@ sudo DEBIAN_FRONTEND=noninteractive apt-get -y install oracle-java8-installer sudo update-java-alternatives -s java-8-oracle sudo DEBIAN_FRONTEND=noninteractive apt-get install oracle-java8-set-default -sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes install mesos=1.1.* marathon=$MARATHONVERSION* +sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes install mesos=1.11.* marathon=$MARATHONVERSION* # WTF MARATHON? # Why does the precise version have java7 hardcoded if it requires java8? From e49f02f6d53c6e890a2bc32816f1e2cbadce21b1 Mon Sep 17 00:00:00 2001 From: Joris De Winne Date: Fri, 3 Aug 2018 14:25:04 -0700 Subject: [PATCH 251/292] Trying with mesos 1.6 --- itests/install-marathon.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/itests/install-marathon.sh b/itests/install-marathon.sh index b86ba98..6bff172 100755 --- a/itests/install-marathon.sh +++ b/itests/install-marathon.sh @@ -19,7 +19,7 @@ sudo DEBIAN_FRONTEND=noninteractive apt-get -y install oracle-java8-installer sudo update-java-alternatives -s java-8-oracle sudo DEBIAN_FRONTEND=noninteractive apt-get install oracle-java8-set-default -sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes install mesos=1.11.* marathon=$MARATHONVERSION* +sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes install mesos=1.6.* marathon=$MARATHONVERSION* # WTF MARATHON? # Why does the precise version have java7 hardcoded if it requires java8? From 25b6a451ae63ec143bd7566f92c5b73b87e1233e Mon Sep 17 00:00:00 2001 From: Joris De Winne Date: Mon, 6 Aug 2018 09:20:22 -0700 Subject: [PATCH 252/292] Adding mesos_bridge_name --- marathon/models/info.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/marathon/models/info.py b/marathon/models/info.py index 803a26a..fdd68b0 100644 --- a/marathon/models/info.py +++ b/marathon/models/info.py @@ -81,6 +81,7 @@ class MarathonConfig(MarathonObject): :param int launch_token: :param int launch_token_refresh_interval: :param int max_instances_per_offer: + :param str mesos_bridge_name: :param int mesos_heartbeat_failure_threshold: :param int mesos_heartbeat_interval: :param int min_revive_offers_interval: @@ -104,7 +105,7 @@ def __init__(self, checkpoint=None, executor=None, failover_timeout=None, framew access_control_allow_origin=None, decline_offer_duration=None, default_network_name=None, env_vars_prefix=None, launch_token=None, launch_token_refresh_interval=None, - max_instances_per_offer=None, + max_instances_per_offer=None, mesos_bridge_name= None, mesos_heartbeat_failure_threshold=None, mesos_heartbeat_interval=None, min_revive_offers_interval=None, offer_matching_timeout=None, on_elected_prepare_timeout=None, @@ -140,6 +141,7 @@ def __init__(self, checkpoint=None, executor=None, failover_timeout=None, framew self.launch_token = launch_token self.launch_token_refresh_interval = launch_token_refresh_interval self.max_instances_per_offer = max_instances_per_offer + self.mesos_bridge_name = mesos_bridge_name self.mesos_heartbeat_failure_threshold = mesos_heartbeat_failure_threshold self.mesos_heartbeat_interval = mesos_heartbeat_interval self.min_revive_offers_interval = min_revive_offers_interval From eec72ae08a199c843f082970bc25046edb3fd652 Mon Sep 17 00:00:00 2001 From: Joris De Winne Date: Mon, 6 Aug 2018 13:35:37 -0700 Subject: [PATCH 253/292] Using correct marathon startup command for 1.6 --- README.md | 1 + itests/start-marathon.sh | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0822d97..f18704a 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ This is a Python library for interfacing with [Marathon](https://github.com/meso #### Compatibility +* For Marathon 1.6.x, use at least 0.10.0 * For Marathon 1.4.1, use at least 0.8.13 * For Marathon 1.1.1 and 0.15.x, use at least 0.8.1 * For Marathon 0.14.x, use at least 0.7.6 diff --git a/itests/start-marathon.sh b/itests/start-marathon.sh index 6b18743..8be80b9 100755 --- a/itests/start-marathon.sh +++ b/itests/start-marathon.sh @@ -1,12 +1,14 @@ #!/bin/bash if [[ $MARATHONVERSION != '0.8.1' ]]; then - LOGGER="--no-logger" + LOGGER="--logging_level info" else LOGGER="" fi java -version export MESOS_WORK_DIR='/tmp/mesos' +export ZK_HOST=`cat /etc/mesos/zk` + mkdir -p "$MESOS_WORK_DIR" -exec /usr/bin/marathon --master local $LOGGER --hostname localhost +exec /usr/bin/marathon --master $ZK_HOST $LOGGER --hostname localhost From 87263c21ea4f873a7657129251b949a673712491 Mon Sep 17 00:00:00 2001 From: Joris De Winne Date: Mon, 6 Aug 2018 19:41:45 -0700 Subject: [PATCH 254/292] Changing docker compose to expose port --- itests/docker-compose.yml | 2 +- itests/itest_utils.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/itests/docker-compose.yml b/itests/docker-compose.yml index 13a0062..809d456 100644 --- a/itests/docker-compose.yml +++ b/itests/docker-compose.yml @@ -2,4 +2,4 @@ marathon: build: . ports: - - 8080 + - 18080:8080 diff --git a/itests/itest_utils.py b/itests/itest_utils.py index 6b4d17c..f5028fe 100644 --- a/itests/itest_utils.py +++ b/itests/itest_utils.py @@ -63,8 +63,7 @@ def get_marathon_connection_string(): return 'localhost:8080' else: service_port = get_service_internal_port('marathon') - local_port = get_compose_service('marathon').get_container().get_local_port(service_port) - return local_port + return "localhost:%s" % service_port.published def get_service_internal_port(service_name): From 81dcdb93341f14c53ccd0f03ed94b9f5fa13c733 Mon Sep 17 00:00:00 2001 From: Joris De Winne Date: Tue, 7 Aug 2018 20:57:15 -0700 Subject: [PATCH 255/292] Fixing Dockerfile and compose to run Mesos and Marathon on 1 container for testing --- itests/Dockerfile | 2 +- itests/docker-compose.yml | 1 + itests/start-marathon.sh | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/itests/Dockerfile b/itests/Dockerfile index c43b783..b2223c7 100644 --- a/itests/Dockerfile +++ b/itests/Dockerfile @@ -14,6 +14,6 @@ ADD ./marathon-version /root/marathon-version ADD ./install-marathon.sh /root/install-marathon.sh RUN /root/install-marathon.sh -EXPOSE 8080 +EXPOSE 8080 5050 ADD ./start-marathon.sh /root/start-marathon.sh CMD /etc/init.d/zookeeper start && /root/start-marathon.sh diff --git a/itests/docker-compose.yml b/itests/docker-compose.yml index 809d456..064c756 100644 --- a/itests/docker-compose.yml +++ b/itests/docker-compose.yml @@ -3,3 +3,4 @@ marathon: build: . ports: - 18080:8080 + - 15050:5050 diff --git a/itests/start-marathon.sh b/itests/start-marathon.sh index 8be80b9..2c3cc8a 100755 --- a/itests/start-marathon.sh +++ b/itests/start-marathon.sh @@ -11,4 +11,6 @@ export MESOS_WORK_DIR='/tmp/mesos' export ZK_HOST=`cat /etc/mesos/zk` mkdir -p "$MESOS_WORK_DIR" -exec /usr/bin/marathon --master $ZK_HOST $LOGGER --hostname localhost +nohup mesos-master --work_dir=/tmp/mesosmaster --zk=$ZK_HOST --quorum=1 & +nohup mesos-agent --master=$ZK_HOST --work_dir=/tmp/mesosagent --launcher=posix & +exec /usr/bin/marathon --master $ZK_HOST $LOGGER From a1c614116330e7ccf71f4916a3adeba86b46633c Mon Sep 17 00:00:00 2001 From: Joris De Winne Date: Tue, 7 Aug 2018 21:08:25 -0700 Subject: [PATCH 256/292] Removing unneeded space --- marathon/models/info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marathon/models/info.py b/marathon/models/info.py index fdd68b0..5ca62d4 100644 --- a/marathon/models/info.py +++ b/marathon/models/info.py @@ -105,7 +105,7 @@ def __init__(self, checkpoint=None, executor=None, failover_timeout=None, framew access_control_allow_origin=None, decline_offer_duration=None, default_network_name=None, env_vars_prefix=None, launch_token=None, launch_token_refresh_interval=None, - max_instances_per_offer=None, mesos_bridge_name= None, + max_instances_per_offer=None, mesos_bridge_name=None, mesos_heartbeat_failure_threshold=None, mesos_heartbeat_interval=None, min_revive_offers_interval=None, offer_matching_timeout=None, on_elected_prepare_timeout=None, From b321daa573b3d2bc2b4b9a8d19f102093aa33aa9 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Wed, 8 Aug 2018 10:25:22 -0700 Subject: [PATCH 257/292] Remove support for marathon pre 1.0 --- .travis.yml | 5 ----- README.md | 6 +----- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4c96d9b..b4a62a7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,11 +3,6 @@ env: - MARATHONVERSION: 1.4.11 - MARATHONVERSION: 1.3.0 - MARATHONVERSION: 1.1.2 - - MARATHONVERSION: 0.15.3 - - MARATHONVERSION: 0.14.1 - - MARATHONVERSION: 0.13.1 - - MARATHONVERSION: 0.11.1 - - MARATHONVERSION: 0.10.1 language: python python: diff --git a/README.md b/README.md index f18704a..cde7fde 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,7 @@ This is a Python library for interfacing with [Marathon](https://github.com/meso * For Marathon 1.6.x, use at least 0.10.0 * For Marathon 1.4.1, use at least 0.8.13 -* For Marathon 1.1.1 and 0.15.x, use at least 0.8.1 -* For Marathon 0.14.x, use at least 0.7.6 -* For Marathon 0.8.x-0.11.x, use at least marathon-python 0.7.5 -* For Marathon 0.8.x-0.9.x, use as least marathon-python 0.6.11 - 0.7.4 -* For Marathon 0.7.x, use at least marathon-python 0.6.10 +* For Marathon 1.1.1, use at least 0.8.1 * For all version changes, please see `CHANGELOG.md` If you find a feature that is broken, please submit a PR that adds a test for From e0d00510bef973c77a414d61460942e9b62c8635 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Wed, 8 Aug 2018 10:27:12 -0700 Subject: [PATCH 258/292] Remove support for more older versions of marathon --- .travis.yml | 2 +- README.md | 2 +- itests/start-marathon.sh | 6 +----- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index b4a62a7..1f0831d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,5 +33,5 @@ deploy: secure: "Wl8GWxsfPy4KoORYH26N3FllvMeWrifzeCbEx2Af4corcBQl43heeiFRRTlUOcSX0TIasER21PUvQ0R0cAgCjfknDb3SOROcRtcSBe16+cMmvwysfxcAx2OcF1UYBPY8e/qOsGge2Zyzx2PAPNEmJoWKbIT3vUJ4WvlLVeGYdJ0=" on: tags: true - condition: $MARATHONVERSION == "1.4.7" + condition: $MARATHONVERSION == "1.6.322" repo: thefactory/marathon-python diff --git a/README.md b/README.md index cde7fde..592bb92 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ make itests ### Running The Tests Against a Specific Version of Marathon ```bash -MARATHONVERSION=0.9.0 make itests +MARATHONVERSION=1.6.322 make itests ``` ## Documentation diff --git a/itests/start-marathon.sh b/itests/start-marathon.sh index 2c3cc8a..0da2305 100755 --- a/itests/start-marathon.sh +++ b/itests/start-marathon.sh @@ -1,10 +1,6 @@ #!/bin/bash -if [[ $MARATHONVERSION != '0.8.1' ]]; then - LOGGER="--logging_level info" -else - LOGGER="" -fi +LOGGER="--logging_level info" java -version export MESOS_WORK_DIR='/tmp/mesos' From 6d352c8b48b074fdf4bc4845f91d3eded98460c4 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Wed, 8 Aug 2018 10:28:12 -0700 Subject: [PATCH 259/292] Make a new release to 0.10.0 --- CHANGELOG.md | 35 ++++++++++++++++++++++++++++++++--- setup.py | 2 +- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5c0a40..b29b553 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,33 @@ # Change Log +## [0.10.0](https://github.com/thefactory/marathon-python/tree/0.10.0) (2018-08-08) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.9.3...0.10.0) + +**Closed issues:** + +- Travis tests are broken [\#249](https://github.com/thefactory/marathon-python/issues/249) +- SSE SSL authentication not supported [\#247](https://github.com/thefactory/marathon-python/issues/247) +- Lack of support for container.networks [\#243](https://github.com/thefactory/marathon-python/issues/243) +- \_\_init\_\_\(\) got an unexpected keyword argument 'port\_mappings' [\#237](https://github.com/thefactory/marathon-python/issues/237) +- Wrong health check object generated for COMMAND protocol [\#222](https://github.com/thefactory/marathon-python/issues/222) + +**Merged pull requests:** + +- Add support for mesos 1.6 [\#255](https://github.com/thefactory/marathon-python/pull/255) ([jdewinne](https://github.com/jdewinne)) +- Possibility for send the full json object on create [\#252](https://github.com/thefactory/marathon-python/pull/252) ([kkorekk](https://github.com/kkorekk)) +- events: add a few attributes [\#251](https://github.com/thefactory/marathon-python/pull/251) ([iksaif](https://github.com/iksaif)) +- install-marathon.sh: do not remove oracle-java7-installer [\#250](https://github.com/thefactory/marathon-python/pull/250) ([iksaif](https://github.com/iksaif)) +- MarathonClient: set verify when using sse\_session [\#248](https://github.com/thefactory/marathon-python/pull/248) ([iksaif](https://github.com/iksaif)) +- add reset delay api [\#246](https://github.com/thefactory/marathon-python/pull/246) ([iandyh](https://github.com/iandyh)) +- fixes for issue 244 [\#245](https://github.com/thefactory/marathon-python/pull/245) ([mikekatica](https://github.com/mikekatica)) +- Test against 1.4.11 [\#240](https://github.com/thefactory/marathon-python/pull/240) ([nhandler](https://github.com/nhandler)) +- fix isuuse-238 [\#239](https://github.com/thefactory/marathon-python/pull/239) ([yudong2015](https://github.com/yudong2015)) +- Test against 1.4.10 instead of 1.4.9 [\#236](https://github.com/thefactory/marathon-python/pull/236) ([nhandler](https://github.com/nhandler)) +- make models.info compatible with 1.4.9 [\#233](https://github.com/thefactory/marathon-python/pull/233) ([iandyh](https://github.com/iandyh)) +- Fix health check 'command' [\#231](https://github.com/thefactory/marathon-python/pull/231) ([protetore](https://github.com/protetore)) +- Feature/marathon constraint model improvements [\#229](https://github.com/thefactory/marathon-python/pull/229) ([diogommartins](https://github.com/diogommartins)) +- Removes id validation from MarathonGroup\(\) [\#228](https://github.com/thefactory/marathon-python/pull/228) ([daltonmatos](https://github.com/daltonmatos)) + ## [0.9.3](https://github.com/thefactory/marathon-python/tree/0.9.3) (2017-10-16) [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.9.2...0.9.3) @@ -9,6 +37,7 @@ **Merged pull requests:** +- Release 0.9.3 [\#224](https://github.com/thefactory/marathon-python/pull/224) ([solarkennedy](https://github.com/solarkennedy)) - Make travis automatically upload to pypi on new tags [\#223](https://github.com/thefactory/marathon-python/pull/223) ([solarkennedy](https://github.com/solarkennedy)) - Fix MarathonQueueItem to know about the possible last\_unused\_offers arg [\#221](https://github.com/thefactory/marathon-python/pull/221) ([matthewbentley](https://github.com/matthewbentley)) - support more datetime formats in MarathonAppVersionInfo [\#219](https://github.com/thefactory/marathon-python/pull/219) ([somic](https://github.com/somic)) @@ -44,7 +73,7 @@ - Support filtering applications by labels [\#211](https://github.com/thefactory/marathon-python/pull/211) ([iandyh](https://github.com/iandyh)) - add embed option for /v2/queue [\#210](https://github.com/thefactory/marathon-python/pull/210) ([Rob-Johnson](https://github.com/Rob-Johnson)) - Enable TCP keepalive for sse requests [\#209](https://github.com/thefactory/marathon-python/pull/209) ([fengyehong](https://github.com/fengyehong)) -- Add "udp,tcp" to authorized protocols for containers [\#208](https://github.com/thefactory/marathon-python/pull/208) ([fuegowolf](https://github.com/fuegowolf)) +- Add "udp,tcp" to authorized protocols for containers [\#208](https://github.com/thefactory/marathon-python/pull/208) ([alxkt](https://github.com/alxkt)) - Allow event type filter on event stream [\#207](https://github.com/thefactory/marathon-python/pull/207) ([fengyehong](https://github.com/fengyehong)) - Fix MarathonResource hash as well [\#205](https://github.com/thefactory/marathon-python/pull/205) ([jolynch](https://github.com/jolynch)) @@ -362,7 +391,7 @@ **Merged pull requests:** -- Updated to support Marathon 0.9.1 with get\_info\(\) calls [\#59](https://github.com/thefactory/marathon-python/pull/59) ([pyronicide](https://github.com/pyronicide)) +- Updated to support Marathon 0.9.1 with get\_info\(\) calls [\#59](https://github.com/thefactory/marathon-python/pull/59) ([grampelberg](https://github.com/grampelberg)) - Add support for building with a wheel and cleanup setup.py [\#58](https://github.com/thefactory/marathon-python/pull/58) ([mattrobenolt](https://github.com/mattrobenolt)) - travis should run unit tests [\#55](https://github.com/thefactory/marathon-python/pull/55) ([Rob-Johnson](https://github.com/Rob-Johnson)) - implement \_\_eq\_\_ on base models + fix tests to be useful [\#54](https://github.com/thefactory/marathon-python/pull/54) ([Rob-Johnson](https://github.com/Rob-Johnson)) @@ -408,7 +437,7 @@ **Merged pull requests:** - Added forcePullImage parameter for the container model [\#31](https://github.com/thefactory/marathon-python/pull/31) ([solarkennedy](https://github.com/solarkennedy)) -- Quick fix \#29 - add kwargs to MarathonDockerContainer.\_\_init\_\_ [\#30](https://github.com/thefactory/marathon-python/pull/30) ([g----](https://github.com/g----)) +- Quick fix \#29 - add kwargs to MarathonDockerContainer.\_\_init\_\_ [\#30](https://github.com/thefactory/marathon-python/pull/30) ([ghost](https://github.com/ghost)) - Fixed \#26:Using try/except to get rid of use\_2to3 failing [\#27](https://github.com/thefactory/marathon-python/pull/27) ([vitan](https://github.com/vitan)) ## [0.6.13](https://github.com/thefactory/marathon-python/tree/0.6.13) (2015-03-24) diff --git a/setup.py b/setup.py index 5f0f2d5..dac7cb6 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='marathon', - version='0.9.3', + version='0.10.0', description='Marathon Client Library', long_description="""Python interface to the Mesos Marathon REST API.""", author='Mike Babineau', From 72263bd22673f9eeb398ca74e3b2eae9d24da547 Mon Sep 17 00:00:00 2001 From: Guanglu Guo Date: Tue, 6 Nov 2018 11:45:45 +0800 Subject: [PATCH 260/292] Seperate no response error exception --- marathon/client.py | 4 ++-- marathon/exceptions.py | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index dcb6d06..3750472 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -12,7 +12,7 @@ import marathon from .models import MarathonApp, MarathonDeployment, MarathonGroup, MarathonInfo, MarathonTask, MarathonEndpoint, MarathonQueueItem -from .exceptions import InternalServerError, NotFoundError, MarathonHttpError, MarathonError +from .exceptions import InternalServerError, NotFoundError, MarathonHttpError, MarathonError, NoResponseError from .models.events import EventFactory, MarathonEvent from .util import MarathonJsonEncoder, MarathonMinimalJsonEncoder @@ -96,7 +96,7 @@ def _do_request(self, method, path, params=None, data=None): 'Error while calling %s: %s', url, str(e)) if response is None: - raise MarathonError('No remaining Marathon servers to try') + raise NoResponseError('No remaining Marathon servers to try') if response.status_code >= 500: marathon.log.error('Got HTTP {code}: {body}'.format( diff --git a/marathon/exceptions.py b/marathon/exceptions.py index 8d2d249..2697625 100644 --- a/marathon/exceptions.py +++ b/marathon/exceptions.py @@ -40,3 +40,7 @@ def __init__(self, param, value, options): param=param, value=value, options=options ) ) + + +class NoResponseError(MarathonError): + pass From e18f8e358efd1f26c2e8ae59effa9bc3b2e47d16 Mon Sep 17 00:00:00 2001 From: Guanglu Guo Date: Wed, 7 Nov 2018 21:47:32 +0800 Subject: [PATCH 261/292] Seperate conflict error exception --- marathon/client.py | 4 +++- marathon/exceptions.py | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/marathon/client.py b/marathon/client.py index 3750472..7d16356 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -12,7 +12,7 @@ import marathon from .models import MarathonApp, MarathonDeployment, MarathonGroup, MarathonInfo, MarathonTask, MarathonEndpoint, MarathonQueueItem -from .exceptions import InternalServerError, NotFoundError, MarathonHttpError, MarathonError, NoResponseError +from .exceptions import ConflictError, InternalServerError, NotFoundError, MarathonHttpError, MarathonError, NoResponseError from .models.events import EventFactory, MarathonEvent from .util import MarathonJsonEncoder, MarathonMinimalJsonEncoder @@ -107,6 +107,8 @@ def _do_request(self, method, path, params=None, data=None): code=response.status_code, body=response.text.encode('utf-8'))) if response.status_code == 404: raise NotFoundError(response) + elif response.status_code == 409: + raise ConflictError(response) else: raise MarathonHttpError(response) elif response.status_code >= 300: diff --git a/marathon/exceptions.py b/marathon/exceptions.py index 2697625..a5ffe73 100644 --- a/marathon/exceptions.py +++ b/marathon/exceptions.py @@ -32,6 +32,10 @@ class InternalServerError(MarathonHttpError): pass +class ConflictError(MarathonHttpError): + pass + + class InvalidChoiceError(MarathonError): def __init__(self, param, value, options): From ad4bd03fd194629030badc0c79882c656c652b1e Mon Sep 17 00:00:00 2001 From: Jonathan Meyer Date: Thu, 27 Dec 2018 15:34:50 -0500 Subject: [PATCH 262/292] Added region and zone members to task model. Missing region and zone members resulted in stack trace when making calls to get_app under DCOS EE 1.11+ --- marathon/models/task.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/marathon/models/task.py b/marathon/models/task.py index 6b81f74..c756e12 100644 --- a/marathon/models/task.py +++ b/marathon/models/task.py @@ -21,12 +21,17 @@ class MarathonTask(MarathonResource): :param started_at: when this task was started :type started_at: datetime or str :param str version: app version with which this task was started + :type region: str + :param region: fault domain region support in DCOS EE + :type zone: str + :param zone: fault domain zone support in DCOS EE """ DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ' def __init__(self, app_id=None, health_check_results=None, host=None, id=None, ports=None, service_ports=None, - slave_id=None, staged_at=None, started_at=None, version=None, ip_addresses=[], state=None, local_volumes=None): + slave_id=None, staged_at=None, started_at=None, version=None, ip_addresses=[], state=None, local_volumes=None, + region=None, zone=None): self.app_id = app_id self.health_check_results = health_check_results or [] self.health_check_results = [ @@ -50,6 +55,8 @@ def __init__(self, app_id=None, health_check_results=None, host=None, id=None, p ip_addresses, MarathonIpAddress) else MarathonIpAddress().from_json(ipaddr) for ipaddr in (ip_addresses or [])] self.local_volumes = local_volumes or [] + self.region = region + self.zone = zone class MarathonIpAddress(MarathonObject): From 1407aed3011a0a531f50efcf15982ee31486e314 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Tue, 15 Jan 2019 11:10:13 -0800 Subject: [PATCH 263/292] Release 0.11.0 --- CHANGELOG.md | 9 +++++++++ setup.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b29b553..ded5397 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Change Log +## [0.11.0](https://github.com/thefactory/marathon-python/tree/0.11.0) (2019-01-15) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.10.0...0.11.0) + +**Merged pull requests:** + +- Added region and zone members to task model. [\#260](https://github.com/thefactory/marathon-python/pull/260) ([gisjedi](https://github.com/gisjedi)) +- Exception [\#259](https://github.com/thefactory/marathon-python/pull/259) ([fengyehong](https://github.com/fengyehong)) +- New release 0.10.0 [\#256](https://github.com/thefactory/marathon-python/pull/256) ([solarkennedy](https://github.com/solarkennedy)) + ## [0.10.0](https://github.com/thefactory/marathon-python/tree/0.10.0) (2018-08-08) [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.9.3...0.10.0) diff --git a/setup.py b/setup.py index dac7cb6..2e8edfa 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='marathon', - version='0.10.0', + version='0.11.0', description='Marathon Client Library', long_description="""Python interface to the Mesos Marathon REST API.""", author='Mike Babineau', From 9d2df6aae456dd80c0cd9b78972276fab3845fe4 Mon Sep 17 00:00:00 2001 From: Guanglu Guo Date: Tue, 21 Nov 2017 17:43:47 +0800 Subject: [PATCH 264/292] Compatible with event stream redirect behavior. --- marathon/client.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index 7d16356..9e0f2b9 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -122,9 +122,12 @@ def _do_request(self, method, path, params=None, data=None): def _do_sse_request(self, path, params=None): """Query Marathon server for events.""" - for server in list(self.servers): - url = ''.join([server.rstrip('/'), path]) + urls = [''.join([server.rstrip('/'), path]) for server in self.servers] + while urls: + url = urls.pop() try: + # Requests does not set the original Authorization header on cross origin + # redirects. If set allow_redirects=True we may get a 401 response. response = self.sse_session.get( url, params=params, @@ -132,12 +135,16 @@ def _do_sse_request(self, path, params=None): headers={'Accept': 'text/event-stream'}, auth=self.auth, verify=self.verify, + allow_redirects=False ) except Exception as e: marathon.log.error( 'Error while calling %s: %s', url, e.message) else: - if response.ok: + if response.is_redirect and response.next: + urls.append(response.next.url) + marathon.log.debug("Got redirect to {}".format(response.next.url)) + elif response.ok: return response.iter_lines() raise MarathonError('No remaining Marathon servers to try') From 5fcc3d9f0e4dfe5be0e012da3bf1b6a90edba1f8 Mon Sep 17 00:00:00 2001 From: Evan Krall Date: Wed, 13 Nov 2019 13:12:21 -0800 Subject: [PATCH 265/292] Drop support for python2.7, bump python3 support to 3.6 and 3.7 --- .gitignore | 1 + .pre-commit-config.yaml | 5 +++++ .travis.yml | 3 ++- Makefile | 8 +++---- docs/conf.py | 25 ++++++++++----------- itests/steps/marathon_steps.py | 32 +++++++++++++-------------- marathon/_compat.py | 11 ---------- marathon/client.py | 40 +++++++++++++++++----------------- marathon/exceptions.py | 4 ++-- marathon/models/app.py | 10 ++++----- marathon/models/base.py | 6 ++--- marathon/models/events.py | 4 ++-- marathon/util.py | 10 ++------- setup.py | 5 ++--- tests/test_api.py | 12 +++++----- tests/test_model_app.py | 2 -- tests/test_model_event.py | 2 -- tests/test_model_group.py | 2 -- tests/test_model_object.py | 2 -- tox.ini | 16 ++++++++++---- 20 files changed, 94 insertions(+), 106 deletions(-) create mode 100644 .pre-commit-config.yaml delete mode 100644 marathon/_compat.py diff --git a/.gitignore b/.gitignore index 99e9b12..2947fe4 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,4 @@ packer_cache /gh-pages itests/marathon-version +.pytest_cache/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..80e93db --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,5 @@ +- repo: https://github.com/asottile/pyupgrade + rev: v1.25.1 + hooks: + - id: pyupgrade + args: [--py36-plus] \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 1f0831d..c227dfb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,8 @@ env: language: python python: - - 2.7 + - 3.6 + - 3.7 install: - pip install tox script: diff --git a/Makefile b/Makefile index 6c3bbe8..f64cc3e 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,11 @@ itests: - tox -e itest-py27 - tox -e itest-py33 + tox -e itest-py36 + tox -e itest-py37 test: tox -e pep8 - tox -e test-py27 - tox -e test-py33 + tox -e test-py36 + tox -e test-py37 clean: rm -rf dist/ build/ diff --git a/docs/conf.py b/docs/conf.py index e3a0e64..b5611ef 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # marathon-python documentation build configuration file, created by # sphinx-quickstart on Tue Apr 22 11:36:23 2014. @@ -46,8 +45,8 @@ master_doc = 'index' # General information about the project. -project = u'marathon-python' -copyright = u'2014, The Factory' +project = 'marathon-python' +copyright = '2014, The Factory' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -203,8 +202,8 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - ('index', 'marathon-python.tex', u'marathon-python Documentation', - u'Mike Babineau', 'manual'), + ('index', 'marathon-python.tex', 'marathon-python Documentation', + 'Mike Babineau', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -233,8 +232,8 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', 'marathon-python', u'marathon-python Documentation', - [u'Mike Babineau'], 1) + ('index', 'marathon-python', 'marathon-python Documentation', + ['Mike Babineau'], 1) ] # If true, show URL addresses after external links. @@ -247,8 +246,8 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'marathon-python', u'marathon-python Documentation', - u'Mike Babineau', 'marathon-python', 'One line description of project.', + ('index', 'marathon-python', 'marathon-python Documentation', + 'Mike Babineau', 'marathon-python', 'One line description of project.', 'Miscellaneous'), ] @@ -268,10 +267,10 @@ # -- Options for Epub output ---------------------------------------------- # Bibliographic Dublin Core info. -epub_title = u'marathon-python' -epub_author = u'Mike Babineau' -epub_publisher = u'Mike Babineau' -epub_copyright = u'2014, The Factory' +epub_title = 'marathon-python' +epub_author = 'Mike Babineau' +epub_publisher = 'Mike Babineau' +epub_copyright = '2014, The Factory' # The basename for the epub file. It defaults to the project name. #epub_basename = u'marathon-python' diff --git a/itests/steps/marathon_steps.py b/itests/steps/marathon_steps.py index 434bc04..131c199 100644 --- a/itests/steps/marathon_steps.py +++ b/itests/steps/marathon_steps.py @@ -20,18 +20,18 @@ def working_marathon(context): context.client = marathon.MarathonClient(marathon_connection_string) -@then(u'we get the marathon instance\'s info') +@then('we get the marathon instance\'s info') def get_marathon_info(context): assert context.client.get_info() -@when(u'we create a trivial new app') +@when('we create a trivial new app') def create_trivial_new_app(context): context.client.create_app('test-trivial-app', marathon.MarathonApp( cmd='sleep 3600', mem=16, cpus=0.1, instances=5)) -@then(u'we should be able to kill the tasks') +@then('we should be able to kill the tasks') def kill_a_task(context): time.sleep(5) app = context.client.get_app('test-trivial-app') @@ -40,7 +40,7 @@ def kill_a_task(context): app_id='test-trivial-app', task_id=tasks[0].id, scale=True) -@when(u'we create a complex new app') +@when('we create a complex new app') def create_complex_new_app_with_unicode(context): app_config = { 'container': { @@ -51,13 +51,13 @@ def create_complex_new_app_with_unicode(context): 'name': 'myport', 'containerPort': 8888, 'hostPort': 0}], - 'image': u'localhost/fake_docker_url', + 'image': 'localhost/fake_docker_url', 'network': 'BRIDGE', 'parameters': [{'key': 'add-host', 'value': 'google-public-dns-a.google.com:8.8.8.8'}], }, 'volumes': - [{'hostPath': u'/etc/stuff', - 'containerPath': u'/etc/stuff', + [{'hostPath': '/etc/stuff', + 'containerPath': '/etc/stuff', 'mode': 'RO'}], }, 'instances': 1, @@ -68,7 +68,7 @@ def create_complex_new_app_with_unicode(context): 'uris': ['file:///root/.dockercfg'], 'backoff_seconds': 1, 'constraints': None, - 'cmd': u'/bin/true', + 'cmd': '/bin/true', 'health_checks': [ { 'protocol': 'HTTP', @@ -85,13 +85,13 @@ def create_complex_new_app_with_unicode(context): 'test-complex-app', marathon.MarathonApp(**app_config)) -@then(u'we should see the {which} app running via the marathon api') +@then('we should see the {which} app running via the marathon api') def see_complext_app_running(context, which): print(context.client.list_apps()) assert context.client.get_app('test-%s-app' % which) -@when(u'we wait the {which} app deployment finish') +@when('we wait the {which} app deployment finish') def wait_deployment_finish(context, which): while True: time.sleep(1) @@ -100,7 +100,7 @@ def wait_deployment_finish(context, which): break -@then(u'we should be able to kill the #{to_kill} tasks of the {which} app') +@then('we should be able to kill the #{to_kill} tasks of the {which} app') def kill_tasks(context, to_kill, which): app_tasks = context.client.get_app( 'test-%s-app' % which, embed_tasks=True).tasks @@ -111,11 +111,11 @@ def kill_tasks(context, to_kill, which): context.client.kill_given_tasks(task_to_kill) -@then(u'we should be able to list tasks of the {which} app') +@then('we should be able to list tasks of the {which} app') def list_tasks(context, which): app = context.client.get_app('test-%s-app' % which) tasks = context.client.list_tasks('test-%s-app' % which) - assert len(tasks) == app.instances, "we defined %s tasks, got %s tasks" % (app.instances, len(tasks)) + assert len(tasks) == app.instances, "we defined {} tasks, got {} tasks".format(app.instances, len(tasks)) def listen_for_events(client, events): @@ -123,14 +123,14 @@ def listen_for_events(client, events): events.append(msg) -@when(u'marathon version is greater than {version}') +@when('marathon version is greater than {version}') def marathon_version_chech(context, version): info = context.client.get_info() if LooseVersion(info.version) < LooseVersion(version): context.scenario.skip(reason='Marathon version is too low for this scenario') -@when(u'we start listening for events') +@when('we start listening for events') def start_listening_stream(context): manager = multiprocessing.Manager() mlist = manager.list() @@ -141,7 +141,7 @@ def start_listening_stream(context): context.p = p -@then(u'we should see list of events') +@then('we should see list of events') def stop_listening_stream(context): time.sleep(10) context.p.terminate() diff --git a/marathon/_compat.py b/marathon/_compat.py deleted file mode 100644 index b2c19ff..0000000 --- a/marathon/_compat.py +++ /dev/null @@ -1,11 +0,0 @@ -""" -Support for python 2 & 3, ripped pieces from six.py -""" -import sys - -PY3 = sys.version_info[0] == 3 - -if PY3: - string_types = str, -else: - string_types = basestring, diff --git a/marathon/client.py b/marathon/client.py index 9e0f2b9..25782db 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -17,7 +17,7 @@ from .util import MarathonJsonEncoder, MarathonMinimalJsonEncoder -class MarathonClient(object): +class MarathonClient: """Client interface for the Marathon REST API.""" @@ -79,7 +79,7 @@ def _do_request(self, method, path, params=None, data=None): 'Content-Type': 'application/json', 'Accept': 'application/json'} if self.auth_token: - headers['Authorization'] = "token={}".format(self.auth_token) + headers['Authorization'] = f"token={self.auth_token}" response = None servers = list(self.servers) @@ -143,7 +143,7 @@ def _do_sse_request(self, path, params=None): else: if response.is_redirect and response.next: urls.append(response.next.url) - marathon.log.debug("Got redirect to {}".format(response.next.url)) + marathon.log.debug(f"Got redirect to {response.next.url}") elif response.ok: return response.iter_lines() @@ -257,7 +257,7 @@ def get_app(self, app_id, embed_tasks=False, embed_counts=False, params['embed'] = filtered_embed_params response = self._do_request( - 'GET', '/v2/apps/{app_id}'.format(app_id=app_id), params=params) + 'GET', f'/v2/apps/{app_id}', params=params) return self._parse_response(response, MarathonApp, resource_name='app') def restart_app(self, app_id, force=False): @@ -270,7 +270,7 @@ def restart_app(self, app_id, force=False): """ params = {'force': force} response = self._do_request( - 'POST', '/v2/apps/{app_id}/restart'.format(app_id=app_id), params=params) + 'POST', f'/v2/apps/{app_id}/restart', params=params) return response.json() def update_app(self, app_id, app, force=False, minimal=True): @@ -295,7 +295,7 @@ def update_app(self, app_id, app, force=False, minimal=True): data = app.to_json(minimal=minimal) response = self._do_request( - 'PUT', '/v2/apps/{app_id}'.format(app_id=app_id), params=params, data=data) + 'PUT', f'/v2/apps/{app_id}', params=params, data=data) return response.json() def update_apps(self, apps, force=False, minimal=True): @@ -337,7 +337,7 @@ def rollback_app(self, app_id, version, force=False): params = {'force': force} data = json.dumps({'version': version}) response = self._do_request( - 'PUT', '/v2/apps/{app_id}'.format(app_id=app_id), params=params, data=data) + 'PUT', f'/v2/apps/{app_id}', params=params, data=data) return response.json() def delete_app(self, app_id, force=False): @@ -351,7 +351,7 @@ def delete_app(self, app_id, force=False): """ params = {'force': force} response = self._do_request( - 'DELETE', '/v2/apps/{app_id}'.format(app_id=app_id), params=params) + 'DELETE', f'/v2/apps/{app_id}', params=params) return response.json() def scale_app(self, app_id, instances=None, delta=None, force=False): @@ -378,7 +378,7 @@ def scale_app(self, app_id, instances=None, delta=None, force=False): try: app = self.get_app(app_id) except NotFoundError: - marathon.log.error('App "{app}" not found'.format(app=app_id)) + marathon.log.error(f'App "{app_id}" not found') return desired = instances if instances is not None else ( @@ -421,7 +421,7 @@ def get_group(self, group_id): :rtype: :class:`marathon.models.group.MarathonGroup` """ response = self._do_request( - 'GET', '/v2/groups/{group_id}'.format(group_id=group_id)) + 'GET', f'/v2/groups/{group_id}') return self._parse_response(response, MarathonGroup) def update_group(self, group_id, group, force=False, minimal=True): @@ -446,7 +446,7 @@ def update_group(self, group_id, group, force=False, minimal=True): data = group.to_json(minimal=minimal) response = self._do_request( - 'PUT', '/v2/groups/{group_id}'.format(group_id=group_id), data=data, params=params) + 'PUT', f'/v2/groups/{group_id}', data=data, params=params) return response.json() def rollback_group(self, group_id, version, force=False): @@ -478,7 +478,7 @@ def delete_group(self, group_id, force=False): """ params = {'force': force} response = self._do_request( - 'DELETE', '/v2/groups/{group_id}'.format(group_id=group_id), params=params) + 'DELETE', f'/v2/groups/{group_id}', params=params) return response.json() def scale_group(self, group_id, scale_by): @@ -492,7 +492,7 @@ def scale_group(self, group_id, scale_by): """ data = {'scaleBy': scale_by} response = self._do_request( - 'PUT', '/v2/groups/{group_id}'.format(group_id=group_id), data=json.dumps(data)) + 'PUT', f'/v2/groups/{group_id}', data=json.dumps(data)) return response.json() def list_tasks(self, app_id=None, **kwargs): @@ -558,7 +558,7 @@ def batch(iterable, size): if host: params['host'] = host response = self._do_request( - 'DELETE', '/v2/apps/{app_id}/tasks'.format(app_id=app_id), params) + 'DELETE', f'/v2/apps/{app_id}/tasks', params) # Marathon is inconsistent about what type of object it returns on the multi # task deletion endpoint, depending on the version of Marathon. See: # https://github.com/mesosphere/marathon/blob/06a6f763a75fb6d652b4f1660685ae234bd15387/src/main/scala/mesosphere/marathon/api/v2/AppTasksResource.scala#L88-L95 @@ -576,12 +576,12 @@ def batch(iterable, size): # Pause until the tasks have been killed to avoid race # conditions - killed_task_ids = set(t.id for t in killed_tasks) + killed_task_ids = {t.id for t in killed_tasks} running_task_ids = killed_task_ids while killed_task_ids.intersection(running_task_ids): time.sleep(1) - running_task_ids = set( - t.id for t in self.get_app(app_id).tasks) + running_task_ids = { + t.id for t in self.get_app(app_id).tasks} if batch_delay == 0: # Pause until the replacement tasks are healthy @@ -626,7 +626,7 @@ def list_versions(self, app_id): :rtype: list[str] """ response = self._do_request( - 'GET', '/v2/apps/{app_id}/versions'.format(app_id=app_id)) + 'GET', f'/v2/apps/{app_id}/versions') return [version for version in response.json()['versions']] def get_version(self, app_id, version): @@ -715,12 +715,12 @@ def delete_deployment(self, deployment_id, force=False): return {} else: response = self._do_request( - 'DELETE', '/v2/deployments/{deployment}'.format(deployment=deployment_id)) + 'DELETE', f'/v2/deployments/{deployment_id}') return response.json() def reset_delay(self, app_id): self._do_request( - "DELETE", '/v2/queue/{app_id}/delay'.format(app_id=app_id) + "DELETE", f'/v2/queue/{app_id}/delay' ) def get_info(self): diff --git a/marathon/exceptions.py b/marathon/exceptions.py index a5ffe73..0889cb9 100644 --- a/marathon/exceptions.py +++ b/marathon/exceptions.py @@ -14,7 +14,7 @@ def __init__(self, response): self.error_message = content.get('message', self.error_message) self.error_details = content.get('details') self.status_code = response.status_code - super(MarathonHttpError, self).__init__(self.__str__()) + super().__init__(self.__str__()) def __repr__(self): return 'MarathonHttpError: HTTP %s returned with message, "%s"' % \ @@ -39,7 +39,7 @@ class ConflictError(MarathonHttpError): class InvalidChoiceError(MarathonError): def __init__(self, param, value, options): - super(InvalidChoiceError, self).__init__( + super().__init__( 'Invalid choice "{value}" for param "{param}". Must be one of {options}'.format( param=param, value=value, options=options ) diff --git a/marathon/models/app.py b/marathon/models/app.py index cadc3d7..0a3dd5f 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -6,7 +6,7 @@ from .container import MarathonContainer from .deployment import MarathonDeployment from .task import MarathonTask -from ..util import is_stringy, get_log +from ..util import get_log log = get_log() @@ -216,7 +216,7 @@ def __init__(self, command=None, grace_period_seconds=None, interval_seconds=Non if command is None: self.command = None - elif is_stringy(command): + elif isinstance(command, str): self.command = { "value": command } @@ -226,7 +226,7 @@ def __init__(self, command=None, grace_period_seconds=None, interval_seconds=Non "value": command['value'] } else: - raise ValueError('Invalid command format: {}'.format(command)) + raise ValueError(f'Invalid command format: {command}') self.grace_period_seconds = grace_period_seconds self.interval_seconds = interval_seconds @@ -320,7 +320,7 @@ def __init__(self, unreachable_inactive_after_seconds=None, def from_json(cls, attributes): if attributes == cls.DISABLED: return cls.DISABLED - return super(MarathonUnreachableStrategy, cls).from_json(attributes) + return super().from_json(attributes) class MarathonAppVersionInfo(MarathonObject): @@ -352,7 +352,7 @@ def _to_datetime(self, timestamp): return datetime.strptime(timestamp, fmt) except ValueError: pass - raise ValueError('Unrecognized datetime format: {}'.format(timestamp)) + raise ValueError(f'Unrecognized datetime format: {timestamp}') class MarathonTaskStats(MarathonObject): diff --git a/marathon/models/base.py b/marathon/models/base.py index db77076..39aba17 100644 --- a/marathon/models/base.py +++ b/marathon/models/base.py @@ -4,7 +4,7 @@ from marathon.util import to_camel_case, to_snake_case, MarathonJsonEncoder, MarathonMinimalJsonEncoder -class MarathonObject(object): +class MarathonObject: """Base Marathon object.""" def __repr__(self): @@ -61,7 +61,7 @@ class MarathonResource(MarathonObject): def __repr__(self): if 'id' in list(vars(self).keys()): - return "{clazz}::{id}".format(clazz=self.__class__.__name__, id=self.id) + return f"{self.__class__.__name__}::{self.id}" else: return "{clazz}::{obj}".format(clazz=self.__class__.__name__, obj=self.to_json()) @@ -78,7 +78,7 @@ def __hash__(self): return hash(self.to_json()) def __str__(self): - return "{clazz}::".format(clazz=self.__class__.__name__) + str(self.__dict__) + return f"{self.__class__.__name__}::" + str(self.__dict__) # See: diff --git a/marathon/models/events.py b/marathon/models/events.py index 630e360..a0b3d4d 100644 --- a/marathon/models/events.py +++ b/marathon/models/events.py @@ -206,7 +206,7 @@ def __init__(self): 'pod_deleted_event': MarathonPodDeletedEvent, } - class_to_event = dict((v, k) for k, v in event_to_class.items()) + class_to_event = {v: k for k, v in event_to_class.items()} def process(self, event): event_type = event['eventType'] @@ -214,4 +214,4 @@ def process(self, event): clazz = self.event_to_class[event_type] return clazz.from_json(event) else: - raise MarathonError('Unknown event_type: {}, data: {}'.format(event_type, event)) + raise MarathonError(f'Unknown event_type: {event_type}, data: {event}') diff --git a/marathon/util.py b/marathon/util.py index 571a072..539bb5d 100644 --- a/marathon/util.py +++ b/marathon/util.py @@ -8,17 +8,11 @@ import simplejson as json import re -from ._compat import string_types - def get_log(): return logging.getLogger(__name__.split('.')[0]) -def is_stringy(obj): - return isinstance(obj, string_types) - - class MarathonJsonEncoder(json.JSONEncoder): """Custom JSON encoder for Marathon object serialization.""" @@ -30,7 +24,7 @@ def default(self, obj): if isinstance(obj, datetime.datetime): return obj.strftime('%Y-%m-%dT%H:%M:%S.%fZ') - if isinstance(obj, collections.Iterable) and not is_stringy(obj): + if isinstance(obj, collections.Iterable) and not isinstance(obj, str): try: return {k: self.default(v) for k, v in obj.items()} except AttributeError: @@ -50,7 +44,7 @@ def default(self, obj): if isinstance(obj, datetime.datetime): return obj.strftime('%Y-%m-%dT%H:%M:%S.%fZ') - if isinstance(obj, collections.Iterable) and not is_stringy(obj): + if isinstance(obj, collections.Iterable) and not isinstance(obj, str): try: return {k: self.default(v) for k, v in obj.items() if (v or v in (False, 0))} except AttributeError: diff --git a/setup.py b/setup.py index 2e8edfa..9637850 100755 --- a/setup.py +++ b/setup.py @@ -24,10 +24,9 @@ 'Intended Audience :: System Administrators', 'License :: OSI Approved :: MIT License', 'Operating System :: OS Independent', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Programming Language :: Python', 'Topic :: Software Development :: Libraries :: Python Modules' ], diff --git a/tests/test_api.py b/tests/test_api.py index cee709e..3b0ec7d 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -34,7 +34,7 @@ def test_get_deployments_pre_1_0(): mock_client = MarathonClient(servers='http://fake_server') actual_deployments = mock_client.list_deployments() expected_deployments = [models.MarathonDeployment( - id=u"fakeid", + id="fakeid", steps=[ [models.MarathonDeploymentAction( action="ScaleApplication", app="/test")]], @@ -42,8 +42,8 @@ def test_get_deployments_pre_1_0(): action="ScaleApplication", app="/test")], current_step=1, total_steps=1, - affected_apps=[u"/test"], - version=u"fakeversion" + affected_apps=["/test"], + version="fakeversion" )] assert expected_deployments == actual_deployments @@ -90,7 +90,7 @@ def test_get_deployments_post_1_0(): mock_client = MarathonClient(servers='http://fake_server') actual_deployments = mock_client.list_deployments() expected_deployments = [models.MarathonDeployment( - id=u"4d2ff4d8-fbe5-4239-a886-f0831ed68d20", + id="4d2ff4d8-fbe5-4239-a886-f0831ed68d20", steps=[ models.MarathonDeploymentStep( actions=[models.MarathonDeploymentAction( @@ -106,8 +106,8 @@ def test_get_deployments_post_1_0(): ], current_step=2, total_steps=2, - affected_apps=[u"/test-trivial-app"], - version=u"2016-04-20T18:00:20.084Z" + affected_apps=["/test-trivial-app"], + version="2016-04-20T18:00:20.084Z" )] # Helpful for tox to see the diff assert expected_deployments[0].__dict__ == actual_deployments[0].__dict__ diff --git a/tests/test_model_app.py b/tests/test_model_app.py index 345aa8c..b86655c 100644 --- a/tests/test_model_app.py +++ b/tests/test_model_app.py @@ -1,5 +1,3 @@ -# encoding: utf-8 - from marathon.models.app import MarathonApp, MarathonAppVersionInfo from datetime import datetime import unittest diff --git a/tests/test_model_event.py b/tests/test_model_event.py index e62c0b3..22453e3 100644 --- a/tests/test_model_event.py +++ b/tests/test_model_event.py @@ -1,5 +1,3 @@ -# encoding: utf-8 - from marathon.models.events import EventFactory, MarathonStatusUpdateEvent from marathon.models.task import MarathonIpAddress import unittest diff --git a/tests/test_model_group.py b/tests/test_model_group.py index fb84c04..e9fb340 100644 --- a/tests/test_model_group.py +++ b/tests/test_model_group.py @@ -1,5 +1,3 @@ -# encoding: utf-8 - from marathon.models.group import MarathonGroup import unittest diff --git a/tests/test_model_object.py b/tests/test_model_object.py index 89795fb..05c6ee4 100644 --- a/tests/test_model_object.py +++ b/tests/test_model_object.py @@ -1,5 +1,3 @@ -# encoding: utf-8 - from marathon.models.base import MarathonObject from marathon.models.base import MarathonResource import unittest diff --git a/tox.ini b/tox.ini index 7841c6e..300705a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,13 +1,13 @@ [tox] passenv = TRAVIS usedevelop=True -envlist={test,itest}-{py27,py33},pep8 +envlist={test,itest}-{py36,py37},pep8 [testenv] passenv = TRAVIS MARATHONVERSION DOCKER_HOST DOCKER_TLS_VERIFY DOCKER_CERT_PATH DOCKER_MACHINE_NAME basepython = - py27: python2.7 - py33: python3 + py36: python3.6 + py37: python3.7 whitelist_externals=/bin/bash skipsdist=True changedir = @@ -25,10 +25,18 @@ commands = itest: ./itest.sh {posargs} [testenv:pep8] -basepython = python2.7 +basepython = python3.6 deps = flake8 commands = flake8 . [flake8] exclude = .tox,*.egg,docs,build,__init__.py max-line-length = 160 + +[testenv:pre-commit] +basepython = python3.7 +deps = + pre-commit>=1.20.0 +commands = + pre-commit install -f --install-hooks + pre-commit run --all-files \ No newline at end of file From 79322b645e2a056e39f01b6beaffa7d2a14fb695 Mon Sep 17 00:00:00 2001 From: Evan Krall Date: Wed, 13 Nov 2019 14:01:33 -0800 Subject: [PATCH 266/292] Refactor all timestamp parsing into one function. --- marathon/models/app.py | 28 ++++------------------------ marathon/models/task.py | 18 ++++++------------ marathon/util.py | 18 ++++++++++++++++++ tests/test_model_app.py | 10 +--------- tests/test_util.py | 10 +++++++++- 5 files changed, 38 insertions(+), 46 deletions(-) diff --git a/marathon/models/app.py b/marathon/models/app.py index 0a3dd5f..00a3d57 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -1,5 +1,3 @@ -from datetime import datetime - from ..exceptions import InvalidChoiceError from .base import MarathonResource, MarathonObject, assert_valid_path from .constraint import MarathonConstraint @@ -7,6 +5,7 @@ from .deployment import MarathonDeployment from .task import MarathonTask from ..util import get_log +from ..util import to_datetime log = get_log() @@ -256,8 +255,6 @@ class MarathonTaskFailure(MarathonObject): :param str version: app version with which this task was started """ - DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ' - def __init__(self, app_id=None, host=None, message=None, task_id=None, instance_id=None, slave_id=None, state=None, timestamp=None, version=None): self.app_id = app_id @@ -267,8 +264,7 @@ def __init__(self, app_id=None, host=None, message=None, task_id=None, instance_ self.instance_id = instance_id self.slave_id = slave_id self.state = state - self.timestamp = timestamp if (timestamp is None or isinstance(timestamp, datetime)) \ - else datetime.strptime(timestamp, self.DATETIME_FORMAT) + self.timestamp = to_datetime(timestamp) self.version = version @@ -334,25 +330,9 @@ class MarathonAppVersionInfo(MarathonObject): :param str host: mesos slave running the task """ - DATETIME_FORMATS = [ - '%Y-%m-%dT%H:%M:%S.%fZ', - '%Y-%m-%dT%H:%M:%SZ', - ] - def __init__(self, last_scaling_at=None, last_config_change_at=None): - self.last_scaling_at = self._to_datetime(last_scaling_at) - self.last_config_change_at = self._to_datetime(last_config_change_at) - - def _to_datetime(self, timestamp): - if (timestamp is None or isinstance(timestamp, datetime)): - return timestamp - else: - for fmt in self.DATETIME_FORMATS: - try: - return datetime.strptime(timestamp, fmt) - except ValueError: - pass - raise ValueError(f'Unrecognized datetime format: {timestamp}') + self.last_scaling_at = to_datetime(last_scaling_at) + self.last_config_change_at = to_datetime(last_config_change_at) class MarathonTaskStats(MarathonObject): diff --git a/marathon/models/task.py b/marathon/models/task.py index c756e12..e049748 100644 --- a/marathon/models/task.py +++ b/marathon/models/task.py @@ -1,6 +1,5 @@ -from datetime import datetime - from .base import MarathonResource, MarathonObject +from ..util import to_datetime class MarathonTask(MarathonResource): @@ -44,10 +43,8 @@ def __init__(self, app_id=None, health_check_results=None, host=None, id=None, p self.ports = ports or [] self.service_ports = service_ports or [] self.slave_id = slave_id - self.staged_at = staged_at if (staged_at is None or isinstance(staged_at, datetime)) \ - else datetime.strptime(staged_at, self.DATETIME_FORMAT) - self.started_at = started_at if (started_at is None or isinstance(started_at, datetime)) \ - else datetime.strptime(started_at, self.DATETIME_FORMAT) + self.staged_at = to_datetime(staged_at) + self.started_at = to_datetime(started_at) self.state = state self.version = version self.ip_addresses = [ @@ -90,12 +87,9 @@ def __init__(self, alive=None, consecutive_failures=None, first_success=None, last_failure_cause=None, instance_id=None): self.alive = alive self.consecutive_failures = consecutive_failures - self.first_success = first_success if (first_success is None or isinstance(first_success, datetime)) \ - else datetime.strptime(first_success, self.DATETIME_FORMAT) - self.last_failure = last_failure if (last_failure is None or isinstance(last_failure, datetime)) \ - else datetime.strptime(last_failure, self.DATETIME_FORMAT) - self.last_success = last_success if (last_success is None or isinstance(last_success, datetime)) \ - else datetime.strptime(last_success, self.DATETIME_FORMAT) + self.first_success = to_datetime(first_success) + self.last_failure = to_datetime(last_failure) + self.last_success = to_datetime(last_success) self.task_id = task_id self.last_failure_cause = last_failure_cause self.instance_id = instance_id diff --git a/marathon/util.py b/marathon/util.py index 539bb5d..722aa5b 100644 --- a/marathon/util.py +++ b/marathon/util.py @@ -61,3 +61,21 @@ def to_camel_case(snake_str): def to_snake_case(camel_str): s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', camel_str) return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() + + +DATETIME_FORMATS = [ + '%Y-%m-%dT%H:%M:%S.%fZ', + '%Y-%m-%dT%H:%M:%SZ', # Marathon omits milliseconds when they would be .000 +] + + +def to_datetime(timestamp): + if (timestamp is None or isinstance(timestamp, datetime.datetime)): + return timestamp + else: + for fmt in DATETIME_FORMATS: + try: + return datetime.datetime.strptime(timestamp, fmt) + except ValueError: + pass + raise ValueError(f'Unrecognized datetime format: {timestamp}') diff --git a/tests/test_model_app.py b/tests/test_model_app.py index b86655c..b211efe 100644 --- a/tests/test_model_app.py +++ b/tests/test_model_app.py @@ -1,5 +1,4 @@ -from marathon.models.app import MarathonApp, MarathonAppVersionInfo -from datetime import datetime +from marathon.models.app import MarathonApp import unittest @@ -23,10 +22,3 @@ def test_add_env_non_empty_dict(self): app.add_env("MY_ENV", "my-value") self.assertDictEqual({"MY_ENV": "my-value", "OTHER_ENV": "other-value"}, app.env) - - def test_version_info_datetime(self): - app_ver_info = MarathonAppVersionInfo() - self.assertEquals(app_ver_info._to_datetime("2017-09-28T00:31:55Z"), datetime(2017, 9, 28, 0, 31, 55)) - self.assertEquals(app_ver_info._to_datetime("2017-09-28T00:31:55.4Z"), datetime(2017, 9, 28, 0, 31, 55, 400000)) - self.assertEquals(app_ver_info._to_datetime("2017-09-28T00:31:55.004Z"), datetime(2017, 9, 28, 0, 31, 55, 4000)) - self.assertEquals(app_ver_info._to_datetime("2017-09-28T00:31:55.00042Z"), datetime(2017, 9, 28, 0, 31, 55, 420)) diff --git a/tests/test_util.py b/tests/test_util.py index 8956051..6ada31c 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -1,4 +1,5 @@ -from marathon.util import to_camel_case, to_snake_case +from datetime import datetime +from marathon.util import to_camel_case, to_snake_case, to_datetime def _apply_on_pairs(f): @@ -27,3 +28,10 @@ def test(camel, snake): assert to_snake_case(camel) == snake _apply_on_pairs(test) + + +def test_version_info_datetime(): + assert to_datetime("2017-09-28T00:31:55Z") == datetime(2017, 9, 28, 0, 31, 55) + assert to_datetime("2017-09-28T00:31:55.4Z") == datetime(2017, 9, 28, 0, 31, 55, 400000) + assert to_datetime("2017-09-28T00:31:55.004Z") == datetime(2017, 9, 28, 0, 31, 55, 4000) + assert to_datetime("2017-09-28T00:31:55.00042Z") == datetime(2017, 9, 28, 0, 31, 55, 420) From ea6e198ff8c13bcbab7d4db37f70af5330604a88 Mon Sep 17 00:00:00 2001 From: Evan Krall Date: Wed, 13 Nov 2019 14:07:19 -0800 Subject: [PATCH 267/292] Always create tz-aware datetime objects. --- marathon/util.py | 2 +- tests/test_util.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/marathon/util.py b/marathon/util.py index 722aa5b..d9f5664 100644 --- a/marathon/util.py +++ b/marathon/util.py @@ -75,7 +75,7 @@ def to_datetime(timestamp): else: for fmt in DATETIME_FORMATS: try: - return datetime.datetime.strptime(timestamp, fmt) + return datetime.datetime.strptime(timestamp, fmt).replace(tzinfo=datetime.timezone.utc) except ValueError: pass raise ValueError(f'Unrecognized datetime format: {timestamp}') diff --git a/tests/test_util.py b/tests/test_util.py index 6ada31c..2a967ba 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timezone from marathon.util import to_camel_case, to_snake_case, to_datetime @@ -31,7 +31,7 @@ def test(camel, snake): def test_version_info_datetime(): - assert to_datetime("2017-09-28T00:31:55Z") == datetime(2017, 9, 28, 0, 31, 55) - assert to_datetime("2017-09-28T00:31:55.4Z") == datetime(2017, 9, 28, 0, 31, 55, 400000) - assert to_datetime("2017-09-28T00:31:55.004Z") == datetime(2017, 9, 28, 0, 31, 55, 4000) - assert to_datetime("2017-09-28T00:31:55.00042Z") == datetime(2017, 9, 28, 0, 31, 55, 420) + assert to_datetime("2017-09-28T00:31:55Z") == datetime(2017, 9, 28, 0, 31, 55, tzinfo=timezone.utc) + assert to_datetime("2017-09-28T00:31:55.4Z") == datetime(2017, 9, 28, 0, 31, 55, 400000, tzinfo=timezone.utc) + assert to_datetime("2017-09-28T00:31:55.004Z") == datetime(2017, 9, 28, 0, 31, 55, 4000, tzinfo=timezone.utc) + assert to_datetime("2017-09-28T00:31:55.00042Z") == datetime(2017, 9, 28, 0, 31, 55, 420, tzinfo=timezone.utc) From fdf0f38287469919bb193524ff5a760bebcf83a3 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Wed, 13 Nov 2019 17:22:06 -0800 Subject: [PATCH 268/292] Release 0.12.0 --- CHANGELOG.md | 87 ++++++++++++++++++++++++++++++++++++++++++++++------ setup.py | 2 +- 2 files changed, 78 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ded5397..8f9e928 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,20 @@ -# Change Log +# Changelog + +## [Unreleased](https://github.com/thefactory/marathon-python/tree/HEAD) + +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.11.0...HEAD) + +**Closed issues:** + +- Downloading Log For App [\#265](https://github.com/thefactory/marathon-python/issues/265) + +**Merged pull requests:** + +- Always create TZ-aware datetime objects. \(also drop support for python 2\) [\#267](https://github.com/thefactory/marathon-python/pull/267) ([EvanKrall](https://github.com/EvanKrall)) +- Compatible with event stream redirect behavior. [\#262](https://github.com/thefactory/marathon-python/pull/262) ([fengyehong](https://github.com/fengyehong)) ## [0.11.0](https://github.com/thefactory/marathon-python/tree/0.11.0) (2019-01-15) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.10.0...0.11.0) **Merged pull requests:** @@ -10,6 +24,7 @@ - New release 0.10.0 [\#256](https://github.com/thefactory/marathon-python/pull/256) ([solarkennedy](https://github.com/solarkennedy)) ## [0.10.0](https://github.com/thefactory/marathon-python/tree/0.10.0) (2018-08-08) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.9.3...0.10.0) **Closed issues:** @@ -38,6 +53,7 @@ - Removes id validation from MarathonGroup\(\) [\#228](https://github.com/thefactory/marathon-python/pull/228) ([daltonmatos](https://github.com/daltonmatos)) ## [0.9.3](https://github.com/thefactory/marathon-python/tree/0.9.3) (2017-10-16) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.9.2...0.9.3) **Closed issues:** @@ -54,6 +70,7 @@ - Make MarathonZooKeeperConfig compatible with maraton 1.5 [\#216](https://github.com/thefactory/marathon-python/pull/216) ([fengyehong](https://github.com/fengyehong)) ## [0.9.2](https://github.com/thefactory/marathon-python/tree/0.9.2) (2017-09-13) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.9.1...0.9.2) **Closed issues:** @@ -68,6 +85,7 @@ - Fix events [\#214](https://github.com/thefactory/marathon-python/pull/214) ([fengyehong](https://github.com/fengyehong)) ## [0.9.1](https://github.com/thefactory/marathon-python/tree/0.9.1) (2017-09-06) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.9.0...0.9.1) **Closed issues:** @@ -87,6 +105,7 @@ - Fix MarathonResource hash as well [\#205](https://github.com/thefactory/marathon-python/pull/205) ([jolynch](https://github.com/jolynch)) ## [0.9.0](https://github.com/thefactory/marathon-python/tree/0.9.0) (2017-06-21) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.14...0.9.0) **Closed issues:** @@ -111,8 +130,13 @@ - Remove out of date constraint validation of operator. [\#190](https://github.com/thefactory/marathon-python/pull/190) ([akatrevorjay](https://github.com/akatrevorjay)) - Add raw\_data option for event\_stream method [\#189](https://github.com/thefactory/marathon-python/pull/189) ([fengyehong](https://github.com/fengyehong)) - handle case when non-ascii char are logged [\#188](https://github.com/thefactory/marathon-python/pull/188) ([tgermain](https://github.com/tgermain)) +- \[fix\] util.to\_camel\_case doesn't handle digits [\#184](https://github.com/thefactory/marathon-python/pull/184) ([hlerebours](https://github.com/hlerebours)) +- \[fix\] broken build: glibc++ not found [\#183](https://github.com/thefactory/marathon-python/pull/183) ([hlerebours](https://github.com/hlerebours)) +- Support for "disabled" unreachableStrategy. [\#182](https://github.com/thefactory/marathon-python/pull/182) ([nihn](https://github.com/nihn)) +- \[fix\] Handle non-JSON errors from Marathon [\#178](https://github.com/thefactory/marathon-python/pull/178) ([hlerebours](https://github.com/hlerebours)) ## [0.8.14](https://github.com/thefactory/marathon-python/tree/0.8.14) (2017-03-24) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.13...0.8.14) **Closed issues:** @@ -122,14 +146,8 @@ - ignoreHttp1xx or ignoreHttp1Xx [\#125](https://github.com/thefactory/marathon-python/issues/125) - ValueError when 401 Unauthorized is received [\#22](https://github.com/thefactory/marathon-python/issues/22) -**Merged pull requests:** - -- \[fix\] util.to\_camel\_case doesn't handle digits [\#184](https://github.com/thefactory/marathon-python/pull/184) ([hlerebours](https://github.com/hlerebours)) -- \[fix\] broken build: glibc++ not found [\#183](https://github.com/thefactory/marathon-python/pull/183) ([hlerebours](https://github.com/hlerebours)) -- Support for "disabled" unreachableStrategy. [\#182](https://github.com/thefactory/marathon-python/pull/182) ([nihn](https://github.com/nihn)) -- \[fix\] Handle non-JSON errors from Marathon [\#178](https://github.com/thefactory/marathon-python/pull/178) ([hlerebours](https://github.com/hlerebours)) - ## [0.8.13](https://github.com/thefactory/marathon-python/tree/0.8.13) (2017-03-17) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.12...0.8.13) **Merged pull requests:** @@ -137,6 +155,7 @@ - Support processed\_offers\_summary attribute [\#177](https://github.com/thefactory/marathon-python/pull/177) ([nhandler](https://github.com/nhandler)) ## [0.8.12](https://github.com/thefactory/marathon-python/tree/0.8.12) (2017-03-17) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.11...0.8.12) **Closed issues:** @@ -151,6 +170,7 @@ - Updated event.py to handle app\_terminated\_event. [\#171](https://github.com/thefactory/marathon-python/pull/171) ([Jbrownstone](https://github.com/Jbrownstone)) ## [0.8.11](https://github.com/thefactory/marathon-python/tree/0.8.11) (2017-02-22) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.10...0.8.11) **Merged pull requests:** @@ -160,6 +180,7 @@ - Adds MarathonApp.add\_env\(\) method [\#166](https://github.com/thefactory/marathon-python/pull/166) ([daltonmatos](https://github.com/daltonmatos)) ## [0.8.10](https://github.com/thefactory/marathon-python/tree/0.8.10) (2017-01-07) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.9...0.8.10) **Closed issues:** @@ -173,6 +194,7 @@ - Add new Marathon 1.4 API keywords [\#162](https://github.com/thefactory/marathon-python/pull/162) ([stj](https://github.com/stj)) ## [0.8.9](https://github.com/thefactory/marathon-python/tree/0.8.9) (2016-12-15) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.8...0.8.9) **Closed issues:** @@ -184,6 +206,7 @@ - Added more unimplemented Marathon 1.4 API keywords [\#161](https://github.com/thefactory/marathon-python/pull/161) ([solarkennedy](https://github.com/solarkennedy)) ## [0.8.8](https://github.com/thefactory/marathon-python/tree/0.8.8) (2016-12-09) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.7...0.8.8) **Closed issues:** @@ -196,6 +219,7 @@ - Expose error details from response object MarathonHttpError [\#157](https://github.com/thefactory/marathon-python/pull/157) ([moonkev](https://github.com/moonkev)) ## [0.8.7](https://github.com/thefactory/marathon-python/tree/0.8.7) (2016-10-24) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.6...0.8.7) **Closed issues:** @@ -213,6 +237,7 @@ - Add external volume support [\#146](https://github.com/thefactory/marathon-python/pull/146) ([drewrobb](https://github.com/drewrobb)) ## [0.8.6](https://github.com/thefactory/marathon-python/tree/0.8.6) (2016-08-29) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.5...0.8.6) **Closed issues:** @@ -230,6 +255,7 @@ - Add support for unhealthy\_task\_kill\_event [\#137](https://github.com/thefactory/marathon-python/pull/137) ([nuclon](https://github.com/nuclon)) ## [0.8.5](https://github.com/thefactory/marathon-python/tree/0.8.5) (2016-08-10) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.4...0.8.5) **Closed issues:** @@ -244,6 +270,7 @@ - Marathon 1.1.2 and Mesos 1.0.\* [\#134](https://github.com/thefactory/marathon-python/pull/134) ([nhandler](https://github.com/nhandler)) ## [0.8.4](https://github.com/thefactory/marathon-python/tree/0.8.4) (2016-07-20) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.3...0.8.4) **Closed issues:** @@ -256,6 +283,7 @@ - Expose id query param in list\_apps [\#129](https://github.com/thefactory/marathon-python/pull/129) ([moonkev](https://github.com/moonkev)) ## [0.8.3](https://github.com/thefactory/marathon-python/tree/0.8.3) (2016-07-19) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.2...0.8.3) **Closed issues:** @@ -285,6 +313,7 @@ - Add message field for MarathonStatusUpdateEvent. [\#109](https://github.com/thefactory/marathon-python/pull/109) ([oilbeater](https://github.com/oilbeater)) ## [0.8.2](https://github.com/thefactory/marathon-python/tree/0.8.2) (2016-06-14) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.1...0.8.2) **Closed issues:** @@ -304,6 +333,7 @@ - add name attribute to port mapping [\#101](https://github.com/thefactory/marathon-python/pull/101) ([Rob-Johnson](https://github.com/Rob-Johnson)) ## [0.8.1](https://github.com/thefactory/marathon-python/tree/0.8.1) (2016-04-21) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.8.0...0.8.1) **Closed issues:** @@ -318,6 +348,7 @@ - Support the deployments endpoint correctly in marathon 1.1.1 [\#95](https://github.com/thefactory/marathon-python/pull/95) ([solarkennedy](https://github.com/solarkennedy)) ## [0.8.0](https://github.com/thefactory/marathon-python/tree/0.8.0) (2016-04-18) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.7.7...0.8.0) **Closed issues:** @@ -332,6 +363,7 @@ - update for v2/queue and v2/apps?embed=apps.taskStats [\#89](https://github.com/thefactory/marathon-python/pull/89) ([bergerx](https://github.com/bergerx)) ## [0.7.7](https://github.com/thefactory/marathon-python/tree/0.7.7) (2016-02-29) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.7.6...0.7.7) **Merged pull requests:** @@ -340,6 +372,7 @@ - a small fix for fetching apps for marathon v0.15 [\#87](https://github.com/thefactory/marathon-python/pull/87) ([burakbostancioglu](https://github.com/burakbostancioglu)) ## [0.7.6](https://github.com/thefactory/marathon-python/tree/0.7.6) (2016-02-12) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.7.5...0.7.6) **Closed issues:** @@ -359,23 +392,26 @@ - Use the /v2/tasks/delete endpoint for taskkill [\#67](https://github.com/thefactory/marathon-python/pull/67) ([fengyehong](https://github.com/fengyehong)) ## [0.7.5](https://github.com/thefactory/marathon-python/tree/0.7.5) (2015-12-09) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.7.4...0.7.5) **Merged pull requests:** - Release 0.7.5 for official Marathon 11 support [\#73](https://github.com/thefactory/marathon-python/pull/73) ([solarkennedy](https://github.com/solarkennedy)) - Added tests for killing tasks on an app [\#72](https://github.com/thefactory/marathon-python/pull/72) ([solarkennedy](https://github.com/solarkennedy)) +- Use automatic changelog generation [\#69](https://github.com/thefactory/marathon-python/pull/69) ([solarkennedy](https://github.com/solarkennedy)) - Provide proper compatability support for str/unicode in py3 [\#57](https://github.com/thefactory/marathon-python/pull/57) ([mattrobenolt](https://github.com/mattrobenolt)) ## [0.7.4](https://github.com/thefactory/marathon-python/tree/0.7.4) (2015-11-20) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.7.3...0.7.4) **Merged pull requests:** -- Use automatic changelog generation [\#69](https://github.com/thefactory/marathon-python/pull/69) ([solarkennedy](https://github.com/solarkennedy)) - Marathon 11 Support [\#68](https://github.com/thefactory/marathon-python/pull/68) ([solarkennedy](https://github.com/solarkennedy)) ## [0.7.3](https://github.com/thefactory/marathon-python/tree/0.7.3) (2015-11-12) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.7.2...0.7.3) **Closed issues:** @@ -389,6 +425,7 @@ - Remove call to logging.basicConfig [\#64](https://github.com/thefactory/marathon-python/pull/64) ([itamaro](https://github.com/itamaro)) ## [0.7.2](https://github.com/thefactory/marathon-python/tree/0.7.2) (2015-09-18) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.7.0...0.7.2) **Closed issues:** @@ -412,6 +449,7 @@ - First pass at adding an itest framework [\#42](https://github.com/thefactory/marathon-python/pull/42) ([solarkennedy](https://github.com/solarkennedy)) ## [0.7.0](https://github.com/thefactory/marathon-python/tree/0.7.0) (2015-07-06) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.6.15...0.7.0) **Closed issues:** @@ -427,6 +465,7 @@ - Feature/event factory [\#32](https://github.com/thefactory/marathon-python/pull/32) ([kevinschoon](https://github.com/kevinschoon)) ## [0.6.15](https://github.com/thefactory/marathon-python/tree/0.6.15) (2015-06-05) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.6.14...0.6.15) **Merged pull requests:** @@ -434,6 +473,7 @@ - Make `force\_pull\_image` actually work [\#33](https://github.com/thefactory/marathon-python/pull/33) ([mattrobenolt](https://github.com/mattrobenolt)) ## [0.6.14](https://github.com/thefactory/marathon-python/tree/0.6.14) (2015-05-28) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.6.13...0.6.14) **Closed issues:** @@ -450,6 +490,7 @@ - Fixed \#26:Using try/except to get rid of use\_2to3 failing [\#27](https://github.com/thefactory/marathon-python/pull/27) ([vitan](https://github.com/vitan)) ## [0.6.13](https://github.com/thefactory/marathon-python/tree/0.6.13) (2015-03-24) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.6.12...0.6.13) **Merged pull requests:** @@ -460,9 +501,11 @@ - Possibility to send the full object to Marathon on update [\#20](https://github.com/thefactory/marathon-python/pull/20) ([wndhydrnt](https://github.com/wndhydrnt)) ## [0.6.12](https://github.com/thefactory/marathon-python/tree/0.6.12) (2015-03-07) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.6.11...0.6.12) ## [0.6.11](https://github.com/thefactory/marathon-python/tree/0.6.11) (2015-03-06) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.6.10...0.6.11) **Merged pull requests:** @@ -470,6 +513,7 @@ - Small changes to fix compatibility issues with Marathon 0.8.0 [\#19](https://github.com/thefactory/marathon-python/pull/19) ([cloudify](https://github.com/cloudify)) ## [0.6.10](https://github.com/thefactory/marathon-python/tree/0.6.10) (2014-12-17) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.6.8...0.6.10) **Merged pull requests:** @@ -479,9 +523,11 @@ - apparently undocumented API in Marathon [\#16](https://github.com/thefactory/marathon-python/pull/16) ([elyast](https://github.com/elyast)) ## [0.6.8](https://github.com/thefactory/marathon-python/tree/0.6.8) (2014-11-19) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.6.7...0.6.8) ## [0.6.7](https://github.com/thefactory/marathon-python/tree/0.6.7) (2014-11-18) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.6.6...0.6.7) **Closed issues:** @@ -493,6 +539,7 @@ - fixing issues with resources /v2/tasks, v2/info [\#15](https://github.com/thefactory/marathon-python/pull/15) ([elyast](https://github.com/elyast)) ## [0.6.6](https://github.com/thefactory/marathon-python/tree/0.6.6) (2014-11-17) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.6.5...0.6.6) **Closed issues:** @@ -500,9 +547,11 @@ - scale\_app\(...\) calls update\_app\(...\) with only 1 argument [\#13](https://github.com/thefactory/marathon-python/issues/13) ## [0.6.5](https://github.com/thefactory/marathon-python/tree/0.6.5) (2014-11-14) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.6.4...0.6.5) ## [0.6.4](https://github.com/thefactory/marathon-python/tree/0.6.4) (2014-11-14) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.6.3...0.6.4) **Merged pull requests:** @@ -510,6 +559,7 @@ - Add MarathonHealthCheckResult Class to tasks File and Include it in MarathonTask [\#12](https://github.com/thefactory/marathon-python/pull/12) ([JTCunning](https://github.com/JTCunning)) ## [0.6.3](https://github.com/thefactory/marathon-python/tree/0.6.3) (2014-10-10) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.6.2...0.6.3) **Merged pull requests:** @@ -517,6 +567,7 @@ - add service\_port argument [\#11](https://github.com/thefactory/marathon-python/pull/11) ([danielfrg](https://github.com/danielfrg)) ## [0.6.2](https://github.com/thefactory/marathon-python/tree/0.6.2) (2014-10-09) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.6.1...0.6.2) **Merged pull requests:** @@ -524,9 +575,11 @@ - Add `LIKE` and `UNLIKE` constraint [\#10](https://github.com/thefactory/marathon-python/pull/10) ([iven](https://github.com/iven)) ## [0.6.1](https://github.com/thefactory/marathon-python/tree/0.6.1) (2014-09-29) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.6.0...0.6.1) ## [0.6.0](https://github.com/thefactory/marathon-python/tree/0.6.0) (2014-09-29) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.5.1...0.6.0) **Closed issues:** @@ -534,9 +587,11 @@ - Support for HA nodes [\#8](https://github.com/thefactory/marathon-python/issues/8) ## [0.5.1](https://github.com/thefactory/marathon-python/tree/0.5.1) (2014-09-18) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.5.0...0.5.1) ## [0.5.0](https://github.com/thefactory/marathon-python/tree/0.5.0) (2014-09-18) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.4.0...0.5.0) **Merged pull requests:** @@ -544,6 +599,7 @@ - Bug Fix: Cannot define constraints with a tuple of strings [\#6](https://github.com/thefactory/marathon-python/pull/6) ([adgaudio](https://github.com/adgaudio)) ## [0.4.0](https://github.com/thefactory/marathon-python/tree/0.4.0) (2014-08-19) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.3.1...0.4.0) **Merged pull requests:** @@ -552,6 +608,7 @@ - Fix container options not being sent to marathon [\#4](https://github.com/thefactory/marathon-python/pull/4) ([boffbowsh](https://github.com/boffbowsh)) ## [0.3.1](https://github.com/thefactory/marathon-python/tree/0.3.1) (2014-08-05) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.2.9...0.3.1) **Merged pull requests:** @@ -559,12 +616,15 @@ - Raise exceptions instead of swallowing them silently [\#3](https://github.com/thefactory/marathon-python/pull/3) ([StephanErb](https://github.com/StephanErb)) ## [0.2.9](https://github.com/thefactory/marathon-python/tree/0.2.9) (2014-08-04) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.2.7...0.2.9) ## [0.2.7](https://github.com/thefactory/marathon-python/tree/0.2.7) (2014-07-24) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.2.6...0.2.7) ## [0.2.6](https://github.com/thefactory/marathon-python/tree/0.2.6) (2014-07-24) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.2.5...0.2.6) **Merged pull requests:** @@ -572,6 +632,7 @@ - Updated README.md with correction to create\_app args [\#2](https://github.com/thefactory/marathon-python/pull/2) ([rasathus](https://github.com/rasathus)) ## [0.2.5](https://github.com/thefactory/marathon-python/tree/0.2.5) (2014-07-02) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.2.3...0.2.5) **Merged pull requests:** @@ -579,15 +640,21 @@ - allowing stagedAt and startedAt keys to be null [\#1](https://github.com/thefactory/marathon-python/pull/1) ([Codeacious](https://github.com/Codeacious)) ## [0.2.3](https://github.com/thefactory/marathon-python/tree/0.2.3) (2014-06-02) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.2.0...0.2.3) ## [0.2.0](https://github.com/thefactory/marathon-python/tree/0.2.0) (2014-04-28) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.1.1...0.2.0) ## [0.1.1](https://github.com/thefactory/marathon-python/tree/0.1.1) (2014-04-23) + [Full Changelog](https://github.com/thefactory/marathon-python/compare/0.1.0...0.1.1) ## [0.1.0](https://github.com/thefactory/marathon-python/tree/0.1.0) (2014-04-23) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/8060b138250686d1fe2f79d4d5118fef39aa553e...0.1.0) + + -\* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)* \ No newline at end of file +\* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)* diff --git a/setup.py b/setup.py index 9637850..8adc7f0 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='marathon', - version='0.11.0', + version='0.12.0', description='Marathon Client Library', long_description="""Python interface to the Mesos Marathon REST API.""", author='Mike Babineau', From 7a16250db8a61e350dcf63cf381e50d7d9a684a0 Mon Sep 17 00:00:00 2001 From: "Tilian R. Honig" Date: Wed, 15 Jan 2020 20:06:34 +0100 Subject: [PATCH 269/292] Fix return value for kill_given_tasks. --- marathon/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marathon/client.py b/marathon/client.py index 25782db..8e61973 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -531,7 +531,7 @@ def kill_given_tasks(self, task_ids, scale=False, force=None): data = json.dumps({"ids": task_ids}) response = self._do_request( 'POST', '/v2/tasks/delete', params=params, data=data) - return response == 200 + return response.status_code == 200 def kill_tasks(self, app_id, scale=False, wipe=False, host=None, batch_size=0, batch_delay=0): From d8992e4753453623035ee892f37d83b47e6713b7 Mon Sep 17 00:00:00 2001 From: Ricardo Rosales Date: Wed, 4 Mar 2020 18:27:20 -0600 Subject: [PATCH 270/292] Added role to task model --- marathon/models/task.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/marathon/models/task.py b/marathon/models/task.py index e049748..0348c6a 100644 --- a/marathon/models/task.py +++ b/marathon/models/task.py @@ -24,13 +24,15 @@ class MarathonTask(MarathonResource): :param region: fault domain region support in DCOS EE :type zone: str :param zone: fault domain zone support in DCOS EE + :type role: str + :param role: mesos role """ DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ' def __init__(self, app_id=None, health_check_results=None, host=None, id=None, ports=None, service_ports=None, slave_id=None, staged_at=None, started_at=None, version=None, ip_addresses=[], state=None, local_volumes=None, - region=None, zone=None): + region=None, zone=None, role=None): self.app_id = app_id self.health_check_results = health_check_results or [] self.health_check_results = [ @@ -54,6 +56,7 @@ def __init__(self, app_id=None, health_check_results=None, host=None, id=None, p self.local_volumes = local_volumes or [] self.region = region self.zone = zone + self.role = role class MarathonIpAddress(MarathonObject): From f4b32d372c10b492cf1108cc93ea8d03f66a914d Mon Sep 17 00:00:00 2001 From: Ricardo Rosales Date: Wed, 4 Mar 2020 19:23:31 -0600 Subject: [PATCH 271/292] Adding role to app model --- marathon/models/app.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/marathon/models/app.py b/marathon/models/app.py index 00a3d57..e290fd9 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -38,6 +38,7 @@ class MarathonApp(MarathonResource): :param health_checks: health checks :type health_checks: list[:class:`marathon.models.MarathonHealthCheck`] or list[dict] :param str id: app id + :param str role: mesos role :param int instances: instances :param last_task_failure: last task failure :type last_task_failure: :class:`marathon.models.app.MarathonTaskFailure` or dict @@ -75,7 +76,7 @@ class MarathonApp(MarathonResource): 'args', 'backoff_factor', 'backoff_seconds', 'cmd', 'constraints', 'container', 'cpus', 'dependencies', 'disk', 'env', 'executor', 'gpus', 'health_checks', 'instances', 'kill_selection', 'labels', 'max_launch_delay_seconds', 'mem', 'ports', 'require_ports', 'store_urls', 'task_rate_limit', 'upgrade_strategy', 'unreachable_strategy', - 'uris', 'user', 'version' + 'uris', 'user', 'version', 'role' ] """List of attributes which may be updated/changed after app creation""" @@ -90,7 +91,7 @@ class MarathonApp(MarathonResource): def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, backoff_seconds=None, cmd=None, constraints=None, container=None, cpus=None, dependencies=None, deployments=None, disk=None, env=None, - executor=None, health_checks=None, id=None, instances=None, kill_selection=None, labels=None, + executor=None, health_checks=None, id=None, role=None, instances=None, kill_selection=None, labels=None, last_task_failure=None, max_launch_delay_seconds=None, mem=None, ports=None, require_ports=None, store_urls=None, task_rate_limit=None, tasks=None, tasks_running=None, tasks_staged=None, tasks_healthy=None, task_kill_grace_period_seconds=None, tasks_unhealthy=None, upgrade_strategy=None, @@ -131,6 +132,7 @@ def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, for hc in (health_checks or []) ] self.id = assert_valid_path(id) + self.role = role self.instances = instances if kill_selection and kill_selection not in self.KILL_SELECTIONS: raise InvalidChoiceError( From 08e8350d90e1a0d6cacd38f9d3a2e650079d2ca1 Mon Sep 17 00:00:00 2001 From: Ricardo Rosales Date: Thu, 5 Mar 2020 16:01:15 -0600 Subject: [PATCH 272/292] Added debug log when creating app --- marathon/client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/marathon/client.py b/marathon/client.py index 8e61973..9db890c 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -169,6 +169,7 @@ def create_app(self, app_id, app, minimal=True): """ app.id = app_id data = app.to_json(minimal=minimal) + marathon.log.debug('create app JSON sent: {}'.format(data)) response = self._do_request('POST', '/v2/apps', data=data) if response.status_code == 201: return self._parse_response(response, MarathonApp) From 2b73d7cc373814ab07ed6c341c36a536ac6b1ff8 Mon Sep 17 00:00:00 2001 From: Ricardo Rosales Date: Thu, 5 Mar 2020 16:50:59 -0600 Subject: [PATCH 273/292] Now we validate app id is valid on creation --- marathon/client.py | 3 ++- marathon/models/deployment.py | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index 9db890c..26c7914 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -13,6 +13,7 @@ import marathon from .models import MarathonApp, MarathonDeployment, MarathonGroup, MarathonInfo, MarathonTask, MarathonEndpoint, MarathonQueueItem from .exceptions import ConflictError, InternalServerError, NotFoundError, MarathonHttpError, MarathonError, NoResponseError +from .models.base import assert_valid_path from .models.events import EventFactory, MarathonEvent from .util import MarathonJsonEncoder, MarathonMinimalJsonEncoder @@ -167,7 +168,7 @@ def create_app(self, app_id, app, minimal=True): :returns: the created app (on success) :rtype: :class:`marathon.models.app.MarathonApp` or False """ - app.id = app_id + app.id = assert_valid_path(app_id) data = app.to_json(minimal=minimal) marathon.log.debug('create app JSON sent: {}'.format(data)) response = self._do_request('POST', '/v2/apps', data=data) diff --git a/marathon/models/deployment.py b/marathon/models/deployment.py index 97e73ec..4885689 100644 --- a/marathon/models/deployment.py +++ b/marathon/models/deployment.py @@ -1,4 +1,4 @@ -from .base import MarathonObject, MarathonResource +from .base import MarathonObject, MarathonResource assert_valid_path class MarathonDeployment(MarathonResource): @@ -60,8 +60,8 @@ class MarathonDeploymentAction(MarathonObject): def __init__(self, action=None, app=None, apps=None, type=None, readiness_check_results=None, pod=None): self.action = action - self.app = app - self.apps = apps + self.app = assert_valid_path(app) + self.apps = assert_valid_path(apps) self.pod = pod self.type = type # TODO: Remove builtin shadow self.readiness_check_results = readiness_check_results # TODO: The docs say this is called just "readinessChecks?" From c1f802d92b924e1e96cf965f900e2e9073eb738f Mon Sep 17 00:00:00 2001 From: Ricardo Rosales Date: Thu, 5 Mar 2020 16:52:26 -0600 Subject: [PATCH 274/292] Separating import of assert_valid_path on deployment model --- marathon/models/deployment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marathon/models/deployment.py b/marathon/models/deployment.py index 4885689..d7ac7cd 100644 --- a/marathon/models/deployment.py +++ b/marathon/models/deployment.py @@ -1,4 +1,4 @@ -from .base import MarathonObject, MarathonResource assert_valid_path +from .base import MarathonObject, MarathonResource, assert_valid_path class MarathonDeployment(MarathonResource): From ec1c153ddb604c0516eedbbbd46d59e13664f5a3 Mon Sep 17 00:00:00 2001 From: Ricardo Rosales Date: Thu, 5 Mar 2020 17:33:02 -0600 Subject: [PATCH 275/292] lower case app id --- marathon/client.py | 2 +- marathon/models/app.py | 2 +- marathon/models/deployment.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index 26c7914..150f534 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -168,7 +168,7 @@ def create_app(self, app_id, app, minimal=True): :returns: the created app (on success) :rtype: :class:`marathon.models.app.MarathonApp` or False """ - app.id = assert_valid_path(app_id) + app.id = assert_valid_path(app_id.lower()) data = app.to_json(minimal=minimal) marathon.log.debug('create app JSON sent: {}'.format(data)) response = self._do_request('POST', '/v2/apps', data=data) diff --git a/marathon/models/app.py b/marathon/models/app.py index e290fd9..ede2341 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -131,7 +131,7 @@ def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, hc, MarathonHealthCheck) else MarathonHealthCheck().from_json(hc) for hc in (health_checks or []) ] - self.id = assert_valid_path(id) + self.id = assert_valid_path(id.lower()) self.role = role self.instances = instances if kill_selection and kill_selection not in self.KILL_SELECTIONS: diff --git a/marathon/models/deployment.py b/marathon/models/deployment.py index d7ac7cd..c3398d2 100644 --- a/marathon/models/deployment.py +++ b/marathon/models/deployment.py @@ -60,8 +60,8 @@ class MarathonDeploymentAction(MarathonObject): def __init__(self, action=None, app=None, apps=None, type=None, readiness_check_results=None, pod=None): self.action = action - self.app = assert_valid_path(app) - self.apps = assert_valid_path(apps) + self.app = assert_valid_path(app.lower()) + self.apps = assert_valid_path(apps.lower()) self.pod = pod self.type = type # TODO: Remove builtin shadow self.readiness_check_results = readiness_check_results # TODO: The docs say this is called just "readinessChecks?" From d68374ab8a1873492b24a3897090544666519cd4 Mon Sep 17 00:00:00 2001 From: Ricardo Rosales Date: Fri, 15 May 2020 11:27:36 -0500 Subject: [PATCH 276/292] Updating compatibility and tests --- .travis.yml | 1 + README.md | 1 + 2 files changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index c227dfb..8d89ecf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,5 @@ env: + - MARATHONVERSION: 1.9.109 - MARATHONVERSION: 1.6.322 - MARATHONVERSION: 1.4.11 - MARATHONVERSION: 1.3.0 diff --git a/README.md b/README.md index 592bb92..c109c83 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ This is a Python library for interfacing with [Marathon](https://github.com/meso #### Compatibility +* For Marathon 1.9.x, use at least 0.13.0 * For Marathon 1.6.x, use at least 0.10.0 * For Marathon 1.4.1, use at least 0.8.13 * For Marathon 1.1.1, use at least 0.8.1 From f215d7c1dc09fdacec5842d910252dcb53bf9b1d Mon Sep 17 00:00:00 2001 From: Ricardo Rosales Date: Wed, 20 May 2020 11:20:26 -0500 Subject: [PATCH 277/292] Trying without app.lower and apps.lower --- marathon/client.py | 2 +- marathon/models/app.py | 2 +- marathon/models/deployment.py | 4 ++-- marathon/models/info.py | 16 ++++++++++------ 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index 150f534..1e50c98 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -168,7 +168,7 @@ def create_app(self, app_id, app, minimal=True): :returns: the created app (on success) :rtype: :class:`marathon.models.app.MarathonApp` or False """ - app.id = assert_valid_path(app_id.lower()) + app.id = assert_valid_path(app_id.lower() if type(app_id) is str else app_id) data = app.to_json(minimal=minimal) marathon.log.debug('create app JSON sent: {}'.format(data)) response = self._do_request('POST', '/v2/apps', data=data) diff --git a/marathon/models/app.py b/marathon/models/app.py index ede2341..b4d6aed 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -131,7 +131,7 @@ def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, hc, MarathonHealthCheck) else MarathonHealthCheck().from_json(hc) for hc in (health_checks or []) ] - self.id = assert_valid_path(id.lower()) + self.id = assert_valid_path(id.lower() if type(id) is str else id) self.role = role self.instances = instances if kill_selection and kill_selection not in self.KILL_SELECTIONS: diff --git a/marathon/models/deployment.py b/marathon/models/deployment.py index c3398d2..d7ac7cd 100644 --- a/marathon/models/deployment.py +++ b/marathon/models/deployment.py @@ -60,8 +60,8 @@ class MarathonDeploymentAction(MarathonObject): def __init__(self, action=None, app=None, apps=None, type=None, readiness_check_results=None, pod=None): self.action = action - self.app = assert_valid_path(app.lower()) - self.apps = assert_valid_path(apps.lower()) + self.app = assert_valid_path(app) + self.apps = assert_valid_path(apps) self.pod = pod self.type = type # TODO: Remove builtin shadow self.readiness_check_results = readiness_check_results # TODO: The docs say this is called just "readinessChecks?" diff --git a/marathon/models/info.py b/marathon/models/info.py index 5ca62d4..68622a3 100644 --- a/marathon/models/info.py +++ b/marathon/models/info.py @@ -64,10 +64,12 @@ class MarathonConfig(MarathonObject): :param int leader_proxy_read_timeout_ms: :param int local_port_min: :param int local_port_max: + :param bool maintenance_mode: :param str master: :param str mesos_leader_ui_url: :param str mesos_role: :param str mesos_user: + :param str new_group_enforce_role: :param str webui_url: :param int reconciliation_initial_delay: :param int reconciliation_interval: @@ -99,12 +101,12 @@ class MarathonConfig(MarathonObject): def __init__(self, checkpoint=None, executor=None, failover_timeout=None, framework_name=None, ha=None, hostname=None, leader_proxy_connection_timeout_ms=None, leader_proxy_read_timeout_ms=None, - local_port_min=None, local_port_max=None, master=None, mesos_leader_ui_url=None, mesos_role=None, mesos_user=None, - webui_url=None, reconciliation_initial_delay=None, reconciliation_interval=None, - task_launch_timeout=None, marathon_store_timeout=None, task_reservation_timeout=None, features=None, - access_control_allow_origin=None, decline_offer_duration=None, - default_network_name=None, env_vars_prefix=None, - launch_token=None, launch_token_refresh_interval=None, + local_port_min=None, local_port_max=None, maintenance_mode=None, master=None, mesos_leader_ui_url=None, + mesos_role=None, mesos_user=None, new_group_enforce_role=None, webui_url=None, + reconciliation_initial_delay=None, reconciliation_interval=None, task_launch_timeout=None, + marathon_store_timeout=None, task_reservation_timeout=None, features=None, + access_control_allow_origin=None, decline_offer_duration=None, default_network_name=None, + env_vars_prefix=None, launch_token=None, launch_token_refresh_interval=None, max_instances_per_offer=None, mesos_bridge_name=None, mesos_heartbeat_failure_threshold=None, mesos_heartbeat_interval=None, min_revive_offers_interval=None, @@ -124,10 +126,12 @@ def __init__(self, checkpoint=None, executor=None, failover_timeout=None, framew self.hostname = hostname self.local_port_min = local_port_min self.local_port_max = local_port_max + self.maintenance_mode = maintenance_mode self.master = master self.mesos_leader_ui_url = mesos_leader_ui_url self.mesos_role = mesos_role self.mesos_user = mesos_user + self.new_group_enforce_role = new_group_enforce_role self.webui_url = webui_url self.reconciliation_initial_delay = reconciliation_initial_delay self.reconciliation_interval = reconciliation_interval From 5eb3848cc2ff8c2c66af581c2b2a35b1c7872639 Mon Sep 17 00:00:00 2001 From: Ricardo Rosales Date: Wed, 20 May 2020 15:13:07 -0500 Subject: [PATCH 278/292] Removing .lower from app_id and id --- marathon/client.py | 2 +- marathon/models/app.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/marathon/client.py b/marathon/client.py index 1e50c98..26c7914 100644 --- a/marathon/client.py +++ b/marathon/client.py @@ -168,7 +168,7 @@ def create_app(self, app_id, app, minimal=True): :returns: the created app (on success) :rtype: :class:`marathon.models.app.MarathonApp` or False """ - app.id = assert_valid_path(app_id.lower() if type(app_id) is str else app_id) + app.id = assert_valid_path(app_id) data = app.to_json(minimal=minimal) marathon.log.debug('create app JSON sent: {}'.format(data)) response = self._do_request('POST', '/v2/apps', data=data) diff --git a/marathon/models/app.py b/marathon/models/app.py index b4d6aed..e290fd9 100644 --- a/marathon/models/app.py +++ b/marathon/models/app.py @@ -131,7 +131,7 @@ def __init__(self, accepted_resource_roles=None, args=None, backoff_factor=None, hc, MarathonHealthCheck) else MarathonHealthCheck().from_json(hc) for hc in (health_checks or []) ] - self.id = assert_valid_path(id.lower() if type(id) is str else id) + self.id = assert_valid_path(id) self.role = role self.instances = instances if kill_selection and kill_selection not in self.KILL_SELECTIONS: From 6cb62ed8017de542eaafd2605180152cadc1545d Mon Sep 17 00:00:00 2001 From: Ricardo Rosales Date: Thu, 21 May 2020 18:57:26 -0500 Subject: [PATCH 279/292] Moving marathon away from ubuntu and starting from mesosphere dockerhub --- .travis.yml | 24 +++++++------- README.md | 2 +- itests/.dockerignore | 66 ++++++++++++++++++++++++++++++++++++++ itests/Dockerfile | 16 +++------ itests/docker-compose.yml | 13 +++++--- itests/install-marathon.sh | 51 +++++++++++++++++------------ itests/itest.sh | 4 +-- itests/start-marathon.sh | 29 +++++++++++++---- 8 files changed, 147 insertions(+), 58 deletions(-) create mode 100644 itests/.dockerignore diff --git a/.travis.yml b/.travis.yml index 8d89ecf..37aa8b6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,30 +1,28 @@ env: - - MARATHONVERSION: 1.9.109 - - MARATHONVERSION: 1.6.322 - - MARATHONVERSION: 1.4.11 - - MARATHONVERSION: 1.3.0 - - MARATHONVERSION: 1.1.2 + - MARATHONVERSION: v1.9.109 + - MARATHONVERSION: v1.6.322 + - MARATHONVERSION: v1.4.11 + - MARATHONVERSION: v1.3.0 + - MARATHONVERSION: v1.1.2 language: python +services: + - docker python: - 3.6 - 3.7 +before_install: + - docker pull "missingcharacter/marathon-python:${MARATHONVERSION}" + - docker run -d -p 18080:8080 -p 15050:5050 "missingcharacter/marathon-python:${MARATHONVERSION}" install: - pip install tox script: - make test - - ./itests/install-marathon.sh - - ./itests/start-marathon.sh & - make itests # Work around travis-ci/travis-ci#5227 addons: hostname: localhost - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - libstdc++6-4.7-dev sudo: required @@ -35,5 +33,5 @@ deploy: secure: "Wl8GWxsfPy4KoORYH26N3FllvMeWrifzeCbEx2Af4corcBQl43heeiFRRTlUOcSX0TIasER21PUvQ0R0cAgCjfknDb3SOROcRtcSBe16+cMmvwysfxcAx2OcF1UYBPY8e/qOsGge2Zyzx2PAPNEmJoWKbIT3vUJ4WvlLVeGYdJ0=" on: tags: true - condition: $MARATHONVERSION == "1.6.322" + condition: $MARATHONVERSION == "v1.6.322" repo: thefactory/marathon-python diff --git a/README.md b/README.md index c109c83..43b8f4f 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ make itests ### Running The Tests Against a Specific Version of Marathon ```bash -MARATHONVERSION=1.6.322 make itests +MARATHONVERSION=v1.6.322 make itests ``` ## Documentation diff --git a/itests/.dockerignore b/itests/.dockerignore new file mode 100644 index 0000000..2947fe4 --- /dev/null +++ b/itests/.dockerignore @@ -0,0 +1,66 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +bin/ +build/ +develop-eggs/ +dist/ +eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.cache +nosetests.xml +coverage.xml + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Rope +.ropeproject + +# Django stuff: +*.log +*.pot + +# Sphinx documentation +docs/_build/ + +.DS_Store + +# IntelliJ +.idea +*.iml + +# Packer http://packer.io +packer_cache + +/gh-pages +itests/marathon-version +.pytest_cache/ diff --git a/itests/Dockerfile b/itests/Dockerfile index b2223c7..0a3bf17 100644 --- a/itests/Dockerfile +++ b/itests/Dockerfile @@ -1,13 +1,6 @@ -FROM ubuntu:14.04 - -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install \ - software-properties-common -RUN add-apt-repository ppa:webupd8team/java -RUN echo "debconf shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections -RUN echo "debconf shared/accepted-oracle-license-v1-1 seen true" | debconf-set-selections -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y -q install \ - lsb-release \ - oracle-java8-installer +ARG MARATHONVERSION=v1.6.322 +FROM mesosphere/marathon:$MARATHONVERSION +USER root # Setup ADD ./marathon-version /root/marathon-version @@ -16,4 +9,5 @@ RUN /root/install-marathon.sh EXPOSE 8080 5050 ADD ./start-marathon.sh /root/start-marathon.sh -CMD /etc/init.d/zookeeper start && /root/start-marathon.sh +ENTRYPOINT [] +CMD ["/root/start-marathon.sh"] diff --git a/itests/docker-compose.yml b/itests/docker-compose.yml index 064c756..8c2a4c1 100644 --- a/itests/docker-compose.yml +++ b/itests/docker-compose.yml @@ -1,6 +1,9 @@ --- -marathon: - build: . - ports: - - 18080:8080 - - 15050:5050 +version: "3.8" +services: + marathon: + build: + context: . + ports: + - 18080:8080 + - 15050:5050 diff --git a/itests/install-marathon.sh b/itests/install-marathon.sh index 6bff172..e8e403a 100755 --- a/itests/install-marathon.sh +++ b/itests/install-marathon.sh @@ -1,27 +1,38 @@ -#!/bin/bash -set -vxeu +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' # Default version of marathon to test against if not set by the user [[ -f /root/marathon-version ]] && source /root/marathon-version -MARATHONVERSION="${MARATHONVERSION:-1.4.0}" +MARATHONVERSION="${MARATHONVERSION:-v1.6.322}" -# Setup -sudo apt-key adv --keyserver keyserver.ubuntu.com --recv 81026D0004C44CF7EF55ADF8DF7D54CBE56151BF -DISTRO=$(lsb_release -is | tr '[:upper:]' '[:lower:]') -CODENAME=$(lsb_release -cs) +export DEBIAN_FRONTEND=noninteractive -# Add the repository -echo "deb http://repos.mesosphere.com/${DISTRO} ${CODENAME} main" | sudo tee /etc/apt/sources.list.d/mesosphere.list -sudo apt-get update +shopt -s extglob -# Install packages -sudo DEBIAN_FRONTEND=noninteractive apt-get -y install oracle-java8-installer -sudo update-java-alternatives -s java-8-oracle -sudo DEBIAN_FRONTEND=noninteractive apt-get install oracle-java8-set-default +case "${MARATHONVERSION}" in + v1.9.109) + echo "Marathon version ${MARATHONVERSION} needs no specific changes" + apt update + ;; + v1.6.322) + sed -i 's!deb http://ftp.debian.org/debian jessie-backports main!!g' /etc/apt/sources.list + apt update + apt install -y mesos=1.6.* + ;; + v1.4.11) + sed -i 's!deb http://ftp.debian.org/debian jessie-backports main!!g' /etc/apt/sources.list + apt update + ;; + @(v1.3.0|v1.1.2)) + rm /etc/apt/sources.list.d/jessie-backports.list + apt update + ;; + *) + echo "Marathon version ${MARATHONVERSION} is not supported" + exit 1 + ;; +esac -sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes install mesos=1.6.* marathon=$MARATHONVERSION* - -# WTF MARATHON? -# Why does the precise version have java7 hardcoded if it requires java8? -sudo mkdir -p /usr/lib/jvm/java-7-oracle/bin/ -sudo ln -s /usr/lib/jvm/java-8-oracle/bin/java /usr/lib/jvm/java-7-oracle/bin/java +apt install -y --force-yes zookeeperd curl lsof +rm -rf /var/log/apt/* /var/log/alternatives.log /var/log/bootstrap.log /var/log/dpkg.log diff --git a/itests/itest.sh b/itests/itest.sh index bbc8b81..157ddf1 100755 --- a/itests/itest.sh +++ b/itests/itest.sh @@ -2,8 +2,8 @@ set -e -[[ -n $TRAVIS ]] || echo MARATHONVERSION=$MARATHONVERSION > marathon-version -[[ -n $TRAVIS ]] || docker-compose build +[[ -n $TRAVIS ]] || echo "MARATHONVERSION=${MARATHONVERSION}" > marathon-version +[[ -n $TRAVIS ]] || docker-compose build --build-arg "MARATHONVERSION=${MARATHONVERSION}" [[ -n $TRAVIS ]] || docker-compose pull [[ -n $TRAVIS ]] || docker-compose up -d behave "$@" diff --git a/itests/start-marathon.sh b/itests/start-marathon.sh index 0da2305..3d99c54 100755 --- a/itests/start-marathon.sh +++ b/itests/start-marathon.sh @@ -1,12 +1,29 @@ -#!/bin/bash +#!/usr/bin/env bash +set -xeuo pipefail +IFS=$'\n\t' LOGGER="--logging_level info" +# Default version of marathon to test against if not set by the user +[[ -f /root/marathon-version ]] && source /root/marathon-version +MARATHONVERSION="${MARATHONVERSION:-v1.6.322}" + +shopt -s extglob + +case "${MARATHONVERSION}" in + @(v1.4.11|v1.3.0|v1.1.2)) + ln -sf /marathon/bin/start /marathon/bin/marathon + ;; + *) + echo "Marathon version ${MARATHONVERSION} needs no specific changes" + ;; +esac java -version export MESOS_WORK_DIR='/tmp/mesos' -export ZK_HOST=`cat /etc/mesos/zk` +export ZK_HOST=$(cat /etc/mesos/zk) -mkdir -p "$MESOS_WORK_DIR" -nohup mesos-master --work_dir=/tmp/mesosmaster --zk=$ZK_HOST --quorum=1 & -nohup mesos-agent --master=$ZK_HOST --work_dir=/tmp/mesosagent --launcher=posix & -exec /usr/bin/marathon --master $ZK_HOST $LOGGER +mkdir -p "${MESOS_WORK_DIR}" +/etc/init.d/zookeeper start +nohup mesos-master --work_dir=/tmp/mesosmaster --zk=${ZK_HOST} --quorum=1 &> mesos-master.log & +nohup /usr/bin/env MESOS_SYSTEMD_ENABLE_SUPPORT=false mesos-slave --master=${ZK_HOST} --work_dir=/tmp/mesosagent --launcher=posix &> mesos-agent.log & +eval "bin/marathon --master ${ZK_HOST} ${LOGGER}" From 4c12ec19a85d284510acd1f7118e3c1e1cc4536b Mon Sep 17 00:00:00 2001 From: Ricardo Rosales Date: Thu, 21 May 2020 19:40:16 -0500 Subject: [PATCH 280/292] Trying to separate tests per python version --- .travis.yml | 11 ++++++++--- Makefile | 12 ++++++++++-- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 37aa8b6..13f03dc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,14 +11,19 @@ services: python: - 3.6 - 3.7 +include: + - python: 3.7 + env: TOX_ENV=py37 + - python: 3.6 + env: TOX_ENV=py36 before_install: - docker pull "missingcharacter/marathon-python:${MARATHONVERSION}" - - docker run -d -p 18080:8080 -p 15050:5050 "missingcharacter/marathon-python:${MARATHONVERSION}" + - docker run --name marathon-python -d -p 8080:8080 -p 5050:5050 "missingcharacter/marathon-python:${MARATHONVERSION}" install: - pip install tox script: - - make test - - make itests + - make test-$TOX_ENV + - make itests-$TOX_ENV # Work around travis-ci/travis-ci#5227 addons: diff --git a/Makefile b/Makefile index f64cc3e..5326277 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,18 @@ -itests: +itests: itests-py36 itests-py37 + +itests-py36: tox -e itest-py36 + +itests-py37: tox -e itest-py37 -test: +test: test-py36 test-py37 + +test-py36: tox -e pep8 tox -e test-py36 + +test-py37: tox -e test-py37 clean: From ba4cbbe4bd2820eeda8d63fe571e42ef5dd9977e Mon Sep 17 00:00:00 2001 From: Ricardo Rosales Date: Thu, 21 May 2020 19:48:59 -0500 Subject: [PATCH 281/292] travis lint says .travis.yml is valid --- .travis.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 13f03dc..ceb9b3d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,11 +11,12 @@ services: python: - 3.6 - 3.7 -include: - - python: 3.7 - env: TOX_ENV=py37 - - python: 3.6 - env: TOX_ENV=py36 +jobs: + include: + - python: 3.7 + env: TOX_ENV=py37 + - python: 3.6 + env: TOX_ENV=py36 before_install: - docker pull "missingcharacter/marathon-python:${MARATHONVERSION}" - docker run --name marathon-python -d -p 8080:8080 -p 5050:5050 "missingcharacter/marathon-python:${MARATHONVERSION}" @@ -29,7 +30,8 @@ script: addons: hostname: localhost -sudo: required +os: linux +dist: xenial deploy: - provider: pypi From 5c0f84101d1cf11e32c873c037ecf25caae7763c Mon Sep 17 00:00:00 2001 From: Ricardo Rosales Date: Thu, 21 May 2020 19:57:37 -0500 Subject: [PATCH 282/292] Trying to get tox environment from TRAVIS_PYTHON_VERSION --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index ceb9b3d..de4e6fb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,8 +23,8 @@ before_install: install: - pip install tox script: - - make test-$TOX_ENV - - make itests-$TOX_ENV + - make test-py${TRAVIS_PYTHON_VERSION/./} + - make itests-py${TRAVIS_PYTHON_VERSION/./} # Work around travis-ci/travis-ci#5227 addons: From fa3b4aba8d4b0a0346c50dfc6e05a40120054f58 Mon Sep 17 00:00:00 2001 From: Ricardo Rosales Date: Thu, 21 May 2020 19:58:59 -0500 Subject: [PATCH 283/292] Removing jobs from .travis.yml --- .travis.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index de4e6fb..7429125 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,12 +11,6 @@ services: python: - 3.6 - 3.7 -jobs: - include: - - python: 3.7 - env: TOX_ENV=py37 - - python: 3.6 - env: TOX_ENV=py36 before_install: - docker pull "missingcharacter/marathon-python:${MARATHONVERSION}" - docker run --name marathon-python -d -p 8080:8080 -p 5050:5050 "missingcharacter/marathon-python:${MARATHONVERSION}" From dfb944cb76b71a69fa589119e4c0179e5ab736c0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 22 May 2020 01:37:41 +0000 Subject: [PATCH 284/292] Bump requests from 2.11.1 to 2.20.0 Bumps [requests](https://github.com/requests/requests) from 2.11.1 to 2.20.0. - [Release notes](https://github.com/requests/requests/releases) - [Changelog](https://github.com/psf/requests/blob/master/HISTORY.md) - [Commits](https://github.com/requests/requests/compare/v2.11.1...v2.20.0) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4661030..c20f36f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -requests==2.11.1 +requests==2.20.0 From acffecd307c38c3512b77487e2e83806963c7a8d Mon Sep 17 00:00:00 2001 From: Ricardo Rosales Date: Thu, 21 May 2020 20:57:41 -0500 Subject: [PATCH 285/292] Trying to add support for v1.10.19 --- .travis.yml | 1 + itests/install-marathon.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7429125..542dd88 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,5 @@ env: + - MARATHONVERSION: v1.10.19 - MARATHONVERSION: v1.9.109 - MARATHONVERSION: v1.6.322 - MARATHONVERSION: v1.4.11 diff --git a/itests/install-marathon.sh b/itests/install-marathon.sh index e8e403a..5a4ff52 100755 --- a/itests/install-marathon.sh +++ b/itests/install-marathon.sh @@ -11,7 +11,7 @@ export DEBIAN_FRONTEND=noninteractive shopt -s extglob case "${MARATHONVERSION}" in - v1.9.109) + @(v1.10.19|v1.9.109)) echo "Marathon version ${MARATHONVERSION} needs no specific changes" apt update ;; From 4a109c3fa287085d3d1d7348f595593bc6e79866 Mon Sep 17 00:00:00 2001 From: Ricardo Rosales Date: Thu, 21 May 2020 21:13:24 -0500 Subject: [PATCH 286/292] Updating README with support for 1.10.x --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 43b8f4f..d4e38cf 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ This is a Python library for interfacing with [Marathon](https://github.com/meso #### Compatibility -* For Marathon 1.9.x, use at least 0.13.0 +* For Marathon 1.9.x and 1.10.x, use at least 0.13.0 * For Marathon 1.6.x, use at least 0.10.0 * For Marathon 1.4.1, use at least 0.8.13 * For Marathon 1.1.1, use at least 0.8.1 From ab57d0da799f2f7590a829aa26d4538c6031c008 Mon Sep 17 00:00:00 2001 From: Ricardo Rosales Date: Fri, 22 May 2020 12:27:17 -0500 Subject: [PATCH 287/292] Local tests no longer need docker-compose and rely on "mini-marathon" --- itests/docker-compose.yml | 9 --------- itests/{ => docker}/.dockerignore | 0 itests/{ => docker}/Dockerfile | 5 +++-- itests/docker/README.md | 9 +++++++++ itests/{ => docker}/install-marathon.sh | 0 itests/{ => docker}/start-marathon.sh | 2 +- itests/itest.sh | 9 +++------ itests/itest_utils.py | 16 +--------------- 8 files changed, 17 insertions(+), 33 deletions(-) delete mode 100644 itests/docker-compose.yml rename itests/{ => docker}/.dockerignore (100%) rename itests/{ => docker}/Dockerfile (67%) create mode 100644 itests/docker/README.md rename itests/{ => docker}/install-marathon.sh (100%) rename itests/{ => docker}/start-marathon.sh (97%) diff --git a/itests/docker-compose.yml b/itests/docker-compose.yml deleted file mode 100644 index 8c2a4c1..0000000 --- a/itests/docker-compose.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -version: "3.8" -services: - marathon: - build: - context: . - ports: - - 18080:8080 - - 15050:5050 diff --git a/itests/.dockerignore b/itests/docker/.dockerignore similarity index 100% rename from itests/.dockerignore rename to itests/docker/.dockerignore diff --git a/itests/Dockerfile b/itests/docker/Dockerfile similarity index 67% rename from itests/Dockerfile rename to itests/docker/Dockerfile index 0a3bf17..511fbbf 100644 --- a/itests/Dockerfile +++ b/itests/docker/Dockerfile @@ -1,11 +1,12 @@ ARG MARATHONVERSION=v1.6.322 FROM mesosphere/marathon:$MARATHONVERSION +ARG MARATHONVERSION USER root # Setup -ADD ./marathon-version /root/marathon-version ADD ./install-marathon.sh /root/install-marathon.sh -RUN /root/install-marathon.sh +RUN echo "MARATHONVERSION=${MARATHONVERSION}" > /root/marathon-version \ + && /root/install-marathon.sh EXPOSE 8080 5050 ADD ./start-marathon.sh /root/start-marathon.sh diff --git a/itests/docker/README.md b/itests/docker/README.md new file mode 100644 index 0000000..807726e --- /dev/null +++ b/itests/docker/README.md @@ -0,0 +1,9 @@ +# mini-marathon + +**Note:** We currently only support the marathon versions listed in [.travis.yml](https://github.com/thefactory/marathon-python/blob/acffecd307c38c3512b77487e2e83806963c7a8d/.travis.yml#L2-L7) + +## How to build + +``` +docker build --build-arg "MARATHONVERSION=v1.6.322" . +``` diff --git a/itests/install-marathon.sh b/itests/docker/install-marathon.sh similarity index 100% rename from itests/install-marathon.sh rename to itests/docker/install-marathon.sh diff --git a/itests/start-marathon.sh b/itests/docker/start-marathon.sh similarity index 97% rename from itests/start-marathon.sh rename to itests/docker/start-marathon.sh index 3d99c54..116e15c 100755 --- a/itests/start-marathon.sh +++ b/itests/docker/start-marathon.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -xeuo pipefail +set -euo pipefail IFS=$'\n\t' LOGGER="--logging_level info" diff --git a/itests/itest.sh b/itests/itest.sh index 157ddf1..6867ef6 100755 --- a/itests/itest.sh +++ b/itests/itest.sh @@ -2,10 +2,7 @@ set -e -[[ -n $TRAVIS ]] || echo "MARATHONVERSION=${MARATHONVERSION}" > marathon-version -[[ -n $TRAVIS ]] || docker-compose build --build-arg "MARATHONVERSION=${MARATHONVERSION}" -[[ -n $TRAVIS ]] || docker-compose pull -[[ -n $TRAVIS ]] || docker-compose up -d +[[ -n $TRAVIS ]] || docker pull "missingcharacter/marathon-python:${MARATHONVERSION}" +[[ -n $TRAVIS ]] || docker run --rm --name marathon-python -d -p 18080:8080 -p 15050:5050 "missingcharacter/marathon-python:${MARATHONVERSION}" behave "$@" -[[ -n $TRAVIS ]] || docker-compose stop -[[ -n $TRAVIS ]] || docker-compose rm --force +[[ -n $TRAVIS ]] || docker kill marathon-python diff --git a/itests/itest_utils.py b/itests/itest_utils.py index f5028fe..0ec865e 100644 --- a/itests/itest_utils.py +++ b/itests/itest_utils.py @@ -5,7 +5,6 @@ import time import requests -import compose.cli.command class TimeoutError(Exception): @@ -51,22 +50,9 @@ def wait_for_marathon(): break -def get_compose_service(service_name): - """Returns a compose object for the service""" - project = compose.cli.command.get_project(os.path.dirname(os.path.realpath(__file__))) - return project.get_service(service_name) - - def get_marathon_connection_string(): # only reliable way I can detect travis.. if '/travis/' in os.environ.get('PATH'): return 'localhost:8080' else: - service_port = get_service_internal_port('marathon') - return "localhost:%s" % service_port.published - - -def get_service_internal_port(service_name): - """Gets the exposed port for service_name from docker-compose.yml. If there are - multiple ports. It returns the first one.""" - return get_compose_service(service_name).options['ports'][0] + return "localhost:18080" From 5e5a2bd884132a757ac12e8b93fd875d1ae0296b Mon Sep 17 00:00:00 2001 From: Shubham Sharma Date: Sat, 30 May 2020 21:07:10 +0530 Subject: [PATCH 288/292] Fix deserialization for Deploment model --- marathon/models/deployment.py | 6 +++--- tests/test_model_deployment.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 tests/test_model_deployment.py diff --git a/marathon/models/deployment.py b/marathon/models/deployment.py index c3398d2..711ea18 100644 --- a/marathon/models/deployment.py +++ b/marathon/models/deployment.py @@ -25,7 +25,7 @@ def __init__(self, affected_apps=None, current_actions=None, current_step=None, self.affected_apps = affected_apps self.current_actions = [ a if isinstance( - a, MarathonDeploymentAction) else MarathonDeploymentAction().from_json(a) + a, MarathonDeploymentAction) else MarathonDeploymentAction.from_json(a) for a in (current_actions or []) ] self.current_step = current_step @@ -41,7 +41,7 @@ def parse_deployment_step(self, step): return MarathonDeploymentStep().from_json(step) elif step.__class__ == list: # This is Marathon < 1.0.0 style, a list of actions - return [s if isinstance(s, MarathonDeploymentAction) else MarathonDeploymentAction().from_json(s) for s in step] + return [s if isinstance(s, MarathonDeploymentAction) else MarathonDeploymentAction.from_json(s) for s in step] else: return step @@ -61,7 +61,7 @@ class MarathonDeploymentAction(MarathonObject): def __init__(self, action=None, app=None, apps=None, type=None, readiness_check_results=None, pod=None): self.action = action self.app = assert_valid_path(app.lower()) - self.apps = assert_valid_path(apps.lower()) + self.apps = assert_valid_path(apps.lower()) if apps != None else None self.pod = pod self.type = type # TODO: Remove builtin shadow self.readiness_check_results = readiness_check_results # TODO: The docs say this is called just "readinessChecks?" diff --git a/tests/test_model_deployment.py b/tests/test_model_deployment.py new file mode 100644 index 0000000..9da4c9e --- /dev/null +++ b/tests/test_model_deployment.py @@ -0,0 +1,16 @@ +from marathon.models.deployment import MarathonDeployment +import unittest + + +class MarathonDeploymentTest(unittest.TestCase): + + def test_env_defaults_to_empty_dict(self): + """ + é testé + """ + deployment_json ={"id": "ID", "version": "2020-05-30T07:35:04.695Z", "affectedApps": ["/app"], "affectedPods": [], "steps": [{"actions": [{"action": "RestartApplication", "app": "/app"}]}], "currentActions": [{"action": "RestartApplication", "app": "/app", "readinessCheckResults": []}], "currentStep": 1, "totalSteps": 1} + + deployment = MarathonDeployment.from_json(deployment_json) + self.assertEquals(deployment.id, "ID") + self.assertEquals(deployment.current_actions[0].app, "/app") + From 27c5bd836e8743c15dd8299512f8f1a9fe0d3525 Mon Sep 17 00:00:00 2001 From: Ricardo Rosales Date: Fri, 19 Jun 2020 17:24:15 -0500 Subject: [PATCH 289/292] test_model_deployment flake8 formatting --- tests/test_model_deployment.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/tests/test_model_deployment.py b/tests/test_model_deployment.py index 9da4c9e..9e9723a 100644 --- a/tests/test_model_deployment.py +++ b/tests/test_model_deployment.py @@ -8,9 +8,26 @@ def test_env_defaults_to_empty_dict(self): """ é testé """ - deployment_json ={"id": "ID", "version": "2020-05-30T07:35:04.695Z", "affectedApps": ["/app"], "affectedPods": [], "steps": [{"actions": [{"action": "RestartApplication", "app": "/app"}]}], "currentActions": [{"action": "RestartApplication", "app": "/app", "readinessCheckResults": []}], "currentStep": 1, "totalSteps": 1} - + deployment_json = { + "id": "ID", + "version": "2020-05-30T07:35:04.695Z", + "affectedApps": ["/app"], + "affectedPods": [], + "steps": [{ + "actions": [{ + "action": "RestartApplication", + "app": "/app" + }] + }], + "currentActions": [{ + "action": "RestartApplication", + "app": "/app", + "readinessCheckResults": [] + }], + "currentStep": 1, + "totalSteps": 1 + } + deployment = MarathonDeployment.from_json(deployment_json) self.assertEquals(deployment.id, "ID") self.assertEquals(deployment.current_actions[0].app, "/app") - From e2cf2c12cbc830da522a25305f68bca6b921678f Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Fri, 21 Aug 2020 09:28:33 -0700 Subject: [PATCH 290/292] Release 0.13.0 --- CHANGELOG.md | 24 +++++++++++++++++++++--- Makefile | 2 +- setup.py | 2 +- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f9e928..dc2946f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,26 @@ # Changelog -## [Unreleased](https://github.com/thefactory/marathon-python/tree/HEAD) +## [0.13.0](https://github.com/thefactory/marathon-python/tree/0.13.0) (2020-08-21) -[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.11.0...HEAD) +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.12.0...0.13.0) + +**Closed issues:** + +- about view one instance app logs real time [\#277](https://github.com/thefactory/marathon-python/issues/277) + +**Merged pull requests:** + +- Fix deserialization for Deploment model [\#276](https://github.com/thefactory/marathon-python/pull/276) ([missingcharacter](https://github.com/missingcharacter)) +- Local tests no longer need docker-compose and rely on "mini-marathon" [\#274](https://github.com/thefactory/marathon-python/pull/274) ([missingcharacter](https://github.com/missingcharacter)) +- Adding support for v1.10.19 [\#273](https://github.com/thefactory/marathon-python/pull/273) ([missingcharacter](https://github.com/missingcharacter)) +- Moving marathon away from ubuntu and starting from mesosphere dockerhub [\#272](https://github.com/thefactory/marathon-python/pull/272) ([missingcharacter](https://github.com/missingcharacter)) +- Updates for Marathon 1.9.109 [\#270](https://github.com/thefactory/marathon-python/pull/270) ([missingcharacter](https://github.com/missingcharacter)) +- Fix return value for kill\_given\_tasks. [\#268](https://github.com/thefactory/marathon-python/pull/268) ([Tilian](https://github.com/Tilian)) +- Bump requests from 2.11.1 to 2.20.0 [\#266](https://github.com/thefactory/marathon-python/pull/266) ([dependabot[bot]](https://github.com/apps/dependabot)) + +## [0.12.0](https://github.com/thefactory/marathon-python/tree/0.12.0) (2019-11-14) + +[Full Changelog](https://github.com/thefactory/marathon-python/compare/0.11.0...0.12.0) **Closed issues:** @@ -100,7 +118,7 @@ - Support filtering applications by labels [\#211](https://github.com/thefactory/marathon-python/pull/211) ([iandyh](https://github.com/iandyh)) - add embed option for /v2/queue [\#210](https://github.com/thefactory/marathon-python/pull/210) ([Rob-Johnson](https://github.com/Rob-Johnson)) - Enable TCP keepalive for sse requests [\#209](https://github.com/thefactory/marathon-python/pull/209) ([fengyehong](https://github.com/fengyehong)) -- Add "udp,tcp" to authorized protocols for containers [\#208](https://github.com/thefactory/marathon-python/pull/208) ([alxkt](https://github.com/alxkt)) +- Add "udp,tcp" to authorized protocols for containers [\#208](https://github.com/thefactory/marathon-python/pull/208) ([fuegoio](https://github.com/fuegoio)) - Allow event type filter on event stream [\#207](https://github.com/thefactory/marathon-python/pull/207) ([fengyehong](https://github.com/fengyehong)) - Fix MarathonResource hash as well [\#205](https://github.com/thefactory/marathon-python/pull/205) ([jolynch](https://github.com/jolynch)) diff --git a/Makefile b/Makefile index 5326277..1d94d86 100644 --- a/Makefile +++ b/Makefile @@ -19,12 +19,12 @@ clean: rm -rf dist/ build/ package: clean + github_changelog_generator --user=thefactory --project=marathon-python --future-release=0.13.0 pip install wheel python setup.py sdist bdist_wheel publish: package pip install twine twine upload dist/* - github_changelog_generator .PHONY: itests test clean package publish diff --git a/setup.py b/setup.py index 8adc7f0..fa25a98 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='marathon', - version='0.12.0', + version='0.13.0', description='Marathon Client Library', long_description="""Python interface to the Mesos Marathon REST API.""", author='Mike Babineau', From 21c9b1c55c64d4af31da78d5d64315c00ea1b16d Mon Sep 17 00:00:00 2001 From: Harold Dost Date: Wed, 26 Aug 2020 10:17:36 +0200 Subject: [PATCH 291/292] Add attribute enforce_role to groups. This was introduced in Marathon v1.9.32 Fixes #280 --- marathon/models/group.py | 3 ++- tests/test_model_group.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/marathon/models/group.py b/marathon/models/group.py index bb4f6bc..f68e230 100644 --- a/marathon/models/group.py +++ b/marathon/models/group.py @@ -20,7 +20,7 @@ class MarathonGroup(MarathonResource): """ def __init__(self, apps=None, dependencies=None, - groups=None, id=None, pods=None, version=None): + groups=None, id=None, pods=None, version=None, enforce_role=None): self.apps = [ a if isinstance(a, MarathonApp) else MarathonApp().from_json(a) for a in (apps or []) @@ -38,3 +38,4 @@ def __init__(self, apps=None, dependencies=None, # ] self.id = id self.version = version + self.enforce_role = enforce_role diff --git a/tests/test_model_group.py b/tests/test_model_group.py index e9fb340..45b20a3 100644 --- a/tests/test_model_group.py +++ b/tests/test_model_group.py @@ -15,3 +15,16 @@ def test_from_json_parses_root_group(self): } group = MarathonGroup().from_json(data) self.assertEqual("/", group.id) + + def test_from_json_parses_group_with_enforce_role(self): + data = { + "id": "/mygroup/works", + "groups": [ + {"id": "/foo", "apps": []}, + ], + "apps": [], + "enforceRole": False, + + } + group = MarathonGroup().from_json(data) + self.assertEqual("/mygroup/works", group.id) From ebb2ee3d5249b8e1454687ab4c71347b4ccd9c7e Mon Sep 17 00:00:00 2001 From: Harold Dost Date: Mon, 26 Oct 2020 14:32:06 +0100 Subject: [PATCH 292/292] Use collections.abc where available. As of Python 3.3 collections.abc should be used in place of collections. In Python 3.9 use of collections will be removed. --- marathon/util.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/marathon/util.py b/marathon/util.py index d9f5664..af2932e 100644 --- a/marathon/util.py +++ b/marathon/util.py @@ -1,4 +1,10 @@ -import collections +# collections.abc new as of 3.3, and collections is deprecated. collections +# will be unavailable in 3.9 +try: + import collections.abc as collections +except ImportError: + import collections + import datetime import logging