From 9e66736f4828d0a2cb750810b81b241fcf63832b Mon Sep 17 00:00:00 2001 From: Mitsuo Takaki Date: Sun, 18 Mar 2018 14:43:41 -0700 Subject: [PATCH 1/6] Fixing the build and upgrading pytest and pytest-cov to latest --- .gitignore | 2 ++ dev_requirements.txt | 4 ++-- tests/__init__.py | 0 tests/test_configuration.py | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 tests/__init__.py diff --git a/.gitignore b/.gitignore index 09be2a2..fb9467c 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,5 @@ share/ MANIFEST dist/ .idea +.pytest_cache/ +pip-selfcheck.json diff --git a/dev_requirements.txt b/dev_requirements.txt index 5f665b7..f9e857c 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -3,7 +3,7 @@ codacy-coverage==1.2.18 ipython==4.2.0 mock==2.0.0 pudb==2016.1 -pytest==2.9.1 -pytest-cov==2.2.1 +pytest==3.4.2 +pytest-cov==2.5.1 requests==2.9.1 schedule==0.3.2 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 2aa118f..158a29d 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -24,7 +24,7 @@ class ConfigurationTest(unittest.TestCase): get_return = mock.Mock() get_return.ok = True get_return.json = mock.Mock() - get_return.json.return_value = {'data': {'status': 1}} + get_return.json.return_value = {'data': {'status': 1, 'default_value': 0.5}} return get_return sys.modules['requests'].get = get From 1899e956422a95aaa075907fc2bfdb434383acee Mon Sep 17 00:00:00 2001 From: Mitsuo Takaki Date: Sun, 18 Mar 2018 16:58:02 -0700 Subject: [PATCH 2/6] #38 - Adding support to milliseconds and different units for latency and changing http status to a range, instead of a single value. --- README.md | 6 ++++-- cachet_url_monitor/configuration.py | 28 ++++++++++++++++++++++------ cachet_url_monitor/latency_unit.py | 16 ++++++++++++++++ cachet_url_monitor/status.py | 5 +++++ config.yml | 3 ++- tests/test_configuration.py | 5 +++-- tests/test_expectation.py | 19 ++++++++++++++++--- tests/test_latency_unit.py | 18 ++++++++++++++++++ 8 files changed, 86 insertions(+), 14 deletions(-) create mode 100644 cachet_url_monitor/latency_unit.py create mode 100644 tests/test_latency_unit.py diff --git a/README.md b/README.md index 02b034b..b102bd4 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ endpoint: timeout: 1 # seconds expectation: - type: HTTP_STATUS - status: 200 + status_range: 200-300 - type: LATENCY threshold: 1 - type: REGEX @@ -38,6 +38,7 @@ cachet: - UPDATE_STATUS public_incidents: true frequency: 30 +latency_unit: ms ``` - **endpoint**, the configuration about the URL that will be monitored. @@ -45,7 +46,7 @@ frequency: 30 - **method**, the HTTP method that will be used by the monitor. - **timeout**, how long we'll wait to consider the request failed. The unit of it is seconds. - **expectation**, the list of expectations set for the URL. - - **HTTP_STATUS**, we will verify if the response status code matches what we expect. + - **HTTP_STATUS**, we will verify if the response status code falls into the expected range. Please keep in mind the range is inclusive on the first number and exclusive on the second number. If just one value is specified, it will default to only the given value, for example `200` will be converted to `200-201`. - **LATENCY**, we measure how long the request took to get a response and fail if it's above the threshold. The unit is in seconds. - **REGEX**, we verify if the response body matches the given regex. - **cachet**, this is the settings for our cachet server. @@ -58,6 +59,7 @@ frequency: 30 - **UPDATE_STATUS**, updates the component status - **public_incidents**, boolean to decide if created incidents should be visible to everyone or only to logged in users. Important only if `CREATE_INCIDENT` or `UPDATE_STATUS` are set. - **frequency**, how often we'll send a request to the given URL. The unit is in seconds. +- **latency_unit**, the latency unit used when reporting the metrics. It will automatically convert to the specified unit. It's not mandatory and it will default to **seconds**. Available units: `ms`, `s`, `m`, `h`. ## Setting up diff --git a/cachet_url_monitor/configuration.py b/cachet_url_monitor/configuration.py index 34757b0..2132eba 100644 --- a/cachet_url_monitor/configuration.py +++ b/cachet_url_monitor/configuration.py @@ -10,6 +10,7 @@ import requests from yaml import dump from yaml import load +import latency_unit import status as st # This is the mandatory fields that must be in the configuration file in this @@ -89,12 +90,15 @@ class Configuration(object): self.component_id = os.environ.get('CACHET_COMPONENT_ID') or self.data['cachet']['component_id'] self.metric_id = os.environ.get('CACHET_METRIC_ID') or self.data['cachet'].get('metric_id') self.default_metric_value = self.get_default_metric_value() + # The latency_unit configuration is not mandatory and we fallback to seconds, by default. + self.latency_unit = os.environ.get('LATENCY_UNIT') or self.data['cachet'].get('latency_unit') or 's' # We need the current status so we monitor the status changes. This is necessary for creating incidents. self.status = get_current_status(self.api_url, self.component_id, self.headers) # Get remaining settings - self.public_incidents = int(os.environ.get('CACHET_PUBLIC_INCIDENTS') or self.data['cachet']['public_incidents']) + self.public_incidents = int( + os.environ.get('CACHET_PUBLIC_INCIDENTS') or self.data['cachet']['public_incidents']) self.logger.info('Monitoring URL: %s %s' % (self.endpoint_method, self.endpoint_url)) self.expectations = [Expectaction.create(expectation) for expectation in self.data['endpoint']['expectation']] @@ -207,7 +211,9 @@ class Configuration(object): In case of failed connection trial pushes the default metric value. """ if 'metric_id' in self.data['cachet'] and hasattr(self, 'request'): - value = self.default_metric_value if self.status != 1 else self.request.elapsed.total_seconds() + # We convert the elapsed time from the request, in seconds, to the configured unit. + value = self.default_metric_value if self.status != 1 else latency_unit.convert_to_unit(self.latency_unit, + self.request.elapsed.total_seconds()) params = {'id': self.metric_id, 'value': value, 'timestamp': self.current_timestamp} metrics_request = requests.post('%s/metrics/%d/points' % (self.api_url, self.metric_id), params=params, @@ -226,7 +232,8 @@ class Configuration(object): """ if hasattr(self, 'incident_id') and self.status == st.COMPONENT_STATUS_OPERATIONAL: # If the incident already exists, it means it was unhealthy but now it's healthy again. - params = {'status': 4, 'visible': self.public_incidents, 'component_id': self.component_id, 'component_status': self.status, + params = {'status': 4, 'visible': self.public_incidents, 'component_id': self.component_id, + 'component_status': self.status, 'notify': True} incident_request = requests.put('%s/incidents/%d' % (self.api_url, self.incident_id), params=params, @@ -287,10 +294,19 @@ class Expectaction(object): class HttpStatus(Expectaction): def __init__(self, configuration): - self.status = configuration['status'] + self.status_range = self.parse_range(configuration['status_range']) + + def parse_range(self, range_string): + statuses = range_string.split("-") + if len(statuses) == 1: + # When there was no range given, we should treat the first number as a single status check. + return (int(statuses[0]), int(statuses[0]) + 1) + else: + # We shouldn't look into more than one value, as this is a range value. + return (int(statuses[0]), int(statuses[1])) def get_status(self, response): - if response.status_code == self.status: + if response.status_code >= self.status_range[0] and response.status_code < self.status_range[1]: return st.COMPONENT_STATUS_OPERATIONAL else: return st.COMPONENT_STATUS_PARTIAL_OUTAGE @@ -299,7 +315,7 @@ class HttpStatus(Expectaction): return 'Unexpected HTTP status (%s)' % (response.status_code,) def __str__(self): - return repr('HTTP status: %s' % (self.status,)) + return repr('HTTP status range: %s' % (self.status_range,)) class Latency(Expectaction): diff --git a/cachet_url_monitor/latency_unit.py b/cachet_url_monitor/latency_unit.py new file mode 100644 index 0000000..c5708ba --- /dev/null +++ b/cachet_url_monitor/latency_unit.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python + +seconds_per_unit = {"ms": 1000, "milliseconds": 1000, "s": 1, "seconds": 1, "m": float(1) / 60, + "minutes": float(1) / 60, "h": float(1) / 3600, "hours": float(1) / 3600} + + +def convert_to_unit(time_unit, value): + """ + Will convert the given value from seconds to the given time_unit. + + :param time_unit: The time unit to which the value will be converted to, from seconds. + This is a string parameter. The unit must be in the short form. + :param value: The given value that will be converted. This value must be in seconds. + :return: The converted value. + """ + return value * seconds_per_unit[time_unit] diff --git a/cachet_url_monitor/status.py b/cachet_url_monitor/status.py index 66564de..27a9da5 100644 --- a/cachet_url_monitor/status.py +++ b/cachet_url_monitor/status.py @@ -1,4 +1,9 @@ #!/usr/bin/env python +""" +This file defines all the different status different values. +These are all constants and are coupled to cachet's API configuration. +""" + COMPONENT_STATUS_OPERATIONAL = 1 COMPONENT_STATUS_PERFORMANCE_ISSUES = 2 diff --git a/config.yml b/config.yml index 5015d57..c3671f5 100644 --- a/config.yml +++ b/config.yml @@ -4,7 +4,7 @@ endpoint: timeout: 0.01 expectation: - type: HTTP_STATUS - status: 200 + status_range: 200-300 - type: LATENCY threshold: 1 - type: REGEX @@ -19,3 +19,4 @@ cachet: - UPDATE_STATUS public_incidents: true frequency: 30 +latency_unit: ms diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 158a29d..d65b84d 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -2,10 +2,11 @@ import sys import unittest -import cachet_url_monitor.status import mock from requests import ConnectionError, HTTPError, Timeout +import cachet_url_monitor.status + sys.modules['requests'] = mock.Mock() sys.modules['logging'] = mock.Mock() from cachet_url_monitor.configuration import Configuration @@ -38,7 +39,7 @@ class ConfigurationTest(unittest.TestCase): sys.modules['requests'].HTTPError = HTTPError def test_init(self): - self.assertEqual(len(self.configuration.data), 3, 'Configuration data size is incorrect') + self.assertEqual(len(self.configuration.data), 4, 'Number of root elements in config.yml is incorrect') self.assertEquals(len(self.configuration.expectations), 3, 'Number of expectations read from file is incorrect') self.assertDictEqual(self.configuration.headers, {'X-Cachet-Token': 'token2'}, 'Header was not set correctly') self.assertEquals(self.configuration.api_url, 'https://demo.cachethq.io/api/v1', diff --git a/tests/test_expectation.py b/tests/test_expectation.py index 071b75f..e0483f2 100644 --- a/tests/test_expectation.py +++ b/tests/test_expectation.py @@ -1,8 +1,10 @@ #!/usr/bin/env python +import re import unittest import mock -import re +import pytest + from cachet_url_monitor.configuration import HttpStatus, Regex from cachet_url_monitor.configuration import Latency @@ -51,10 +53,21 @@ class LatencyTest(unittest.TestCase): class HttpStatusTest(unittest.TestCase): def setUp(self): - self.expectation = HttpStatus({'type': 'HTTP_STATUS', 'status': 200}) + self.expectation = HttpStatus({'type': 'HTTP_STATUS', 'status_range': "200-300"}) def test_init(self): - assert self.expectation.status == 200 + assert self.expectation.status_range == (200, 300) + + def test_init_with_one_status(self): + """With only one value, we still expect a valid tuple""" + self.expectation = HttpStatus({'type': 'HTTP_STATUS', 'status_range': "200"}) + + assert self.expectation.status_range == (200, 201) + + def test_init_with_invalid_number(self): + """Invalid values should just fail with a ValueError, as we can't convert it to int.""" + with pytest.raises(ValueError) as excinfo: + self.expectation = HttpStatus({'type': 'HTTP_STATUS', 'status_range': "foo"}) def test_get_status_healthy(self): request = mock.Mock() diff --git a/tests/test_latency_unit.py b/tests/test_latency_unit.py new file mode 100644 index 0000000..7257b2b --- /dev/null +++ b/tests/test_latency_unit.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +import unittest + +from cachet_url_monitor.latency_unit import convert_to_unit + + +class ConfigurationTest(unittest.TestCase): + def test_convert_to_unit_ms(self): + assert convert_to_unit("ms", 1) == 1000 + + def test_convert_to_unit_s(self): + assert convert_to_unit("s", 20) == 20 + + def test_convert_to_unit_m(self): + assert convert_to_unit("m", 3) == float(3) / 60 + + def test_convert_to_unit_h(self): + assert convert_to_unit("h", 7200) == 2 From d3c14e64915581b392744789e8933f5511292afb Mon Sep 17 00:00:00 2001 From: Mitsuo Takaki Date: Sun, 18 Mar 2018 17:44:01 -0700 Subject: [PATCH 3/6] Addressing codacy issues --- cachet_url_monitor/configuration.py | 5 +++-- tests/test_expectation.py | 2 +- tests/test_latency_unit.py | 21 +++++++++++---------- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/cachet_url_monitor/configuration.py b/cachet_url_monitor/configuration.py index 2132eba..f33d2b8 100644 --- a/cachet_url_monitor/configuration.py +++ b/cachet_url_monitor/configuration.py @@ -294,9 +294,10 @@ class Expectaction(object): class HttpStatus(Expectaction): def __init__(self, configuration): - self.status_range = self.parse_range(configuration['status_range']) + self.status_range = HttpStatus.parse_range(configuration['status_range']) - def parse_range(self, range_string): + @staticmethod + def parse_range(range_string): statuses = range_string.split("-") if len(statuses) == 1: # When there was no range given, we should treat the first number as a single status check. diff --git a/tests/test_expectation.py b/tests/test_expectation.py index e0483f2..6f8fd8f 100644 --- a/tests/test_expectation.py +++ b/tests/test_expectation.py @@ -66,7 +66,7 @@ class HttpStatusTest(unittest.TestCase): def test_init_with_invalid_number(self): """Invalid values should just fail with a ValueError, as we can't convert it to int.""" - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): self.expectation = HttpStatus({'type': 'HTTP_STATUS', 'status_range': "foo"}) def test_get_status_healthy(self): diff --git a/tests/test_latency_unit.py b/tests/test_latency_unit.py index 7257b2b..7773049 100644 --- a/tests/test_latency_unit.py +++ b/tests/test_latency_unit.py @@ -1,18 +1,19 @@ #!/usr/bin/env python -import unittest from cachet_url_monitor.latency_unit import convert_to_unit -class ConfigurationTest(unittest.TestCase): - def test_convert_to_unit_ms(self): - assert convert_to_unit("ms", 1) == 1000 +def test_convert_to_unit_ms(): + assert convert_to_unit("ms", 1) == 1000 - def test_convert_to_unit_s(self): - assert convert_to_unit("s", 20) == 20 - def test_convert_to_unit_m(self): - assert convert_to_unit("m", 3) == float(3) / 60 +def test_convert_to_unit_s(): + assert convert_to_unit("s", 20) == 20 - def test_convert_to_unit_h(self): - assert convert_to_unit("h", 7200) == 2 + +def test_convert_to_unit_m(): + assert convert_to_unit("m", 3) == float(3) / 60 + + +def test_convert_to_unit_h(): + assert convert_to_unit("h", 7200) == 2 From 4c336ec7147cb5b9704318139660d7eb699c427d Mon Sep 17 00:00:00 2001 From: Mitsuo Takaki Date: Sun, 18 Mar 2018 22:17:29 -0700 Subject: [PATCH 4/6] Running some manual tests and updating Dockerfile --- Dockerfile | 9 ++++++++- cachet_url_monitor/configuration.py | 25 +++++++++++++++++++++---- dev_requirements.txt | 6 +++--- docker-compose.yml | 6 ------ requirements.txt | 6 +++--- 5 files changed, 35 insertions(+), 17 deletions(-) delete mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile index 58ada48..83f407d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,13 @@ -FROM python:2-onbuild +FROM python:2.7-alpine MAINTAINER Mitsuo Takaki +WORKDIR /usr/src/app + +COPY requirements.txt /usr/src/app/ +RUN pip install --no-cache-dir -r requirements.txt + +COPY cachet_url_monitor/* /usr/src/app/cachet_url_monitor/ + COPY config.yml /usr/src/app/config/ VOLUME /usr/src/app/config/ diff --git a/cachet_url_monitor/configuration.py b/cachet_url_monitor/configuration.py index f33d2b8..0340e2c 100644 --- a/cachet_url_monitor/configuration.py +++ b/cachet_url_monitor/configuration.py @@ -41,6 +41,16 @@ class ComponentNonexistentError(Exception): return repr('Component with id [%d] does not exist.' % (self.component_id,)) +class MetricNonexistentError(Exception): + """Exception raised when the component does not exist.""" + + def __init__(self, metric_id): + self.metric_id = metric_id + + def __str__(self): + return repr('Metric with id [%d] does not exist.' % (self.metric_id,)) + + def get_current_status(endpoint_url, component_id, headers): """Retrieves the current status of the component that is being monitored. It will fail if the component does not exist or doesn't respond with the expected data. @@ -89,7 +99,10 @@ class Configuration(object): self.api_url = os.environ.get('CACHET_API_URL') or self.data['cachet']['api_url'] self.component_id = os.environ.get('CACHET_COMPONENT_ID') or self.data['cachet']['component_id'] self.metric_id = os.environ.get('CACHET_METRIC_ID') or self.data['cachet'].get('metric_id') - self.default_metric_value = self.get_default_metric_value() + + if self.metric_id is not None: + self.default_metric_value = self.get_default_metric_value(self.metric_id) + # The latency_unit configuration is not mandatory and we fallback to seconds, by default. self.latency_unit = os.environ.get('LATENCY_UNIT') or self.data['cachet'].get('latency_unit') or 's' @@ -105,10 +118,14 @@ class Configuration(object): for expectation in self.expectations: self.logger.info('Registered expectation: %s' % (expectation,)) - def get_default_metric_value(self): + def get_default_metric_value(self, metric_id): """Returns default value for configured metric.""" - get_metric_request = requests.get('%s/metrics/%s' % (self.api_url, self.metric_id), headers=self.headers) - return get_metric_request.json()['data']['default_value'] + get_metric_request = requests.get('%s/metrics/%s' % (self.api_url, metric_id), headers=self.headers) + + if get_metric_request.ok: + return get_metric_request.json()['data']['default_value'] + else: + raise MetricNonexistentError(metric_id) def get_action(self): """Retrieves the action list from the configuration. If it's empty, returns an empty list. diff --git a/dev_requirements.txt b/dev_requirements.txt index f9e857c..cbe1a52 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,9 +1,9 @@ -PyYAML==3.11 +PyYAML==3.12 codacy-coverage==1.2.18 ipython==4.2.0 mock==2.0.0 pudb==2016.1 pytest==3.4.2 pytest-cov==2.5.1 -requests==2.9.1 -schedule==0.3.2 +requests==2.18.4 +schedule==0.5.0 diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 511cde9..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: '2' -services: - monitor: - build: . - volumes: - - .:/usr/src/app/ diff --git a/requirements.txt b/requirements.txt index 37bcb9b..afb165a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -PyYAML==3.11 -requests==2.9.1 -schedule==0.3.2 +PyYAML==3.12 +requests==2.18.4 +schedule==0.5.0 From 68a5609abc33abb8671c2de4035238784bf66cc3 Mon Sep 17 00:00:00 2001 From: Mitsuo Takaki Date: Sun, 18 Mar 2018 22:36:58 -0700 Subject: [PATCH 5/6] Bumping up the version to 0.4 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 24893fa..2b66aba 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from distutils.core import setup setup(name='cachet-url-monitor', - version='0.3', + version='0.4', description='Cachet URL monitor plugin', author='Mitsuo Takaki', author_email='mitsuotakaki@gmail.com', From 5cfef6392ef84ad633c19c815e40f0a3d181a2d3 Mon Sep 17 00:00:00 2001 From: Mitsuo Takaki Date: Sun, 18 Mar 2018 22:47:44 -0700 Subject: [PATCH 6/6] Adding setup.cfg for pypi --- setup.cfg | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..b88034e --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[metadata] +description-file = README.md