Skip to content

Enable connection pooling #49

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
abhinav-upadhyay wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Enable connection pooling #49

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9c5290a
Use requests.Session to enable connection pooling
abhinav-upadhyay Feb 19, 2019
71d67fc
Merge branch 'master' into connection_pool
abhinav-upadhyay Feb 22, 2019
d9d019c
Fix tests
abhinav-upadhyay Feb 22, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions apptuit/apptuit_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ def __init__(self, token=None, api_endpoint="https://api.apptuit.ai",
self._global_tags = global_tags
if not self._global_tags and not ignore_environ_tags:
self._global_tags = _get_tags_from_environment()
self.session = requests.Session()

@property
def put_apiurl(self):
Expand Down Expand Up @@ -205,7 +206,7 @@ def __send(self, payload, points_count, timeout):
headers["Content-Type"] = "application/json"
headers["Content-Encoding"] = "deflate"
headers["User-Agent"] = _get_user_agent()
response = requests.post(self.put_apiurl, data=body, headers=headers, timeout=timeout)
response = self.session.post(self.put_apiurl, data=body, headers=headers, timeout=timeout)
if response.status_code != 200 and response.status_code != 204:
status_code = response.status_code
if status_code == 400:
Expand Down Expand Up @@ -269,7 +270,7 @@ def _execute_query(self, query_string, start, end, timeout):
headers["User-Agent"] = _get_user_agent()
if self.token:
headers["Authorization"] = "Bearer " + self.token
hresp = requests.get(query_string, headers=headers, timeout=timeout)
hresp = self.session.get(query_string, headers=headers, timeout=timeout)
body = hresp.content
return _parse_response(body, start, end)

Expand Down
6 changes: 3 additions & 3 deletions apptuit/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@
VALID_CHARSET = set(ascii_letters + digits + "-_./")
INVALID_CHARSET = frozenset(map(chr, range(128))) - VALID_CHARSET

def _contains_valid_chars(string):
return INVALID_CHARSET.isdisjoint(string)
def _contains_valid_chars(val):
return INVALID_CHARSET.isdisjoint(val)

def _validate_tags(tags):
for tagk, tagv in tags.items():
for tagk in tags.keys():
if not tagk or not _contains_valid_chars(tagk):
raise ValueError("Tag key %s contains an invalid character, "
"allowed characters are a-z, A-Z, 0-9, -, _, ., and /" % tagk)
Expand Down
24 changes: 12 additions & 12 deletions tests/test_pyformance_reporter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
except ImportError:
from mock import Mock, patch

@patch('apptuit.apptuit_client.requests.post')
@patch('apptuit.apptuit_client.requests.Session.post')
def test_batch_send(mock_post):
"""
Test that when we create more than BATCH_SIZE number of points
Expand All @@ -42,7 +42,7 @@ def test_batch_send(mock_post):
assert_equals(total_points_sent, points_to_be_created)


@patch('apptuit.apptuit_client.requests.post')
@patch('apptuit.apptuit_client.requests.Session.post')
def test_partially_successful_send(mock_post):
"""
Test that we handle partially successful sends
Expand Down Expand Up @@ -73,7 +73,7 @@ def test_partially_successful_send(mock_post):
assert_equals(failed_points_count, 2)


@patch('apptuit.apptuit_client.requests.post')
@patch('apptuit.apptuit_client.requests.Session.post')
def test_send_negative(mock_post):
"""
Test negative responce from Apptuit backend
Expand All @@ -98,7 +98,7 @@ def test_send_negative(mock_post):
with assert_raises(ApptuitSendException):
reporter.report_now()

@patch('apptuit.apptuit_client.requests.post')
@patch('apptuit.apptuit_client.requests.Session.post')
def test_reporter_thread_active(mock_post):
"""
Test that reporter thread is active even if we are not able to send data
Expand All @@ -120,7 +120,7 @@ def test_reporter_thread_active(mock_post):
time.sleep(3)
assert_greater_equal(mock_post.call_count, 2)

@patch('apptuit.apptuit_client.requests.post')
@patch('apptuit.apptuit_client.requests.Session.post')
def test_invalid_metric_name(mock_post):
"""
Test for invalid metric name when reporting data
Expand All @@ -141,7 +141,7 @@ def test_invalid_metric_name(mock_post):
with assert_raises(ValueError) as ex:
reporter._collect_data_points(reporter.registry, None)

@patch('apptuit.apptuit_client.requests.post')
@patch('apptuit.apptuit_client.requests.Session.post')
def test_invalid_tag(mock_post):
"""
Test for invalid tag key when reporting data
Expand Down Expand Up @@ -178,7 +178,7 @@ def test_invalid_registry():
with assert_raises(AttributeError) as ex:
reporter._collect_data_points(None, None)

@patch('apptuit.apptuit_client.requests.post')
@patch('apptuit.apptuit_client.requests.Session.post')
def test_tags_with_key(mock_post):
"""
Test that additions tags work
Expand All @@ -198,7 +198,7 @@ def test_tags_with_key(mock_post):
cpu.add(random.randint(i, 100))
reporter.report_now()

@patch('apptuit.apptuit_client.requests.post')
@patch('apptuit.apptuit_client.requests.Session.post')
def test_tags_with_key_invalid(mock_post):
"""
Test that invalid tags raise error
Expand Down Expand Up @@ -234,12 +234,12 @@ def test_calling_report_now():
tags=tags)
counter_test = registry.counter("counter")
counter_test.inc(2)
with patch('apptuit.apptuit_client.requests.post') as mock_method:
with patch('apptuit.apptuit_client.requests.Session.post') as mock_method:
mock_method.return_value.status_code = 200
reporter.report_now()
assert_equals(mock_method.called, True)

@patch('apptuit.apptuit_client.requests.post')
@patch('apptuit.apptuit_client.requests.Session.post')
def test_zero_tags(mock_post):
"""
Test that using reporter without tags does not raise error
Expand All @@ -257,7 +257,7 @@ def test_zero_tags(mock_post):
counter_test.inc(2)
reporter.report_now()

@patch('apptuit.apptuit_client.requests.post')
@patch('apptuit.apptuit_client.requests.Session.post')
def test_zero_tags_with_host_disabled(mock_post):
"""
Test that using reporter without tags raises error
Expand Down Expand Up @@ -412,7 +412,7 @@ def test_none_prefix():
dps = reporter._collect_data_points(reporter.registry)
assert_equals(dps[0].metric, "counter1.count")

@patch('apptuit.apptuit_client.requests.post')
@patch('apptuit.apptuit_client.requests.Session.post')
def test_meta_metrics_of_reporter(mock_post):
"""
Test that meta metrics of reporter work
Expand Down
34 changes: 17 additions & 17 deletions tests/test_query.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,15 +43,15 @@ def do_query(mock_get):
end = 1407609000
return client.query(query, start, end)

@patch('apptuit.apptuit_client.requests.get')
@patch('apptuit.apptuit_client.requests.Session.get')
def test_query(mock_get):
"""
Test a valid query and make sure results are returned
"""
resp = do_query(mock_get)
assert_is_not_none(resp[0])

@patch('apptuit.apptuit_client.requests.get')
@patch('apptuit.apptuit_client.requests.Session.get')
def test_query_result_number_index(mock_get):
"""
Test that we can access the output by number based indexing from
Expand All @@ -61,7 +61,7 @@ def test_query_result_number_index(mock_get):
df = resp[0].to_df()
assert_is_not_none(df)

@patch('apptuit.apptuit_client.requests.get')
@patch('apptuit.apptuit_client.requests.Session.get')
def test_query_result_string_index(mock_get):
"""
Test that we can access the output by the name of the metric from the
Expand All @@ -71,7 +71,7 @@ def test_query_result_string_index(mock_get):
df = resp["nyc.taxi.rides"].to_df()
assert_is_not_none(df)

@patch('apptuit.apptuit_client.requests.get')
@patch('apptuit.apptuit_client.requests.Session.get')
def test_df_shape(mock_get):
"""
Verify the dataframe shape
Expand All @@ -80,15 +80,15 @@ def test_df_shape(mock_get):
df = resp[0].to_df()
assert_equals(df.shape, (432, 1))

@patch('apptuit.apptuit_client.requests.get')
@patch('apptuit.apptuit_client.requests.Session.get')
def test_number_of_series(mock_get):
"""
Verify the number of time series in the query result
"""
resp = do_query(mock_get)
assert_equals(len(resp[0].series), 1)

@patch('apptuit.apptuit_client.requests.get')
@patch('apptuit.apptuit_client.requests.Session.get')
def test_data(mock_get):
"""
Verify the data returned from the query
Expand All @@ -98,7 +98,7 @@ def test_data(mock_get):
df = resp[0].to_df()
assert_true(df.equals(expected_df))

@patch('apptuit.apptuit_client.requests.get')
@patch('apptuit.apptuit_client.requests.Session.get')
def test_metadata(mock_get):
"""
Test that the metadata of the query results are as expected
Expand All @@ -110,7 +110,7 @@ def test_metadata(mock_get):
assert_equals(series.name.metric, expected_series_name)
assert_equals(series.name.tags, expected_tags)

@patch('apptuit.apptuit_client.requests.get')
@patch('apptuit.apptuit_client.requests.Session.get')
def test_multiple_retries(mock_get):
"""
Test that the query API attempts retries when an error is returned from
Expand All @@ -128,7 +128,7 @@ def test_multiple_retries(mock_get):
with assert_raises(ApptuitException):
client.query(query, start, end, retry_count=3)

@patch('apptuit.apptuit_client.requests.get')
@patch('apptuit.apptuit_client.requests.Session.get')
def test_get_error(mock_get):
"""
Test that when the retry_count is 0 for the query API we get an exception
Expand All @@ -144,7 +144,7 @@ def test_get_error(mock_get):
with assert_raises(ApptuitException):
client.query(query, start, end, retry_count=0)

@patch('apptuit.apptuit_client.requests.get')
@patch('apptuit.apptuit_client.requests.Session.get')
def test_invalid_metric_name(mock_get):
"""
Test that we get an exception if the metric name contains invalid characters
Expand Down Expand Up @@ -199,7 +199,7 @@ def test_invalid_metric_name(mock_get):
with assert_raises(ValueError):
client.query(query, start, end)

@patch('apptuit.apptuit_client.requests.get')
@patch('apptuit.apptuit_client.requests.Session.get')
def test_empty_dps(mock_get):
"""
Test that we get an exception if the dps array is empty in the JSON response
Expand Down Expand Up @@ -253,7 +253,7 @@ def test_empty_dps(mock_get):
client.query(query, start, end)


@patch('apptuit.apptuit_client.requests.get')
@patch('apptuit.apptuit_client.requests.Session.get')
def test_empty_output(mock_get):
"""
Test the case when the outputs array is empty in the response
Expand Down Expand Up @@ -306,7 +306,7 @@ def test_empty_output(mock_get):
resp = client.query(query, start, end)
assert_is_none(resp)

@patch('apptuit.apptuit_client.requests.get')
@patch('apptuit.apptuit_client.requests.Session.get')
def test_empty_results(mock_get):
"""
Test that when results array is empty in the response and we try to access the
Expand Down Expand Up @@ -368,7 +368,7 @@ def test_empty_results(mock_get):
with assert_raises(KeyError):
_ = resp[0]

@patch('apptuit.apptuit_client.requests.get')
@patch('apptuit.apptuit_client.requests.Session.get')
def test_tags_dict_type(mock_get):
"""
Test that an exception is raised if the tags returned in the response
Expand Down Expand Up @@ -429,7 +429,7 @@ def test_tags_dict_type(mock_get):
with assert_raises(ValueError):
client.query(query, start, end)

@patch('apptuit.apptuit_client.requests.get')
@patch('apptuit.apptuit_client.requests.Session.get')
def test_invalid_char_in_tag_key(mock_get):
"""
Test for invalid character in one of the tag keys in the response
Expand Down Expand Up @@ -491,7 +491,7 @@ def test_invalid_char_in_tag_key(mock_get):
with assert_raises(ValueError):
client.query(query, start, end)

@patch('apptuit.apptuit_client.requests.get')
@patch('apptuit.apptuit_client.requests.Session.get')
def test_invalid_char_in_tag_value(mock_get):
"""
Test for invalid character in tag value in the response
Expand Down Expand Up @@ -570,7 +570,7 @@ def test_timeseries_obj_creation():
with assert_raises(ValueError):
apptuit_client.TimeSeries(metric=None, tags=None)

@patch('apptuit.apptuit_client.requests.get')
@patch('apptuit.apptuit_client.requests.Session.get')
def test_missing_pandas(mock_get):
orig_modules = sys.modules.copy()
orig_pandas = orig_modules['pandas']
Expand Down
18 changes: 9 additions & 9 deletions tests/test_send.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ def test_client_global_tags():
mock_environ.stop()


@patch('apptuit.apptuit_client.requests.post')
@patch('apptuit.apptuit_client.requests.Session.post')
def test_send_positive(mock_post):
"""
Test that send API is working as expected
Expand All @@ -65,7 +65,7 @@ def test_send_positive(mock_post):
if dps:
client.send(dps)

@patch('apptuit.apptuit_client.requests.post')
@patch('apptuit.apptuit_client.requests.Session.post')
def test_send_server_error(mock_post):
"""
Test for the case when there is an error from the backend for send
Expand All @@ -87,7 +87,7 @@ def test_send_server_error(mock_post):
if points_sent > 500:
break

@patch('apptuit.apptuit_client.requests.post')
@patch('apptuit.apptuit_client.requests.Session.post')
def test_send_413_error(mock_post):
"""
Test for the case when we get 413 from the PUT API
Expand Down Expand Up @@ -251,7 +251,7 @@ def test_apptuit_send_exception_without_status():
"datapoint test\n")


@patch('apptuit.apptuit_client.requests.post')
@patch('apptuit.apptuit_client.requests.Session.post')
def test_apptuit_send_exception_400(mock_post):
"""
Test for the case when there is an error from the backend for send
Expand All @@ -265,7 +265,7 @@ def test_apptuit_send_exception_400(mock_post):
with assert_raises(ApptuitSendException):
client.send(dps)

@patch('apptuit.apptuit_client.requests.post')
@patch('apptuit.apptuit_client.requests.Session.post')
def test_apptuit_send_exception_401(mock_post):
"""
Test for the case when there is an error from the backend for send
Expand Down Expand Up @@ -387,7 +387,7 @@ def test_timeseries_payload_with_envtags():
assert_equals(expected_payload, payload)


@patch('apptuit.apptuit_client.requests.post')
@patch('apptuit.apptuit_client.requests.Session.post')
def test_send_timeseries(mock_post):
"""
Test the send_timeseries API
Expand All @@ -411,7 +411,7 @@ def test_send_timeseries(mock_post):
series_list.append(series2)
client.send_timeseries(series_list)

@patch('apptuit.apptuit_client.requests.post')
@patch('apptuit.apptuit_client.requests.Session.post')
def test_send_timeseries_empty(mock_post):
"""
Test the send_timeseries API with an empty series list
Expand Down Expand Up @@ -442,7 +442,7 @@ def test_datapoint_repr():
assert_equals(repr(point), expected_repr)
assert_equals(str(point), expected_repr)

@patch('apptuit.apptuit_client.requests.post')
@patch('apptuit.apptuit_client.requests.Session.post')
def test_tags_limit_direct(mock_post):
"""
Test for failure when too many tags are used for datapoints/series
Expand All @@ -461,7 +461,7 @@ def test_tags_limit_direct(mock_post):
with assert_raises(ValueError):
client.send_timeseries([series1, series2])

@patch('apptuit.apptuit_client.requests.post')
@patch('apptuit.apptuit_client.requests.Session.post')
def test_tags_limit_indirect(mock_post):
"""
Test for failure when too many tags are used indirectly (when combined with global tags)
Expand Down