From 50a9c4b1d764c2d5d9a83c281873ebbdd9a732d5 Mon Sep 17 00:00:00 2001 From: Yuri Nikulin Date: Mon, 20 Jun 2022 23:43:13 +0300 Subject: [PATCH 1/2] Cache requests to /v2/components/ --- README.md | 5 +++++ prpr/main.py | 17 ++++++++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 6b68329..9080a95 100644 --- a/README.md +++ b/README.md @@ -260,6 +260,11 @@ process: ## История изменений +### 2022-06-22 + +* Закэшированы обращения к /v2/components/ (получение названия курса), + время работы скрипта при формировании списка задач сократилось примерно в 2 раза. + ### 2022-06-20 * [Refactor] Метод `PraktikTrackerClient.get_status_history()` переписан с diff --git a/prpr/main.py b/prpr/main.py index 5ae80cc..8862281 100755 --- a/prpr/main.py +++ b/prpr/main.py @@ -18,21 +18,25 @@ from prpr.startrack_client import get_startack_client from prpr.table import DISPLAYED_TAIL_LENGTH, print_issue_table +COMPONENT_SUFFIXES = "component_suffixes" +_components_cache = {} + class InteractiveCommand(Enum): CHECK_AGAIN = "🔁 Check again" -COMPONENT_SUFFIXES = "component_suffixes" - - def get_cohort(cohort, components, config): cohort = str(cohort) if cohort else "?" if not components: return cohort first_component = components[0] - component_name = first_component.name + + if first_component.id not in _components_cache: + _components_cache[first_component.id] = first_component.name + component_name = _components_cache[first_component.id] + suffix_mapper = config.get(COMPONENT_SUFFIXES, {}) return cohort + suffix_mapper.get(component_name, "") @@ -177,7 +181,10 @@ def main(): def extract_course(issue): if components := issue.components: - return components[0].name + component = components[0] + if component.id not in _components_cache: + _components_cache[component.id] = component.name + return _components_cache[component.id] logger.warning(f"{issue.key} doesn't have components 😿") return "unknown_course" From 0ca7cf7069c0353095729851bce62b3bd56aded7 Mon Sep 17 00:00:00 2001 From: Yuri Nikulin Date: Wed, 29 Jun 2022 11:12:12 +0300 Subject: [PATCH 2/2] Use requests-cache to cache API responces --- README.md | 7 ++++--- prpr/main.py | 12 ++---------- prpr/startrack_client.py | 36 ++++++++++++++++++++++++++++++++++-- requirements.txt | 1 + 4 files changed, 41 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 9080a95..a7561c0 100644 --- a/README.md +++ b/README.md @@ -260,10 +260,11 @@ process: ## История изменений -### 2022-06-22 +### 2022-06-29 -* Закэшированы обращения к /v2/components/ (получение названия курса), - время работы скрипта при формировании списка задач сократилось примерно в 2 раза. +* Закэшированы обращения к API. + Время работы скрипта при формировании списка задач сократилось примерно в 2 раза + (главным образом за счёт /v2/components/ - получение названия курса). ### 2022-06-20 diff --git a/prpr/main.py b/prpr/main.py index 8862281..d28fee0 100755 --- a/prpr/main.py +++ b/prpr/main.py @@ -19,7 +19,6 @@ from prpr.table import DISPLAYED_TAIL_LENGTH, print_issue_table COMPONENT_SUFFIXES = "component_suffixes" -_components_cache = {} class InteractiveCommand(Enum): @@ -32,11 +31,7 @@ def get_cohort(cohort, components, config): return cohort first_component = components[0] - - if first_component.id not in _components_cache: - _components_cache[first_component.id] = first_component.name - component_name = _components_cache[first_component.id] - + component_name = first_component.name suffix_mapper = config.get(COMPONENT_SUFFIXES, {}) return cohort + suffix_mapper.get(component_name, "") @@ -181,10 +176,7 @@ def main(): def extract_course(issue): if components := issue.components: - component = components[0] - if component.id not in _components_cache: - _components_cache[component.id] = component.name - return _components_cache[component.id] + return components[0].name logger.warning(f"{issue.key} doesn't have components 😿") return "unknown_course" diff --git a/prpr/startrack_client.py b/prpr/startrack_client.py index 4c43e5a..c19159e 100644 --- a/prpr/startrack_client.py +++ b/prpr/startrack_client.py @@ -1,7 +1,10 @@ from typing import Optional +import requests_cache from loguru import logger +from requests.utils import check_header_validity from yandex_tracker_client import TrackerClient +from yandex_tracker_client.connection import Connection from yandex_tracker_client.exceptions import TrackerClientError from yandex_tracker_client.objects import Resource @@ -12,9 +15,33 @@ STARTREK_TOKEN_KEY_NAME = "startrek_token" +class CachedConnection(Connection): + def __init__(self, token, org_id, headers=None, verify=True, **kwargs): + super().__init__(token, org_id, headers=headers, verify=verify, **kwargs) + self.session = requests_cache.CachedSession(backend='memory') + + self.session.verify = verify + + if headers is not None: + self.session.headers.update(headers) + + self.session.headers['Authorization'] = 'OAuth ' + (token or 'not provided') + self.session.headers['X-Org-Id'] = org_id or 'not provided' + self.session.headers['Content-Type'] = 'application/json' + + # Check validity headers for requests >= 2.11 + for header in self.session.headers.items(): + check_header_validity(header) + + class PraktikTrackerClient(TrackerClient): def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) + self.connector = kwargs.pop('connector', Connection) + connection = kwargs.pop('connection', None) + if connection is None: + connection = self.connector(*args, **kwargs) + + super().__init__(*args, connection=connection, **kwargs) self.token = kwargs["token"] def _get_filter_expression(self, user: Optional[str] = None): @@ -66,7 +93,12 @@ def get_startack_client(config) -> PraktikTrackerClient: logger.error(f"{STARTREK_TOKEN_KEY_NAME} top-level key not found in config 😿") exit(1) token = config[STARTREK_TOKEN_KEY_NAME] - return PraktikTrackerClient(org_id=YANDEX_ORG_ID, base_url="https://st-api.yandex-team.ru", token=token) + return PraktikTrackerClient( + org_id=YANDEX_ORG_ID, + base_url="https://st-api.yandex-team.ru", + token=token, + connector=CachedConnection, + ) def by_issue_key(issue) -> int: diff --git a/requirements.txt b/requirements.txt index cec0bfb..055648b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,7 @@ pytest==5.4.1 python-dateutil==2.8.1 questionary==1.9.0 requests==2.23.0 +requests_cache==0.9.4 rich==10.1.0 selenium==3.141.0 transliterate==1.10.2