ok
Mini Shell
# -*- coding: utf-8 -*-
# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2021 All Rights Reserved
#
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENSE.TXT
"""
This module contains DecisionMaker class
"""
__package__ = 'ssa.modules'
import json
import logging
import os
from os.path import isfile
import numpy as np
from clcommon.lib.cledition import is_cl_solo_edition
from ssa.db import setup_database
from .common import Common
from .storage import (
iter_domains_data,
iter_urls_data,
get_url_durations
)
from ..configuration import load_tunables
from ..configuration.schemes import ssa_tunables_schema
from ..internal.constants import report_path
from ..internal.utils import previous_day_date, sentry_init
class DecisionMaker(Common):
"""
SSA Decision maker implementation.
"""
def __init__(self, engine=None):
super().__init__()
self.logger = logging.getLogger('decision_maker')
self.logger.info('DecisionMaker enabled: %s', __package__)
self.engine = engine if engine else setup_database()
def __call__(self):
self.logger.info('DecisionMaker started')
self.logger.debug('DecisionMaker loaded config: %s', self.config)
self.external_tunables = self.load_external_conf()
self.logger.debug('DecisionMaker loaded tunables: %s',
self.external_tunables)
report = self.data_processing()
self.add_json_report(report)
self.logger.info('DecisionMaker report: %s', report)
return report
@staticmethod
def _report_file(name) -> str:
"""
Full path to given filename in DM reports directory
"""
return os.path.join(report_path, name)
@property
def current_report_file(self) -> str:
"""
Full path to current DM report: report.json in DM reports directory
"""
return self._report_file('report.json')
@property
def _empty_report(self) -> dict:
"""
Returns empty report
"""
return dict(date=previous_day_date(), domains=[])
@property
def solo_filtered_options(self) -> set:
return {'correlation'}
@staticmethod
def load_external_conf():
"""Load external configuration values"""
return load_tunables('ssa.json', ssa_tunables_schema)
def data_processing(self) -> dict:
"""
Going through the list of domains, for each domain we go through
the list of urls. During data processing, we will form
the resulting dictionary.
"""
report = self._empty_report
for domain_data in iter_domains_data(self.engine):
# goes through the list of domains
urls_data = list()
domain_slow_reqs = 0
domain_url_durations = dict(get_url_durations(
self.engine, domain_data.domain_name))
for domain_data_key, domain_data_value in iter_urls_data(self.engine,
domain_data.domain_name,
list(domain_url_durations.keys())):
if self.is_ignored(domain_data_key):
self.logger.debug('%s ignored', domain_data_key)
continue
# goes through the list of urls, "domain_total_reqs" is also here
if domain_data_key not in self.non_url_fields:
# domain_data_key below - it is current url
if not self.is_throttling_suitable(
domain_data_value.get('url_throttled_reqs',
list([0] * 24)),
domain_data_value['url_total_reqs']):
# skip by allowed throttling percentage
continue
correlation_value = self.get_correlation(
domain_data_value['url_total_reqs'], domain_data.domain_total_reqs)
durations = domain_url_durations.get(domain_data_key)
if durations is None:
self.logger.error('Unable to get durations for %s', str(domain_data_key))
continue
if (self.request_number_exceeded(
domain_data_value['url_slow_reqs']) and
self.correlation_conditions(correlation_value)):
average_duration_calculation = np.mean(durations)
sum_url_slow_reqs = sum(
domain_data_value['url_slow_reqs'])
domain_slow_reqs += sum_url_slow_reqs
urls_data.append(dict(
name=domain_data_key, reqs_num=sum_url_slow_reqs,
average_duration=int(average_duration_calculation),
correlation=float(f'{correlation_value:.2f}')))
if urls_data:
sorted_urls = self.report_sorting(
list_to_sort=urls_data, leave_top=self.urls_number,
key_for_sorting='reqs_num')
report['domains'].append(dict(
name=domain_data.domain_name, slow_urls=len(sorted_urls),
slow_reqs=domain_slow_reqs,
total_reqs=sum(domain_data.domain_total_reqs), urls=sorted_urls))
if report['domains']:
report['domains'] = self.report_sorting(
list_to_sort=report['domains'], leave_top=self.domains_number,
key_for_sorting='slow_reqs')
return report
def list_handling_considering_time(self, url_slow_reqs: list) -> list:
"""
Based on the 'url_slow_reqs' list, a new list will be formed,
where the elements of the original list will be iteratively
summed by the number of elements equal to 'time'
"""
time = self.time or 24
return [sum(url_slow_reqs[i:time + i]) for i in
range(0, len(url_slow_reqs), time)]
def compare_elements_with_request_number(self,
url_slow_reqs_by_time: list) -> bool:
"""
This functions will check if any of elements is greater than "request_number"
"""
for i in url_slow_reqs_by_time:
if i >= self.request_number:
return True
return False
def get_correlation(self, url_total_reqs: list, domain_total_reqs: list):
"""
Calculates the correlation coefficient using the "url_total_reqs" and
the "domain_total_reqs" lists
"""
if not self.correlation:
return 0
return np.amin(np.corrcoef(url_total_reqs, domain_total_reqs))
@staticmethod
def report_sorting(list_to_sort: list, leave_top: int,
key_for_sorting: str) -> list:
"""
Will sort the domain list by "slow_reqs", the goal is to leave only
"domains_number" of uppers, also per each domain will sort urls by
"reqs_num", the goal is to leave only "urls_number" of uppers.
leave_top == 0 allows to keep the full list
"""
list_to_sort.sort(key=lambda dict_: dict_[key_for_sorting],
reverse=True)
if leave_top:
return list_to_sort[:leave_top]
else:
return list_to_sort
def rename_old_report(self):
"""
Rename old report
"""
old_report = self.current_report_file
if isfile(old_report):
with open(old_report) as json_data:
try:
d = json.load(json_data)
except json.JSONDecodeError:
date_from_report = 'unknown'
else:
date_from_report = d.get('date', 'dd.mm.yyyy').replace('.', '_')
new_report_name = f'report__{date_from_report}.json'
new_report = self._report_file(new_report_name)
os.rename(old_report, new_report)
def add_json_report(self, report: dict):
"""
Makes json report
"""
self.rename_old_report()
with open(self.current_report_file, 'w', encoding='utf-8') as f:
json.dump(report, f, ensure_ascii=False, indent=4)
def get_json_report(self) -> dict:
"""
Return contents of current report or empty report in case of error
"""
# TODO: [unification] return same json report
if is_cl_solo_edition(skip_jwt_check=True):
def _filtering_hook(value):
"""
Filter decoded object by fields, given in solo_filtered_options
property, e.g. 'correlation'
"""
return {k: v for k, v in value.items() if
k not in self.solo_filtered_options}
else:
_filtering_hook = None
try:
with open(self.current_report_file) as report:
report_dict = json.load(report, object_hook=_filtering_hook)
except (OSError, json.JSONDecodeError):
report_dict = self._empty_report
return report_dict
def correlation_conditions(self, correlation_value: int) -> bool:
"""
If correlation flag is enabled - we'll compare correlation_coefficient
from configuration with calculated correlation coefficient.
If the calculated value exceeds the configuration value - we return
True otherwise False. At the same time if correlation flag is disabled -
we'll also return "True" since in this case the correlation coefficient
is not checked and its value is specified as zero in final report.
"""
if not self.correlation:
return True
return correlation_value > self.correlation_coefficient
def request_number_exceeded(self, url_slow_reqs):
"""
At least one element from the received list (url_slow_reqs_by_time)
must be greater than request_number
"""
url_slow_reqs_by_time = self.list_handling_considering_time(
url_slow_reqs)
return self.compare_elements_with_request_number(url_slow_reqs_by_time)
def is_throttling_suitable(self, url_throttled_reqs: list,
url_total_reqs: list) -> bool:
"""
Check that percent of throttled requests per URL passes given threshold
"""
throttled_percent = (sum(url_throttled_reqs) / sum(
url_total_reqs)) * 100
self.logger.debug('Calculated throttled percent %s', throttled_percent)
return throttled_percent <= self.external_tunables.get(
'allowed_throttling_percentage', 0)
if __name__ == "__main__":
sentry_init()
logging.basicConfig(filename='decision_maker_standalone.log',
level=logging.INFO)
dm = DecisionMaker()
dm()
Zerion Mini Shell 1.0