ok
Mini Shell
# -*- coding: utf-8 -*-
#
# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2019 All Rights Reserved
#
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENSE.TXT
from __future__ import absolute_import
from __future__ import division
from builtins import range
import logging
import string
from random import choice
import sqlalchemy.orm.session # pylint: disable=unused-import
from clcommon.clpwd import ClPwd
from clcommon.cpapi import NotSupported
from clcommon.cpapi.cpapiexceptions import NoDBAccessData, NoPackage
from clcommon.utils import ExternalProgramFailed
from sqlalchemy.orm import sessionmaker
from typing import List, Tuple, Dict, Optional # pylint: disable=unused-import
from lvestats.core.plugin import LveStatsPlugin
from lvestats.lib.commons import proctitle
from lvestats.lib.commons.func import get_chunks, reboot_lock
from lvestats.lib.commons.htpasswd import HtpasswdFile
from lvestats.lib.commons.litespeed import LiteSpeed, LiteSpeedException, LiteSpeedInvalidCredentials
from lvestats.lib.snapshot import Snapshot
from lvestats.lib.ustate import MySQLOperationalError, SQLSnapshot, get_lveps
from lvestats.orm.incident import incident
DEFAULT_PERIOD_BETWEEN_INCIDENTS = 300 # time separating incidents
DEFAULT_SNAPSHOTS_PER_MINUTE = 2 # number of snapshots per minute
DEFAULT_MAX_SNAPSHOTS_PER_INCIDENT = 10 # the maximum number of snapshots in the incident
APACHE = 'apache'
LITESPEED = 'litespeed'
class LitespeedHelper(object):
def __init__(self):
self.is_running = False
self.state_changed = False
# config option; None if server should be
# detected automatically,
# False if we must use apache
# True if we must use litespeed
self.force_litespeed = None
# create random login-password pair
self.login = 'lve-stats-admin'
self.password = self.generate_random_password()
self.broken_config = False
def check_litespeed_state(self):
"""Check litespeed state"""
litespeed_running = LiteSpeed.is_litespeed_running()
self.state_changed = litespeed_running != self.is_running
self.is_running = litespeed_running
def dump_passwd(self):
try:
passwdfile = HtpasswdFile(LiteSpeed.HTPASSWD_PATH)
except ValueError:
self.broken_config = True
logging.warning("Can't change the password. Please, check the file:\n '%s'", LiteSpeed.HTPASSWD_PATH)
return
passwdfile.update(self.login, self.password)
passwdfile.save()
logging.debug("Password successfully changed.")
def get_user_data(self, username):
# type: (str) -> list
"""Get user data proxy method"""
litespeed = LiteSpeed(self.login, self.password)
try:
return litespeed.get_user_data(username)
except LiteSpeedInvalidCredentials:
self.dump_passwd()
raise
@staticmethod
def generate_random_password():
# type: () -> str
chars = string.ascii_letters + string.digits + '!@#$%^&*()'
return ''.join(choice(chars) for _ in range(16))
def get_use_litespeed(self):
# type: () -> bool
"""Get what we must use: litespeed or apache"""
is_litespeed_running = self.is_running
if self.broken_config:
return False
elif self.force_litespeed is not None:
return self.force_litespeed
return is_litespeed_running
class SnapshotHelper(object):
username_dbquery_map = None
ps = dict()
clpwd = None
sql_snap = SQLSnapshot()
def __init__(self):
self.litespeed_died = False
self._sql_snapshot_works = None
self.log = logging.getLogger('SnapshotSaver')
def get_names(self, lve_id):
# type: (int) -> List[str]
try:
return self.clpwd.get_names(lve_id)
except ClPwd.NoSuchUserException:
return list()
def get_snapshot_data(self, lve_id, litespeed_info):
# type: (int, LitespeedHelper) -> Tuple[dict, list, list]
processes = self.ps.get(lve_id, {}).get('TID', {})
queries = []
urls = []
for username in self.get_names(lve_id):
queries += self.username_dbquery_map.get(username, [])
use_litespeed = litespeed_info.get_use_litespeed()
if use_litespeed:
try:
urls += litespeed_info.get_user_data(username)
except LiteSpeedException as e:
# do not save message every time
if not self.litespeed_died:
logging.warning('Error during getting information from Litespeed: %s' % str(e))
self.litespeed_died = True
urls += proctitle.Proctitle().get_user_data(username)
else:
urls += proctitle.Proctitle().get_user_data(username)
return processes, queries, urls
def invalidate(self, lve_id_list):
# type: (List[int]) -> None
self.clpwd = ClPwd()
all_usernames = []
for lve_id in lve_id_list:
all_usernames += self.get_names(int(lve_id))
self.username_dbquery_map = self.retrieve_queries(all_usernames)
try:
self.ps = get_lveps()
except ExternalProgramFailed as e:
self.log.warning('An error occurred while getting processes list; %s', str(e))
def retrieve_queries(self, login_names):
# type: (Optional[List[str]]) -> Dict[str, List[str]]
try:
with self.sql_snap as db_requests:
result = db_requests.get(login_names)
if not self._sql_snapshot_works:
self._sql_snapshot_works = True
self.log.info('SQL snapshot is supported and operational')
return result
except MySQLOperationalError as e:
self.log.warning('An error occurred while getting MySQL process list; %s', str(e))
except (NotSupported, NoDBAccessData, NoPackage) as e:
# errors which we can write only once
# because who needs this message in log each 5 seconds?
if self._sql_snapshot_works in [None, True]:
self._sql_snapshot_works = False
self.log.info('SQL snapshot is not supported by this control panel: %s', e)
return {}
class SnapshotSaver(LveStatsPlugin):
server_id = ''
period_between_incidents = DEFAULT_PERIOD_BETWEEN_INCIDENTS
max_snapshots_per_incident = DEFAULT_MAX_SNAPSHOTS_PER_INCIDENT
_period = 60 // DEFAULT_SNAPSHOTS_PER_MINUTE
def __init__(self):
self.incidents_last_ts = {}
self.log = logging.getLogger('SnapshotSaver')
self.session = None # type: sqlalchemy.orm.session.Session
self._snapshots_data = {}
self.incidents_cache = {}
self.first_run = True
self.last_run = 0
self.litespeed_info = LitespeedHelper()
self.snapshots_enabled = True
self.compresslevel = 1
self._helper = SnapshotHelper()
def set_config(self, config):
# type: (dict) -> None
self.server_id = config.get('server_id', self.server_id)
self.period_between_incidents = int(config.get('period_between_incidents', self.period_between_incidents))
self.max_snapshots_per_incident = int(config.get('max_snapshots_per_incident', self.max_snapshots_per_incident))
self._period = 60 // int(config.get('snapshots_per_minute', DEFAULT_SNAPSHOTS_PER_MINUTE))
self.litespeed_info.force_litespeed = self._get_webserver_option(config)
self.setup_litespeed(force_webserver_message=True)
self.snapshots_enabled = config.get('disable_snapshots', "false").lower() != "true"
self.compresslevel = max(min(int(config.get('compresslevel', 1)), 9), 1)
def _get_webserver_option(self, config):
# type: (dict) -> Optional[bool]
"""
Check which webserver we must force to use: Apache or Litespeed
:return: None if webserver autodetect
False if apache should be used
True if litespeed should be used
"""
if 'litespeed' in config:
option = config['litespeed'].lower()
if option in ['on', '1']:
return True
if option in ['off', '0']:
return False
return None
def _incomplete_incidents_query(self):
# type: () -> sqlalchemy.orm.query.Query
"""Generate sqlalchemy Query instance for select incomplete incidents"""
return self.session.query(incident).filter(
incident.server_id == self.server_id,
incident.incident_end_time.is_(None))
def finalize_incidents(self):
# type: () -> None
not_finalize_incidents_query = self._incomplete_incidents_query().filter(
incident.dump_time < self.now - self.period_between_incidents)
finalized_numb = not_finalize_incidents_query.update({incident.incident_end_time: incident.dump_time})
self.log.debug('%i old incidents period was finalized' % finalized_numb)
self.session.commit()
def save_old_incidents(self):
# type: () -> None
old_incidents = {uid: ts for uid, ts in list(self.incidents_last_ts.items()) if ts < self.now - self.period_between_incidents}
if old_incidents:
try:
for _inc in self.session.query(incident).filter(
incident.server_id == self.server_id,
incident.incident_end_time.is_(None),
incident.uid.in_(list(old_incidents.keys()))).all():
_inc.incident_end_time = old_incidents[_inc.uid]
except Exception as e:
self.session.rollback()
self.log.error("Unable to save old incidents: (%s)", str(e))
else:
self.session.commit()
for uid in list(old_incidents.keys()):
self.incidents_last_ts.pop(uid)
try:
self.incidents_cache.pop(uid)
except KeyError:
pass
def get_incident(self, uid):
# type: (int) -> incident
_inc = self._incomplete_incidents_query().filter(
incident.uid == uid).first()
if not _inc:
_inc = incident(uid=uid, incident_start_time=self.now,
server_id=self.server_id, snapshot_count=0,
incident_end_time=None)
self.session.add(_inc)
self.log.debug(
'New incident-period for uid %s started; incident_start_time=%i'
% (uid, self.now))
self.incidents_last_ts[uid] = self.now
return _inc
def init_session(self):
if self.first_run: # Running for the first time
self.session = sessionmaker(bind=self.engine)()
self.finalize_incidents()
self.first_run = False
def process_lve(self, lve_id, faults):
# type: (int, Dict[str, int]) -> None
try:
lve_id = int(lve_id)
self.incidents_last_ts[lve_id] = self.now
_inc = self.incidents_cache[lve_id]
self.log.debug(
'Faults %s for uid %s detected; timestamp %i' %
(faults, lve_id, self.now))
if _inc["snapshot_count"] < self.max_snapshots_per_incident:
self.save_snapshot(_inc, faults)
self.log.debug(
'Snapshot for uid %s with timestamp %i saved'
% (lve_id, self.now))
except Exception as e:
self.session.rollback()
self.log.warning("Unable to save incident for LVE %s (%s)" % (lve_id, str(e)))
def setup_litespeed(self, force_webserver_message=False):
# type: (bool) -> None
"""Check state and configure access to litespeed"""
self.litespeed_info.check_litespeed_state()
if self.litespeed_info.state_changed and self.litespeed_info.is_running:
self.litespeed_info.dump_passwd()
if self.litespeed_info.state_changed or force_webserver_message:
use_litespeed = self.litespeed_info.get_use_litespeed()
litespeed_running = self.litespeed_info.is_running
if use_litespeed and not litespeed_running:
self.log.info("Litespeed is not running properly. "
"Check litespeed license key.")
webserver = LITESPEED if use_litespeed else APACHE
msg = "{} webserver will be used now to obtain data".format(webserver)
self.log.info(msg)
def execute(self, lve_data):
# type: (dict) -> None
if not self.snapshots_enabled:
return
self.init_session()
self.setup_litespeed()
lve_ids = list(lve_data.get('lve_usage', {}).keys())
lve_faults = lve_data.get('faults', {})
self._helper.invalidate(lve_ids)
self.cache_snapshots(lve_faults)
if self.now - self.last_run >= self._period:
self.last_run = self.now
with reboot_lock():
self.save_old_incidents()
self._insert_new_incidents(lve_faults)
self._increment_snapshot_count(lve_faults)
for lve_id, faults in list(lve_faults.items()):
self.process_lve(lve_id, faults)
lve_data["faults"] = {}
self._snapshots_data = {}
def _increment_snapshot_count(self, lve_faults):
# type: (Dict[int, Dict[str, int]]) -> None
lve_id_list = list(lve_faults.keys())
for chunk in get_chunks(lve_id_list, 250):
self._incomplete_incidents_query().filter(
incident.uid.in_(chunk),
incident.snapshot_count < self.max_snapshots_per_incident
).update({
"snapshot_count": incident.snapshot_count + 1,
"dump_time": int(self.now)
}, synchronize_session=False)
self.session.commit()
def _insert_new_incidents(self, lve_faults):
# type: (Dict[int, Dict[str, int]]) -> None
new_incidents = {lve_id: {
'uid': lve_id,
"incident_start_time": self.now,
"server_id": self.server_id,
"snapshot_count": 0,
"incident_end_time": None
} for lve_id, _ in list(lve_faults.items()) if lve_id not in self.incidents_cache}
if new_incidents:
self.incidents_cache.update(new_incidents)
insert_list = (incident(**_inc) for _inc in list(new_incidents.values()))
try:
# Better to use
# self.session.bulk_insert_mappings(incident, new_incidents)
# but it will be available only in SQLAlchemy version > 1.0
self.session.add_all(insert_list)
except Exception as e:
self.log.error("Unable to save new incidents: %s", str(e))
else:
self.session.commit()
def cache_snapshots(self, lve_faults):
# type: (Dict[int, Dict[str, int]]) -> None
for lve_id in list(lve_faults.keys()):
if lve_id not in self._snapshots_data:
self._snapshots_data[lve_id] = \
self._helper.get_snapshot_data(lve_id, self.litespeed_info)
def save_snapshot(self, _incident, faults):
# type: (dict, Dict[str, int]) -> None
lve_id = _incident["uid"]
processes, queries, urls = self._snapshots_data.get(lve_id, ({}, [], []))
snapshot_ = Snapshot(_incident, self.compresslevel)
data = {
'uid': _incident["uid"],
'dump_time': int(self.now),
'server_id': self.server_id,
'incident_start_time': _incident["incident_start_time"],
'snap_proc': processes,
'snap_sql': queries,
'snap_http': urls,
'snap_faults': faults}
try:
snapshot_.save(data)
except IOError as e:
self.log.error(str(e))
_incident["snapshot_count"] += 1
Zerion Mini Shell 1.0