# coding=utf-8
#
# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2019 All Rights Reserved
#
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENSE.TXT
import logging
import json
import time
import typing
from typing import TypedDict, NamedTuple, Self
from collections import ChainMap
from clcommon.clproc import ProcLve
from lveapi import PyLve
from lvectllib import LVE_DEFAULT
from lvestat import LVEStat
from lvestats.plugins.generic.burster.config import read_raw_config, get_boolean
from lvestats.plugins.generic.burster.overload import OverloadChecker, read_times_from_proc
EFFECTIVE_NORMAL_LIMITS_PATH = '/var/run/cloudlinux/effective-normal-limits'
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger('BurstWatcher')
# Separate debug log handler
handler = logging.FileHandler('/var/log/lvestats-burstwatcher.log')
handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
handler.setLevel(logging.DEBUG)
logger.addHandler(handler)
class Service:
def __init__(self) -> None:
self._pylve = PyLve()
if not self._pylve.initialize():
raise RuntimeError('pylve initialization failed')
config = _Config.read_from_config_file()
self._enabled = config.enabled
self._overload_checker = OverloadChecker(
idle_time_threshold=config.idle_time_threshold,
max_samples_number=config.max_samples,
get_stats=read_times_from_proc
)
self._current_bursting_lves = {}
self._effective_normal_limits = {}
def run(self) -> None:
cnt = 0
while True:
time.sleep(1)
if not self._enabled:
continue
if self._is_server_overloaded():
self._load_effective_normal_limits()
self._get_current_bursting_lves()
self._disable_bursting()
elif cnt % 15 == 0:
self._load_effective_normal_limits()
cnt = 0
cnt += 1
def _load_effective_normal_limits(self):
try:
with open(EFFECTIVE_NORMAL_LIMITS_PATH, 'r', encoding='UTF-8') as f:
loaded_effective_normal_limits = json.loads(f.read())
if len(loaded_effective_normal_limits) > 0:
self._effective_normal_limits = loaded_effective_normal_limits
except (OSError, IOError, json.JSONDecodeError):
logger.exception('Can`t load cached effective normal limits from file: %s', EFFECTIVE_NORMAL_LIMITS_PATH)
logger.debug('Loaded effective normal limits for %s lves', len(self._effective_normal_limits))
def _is_server_overloaded(self) -> bool:
checker_result = self._overload_checker()
logger.debug('Curent server load: %s', checker_result.server_load)
return checker_result.is_overloaded
def _get_current_bursting_lves(self):
proc_lve = ProcLve()
lve_stats = [LVEStat(line, proc_lve.version()) for line in proc_lve.lines(without_limits=True)]
self._current_bursting_lves = {
str(stat.id) for stat in lve_stats
if stat.cpu != self._effective_normal_limits.get(str(stat.id), {}).get('cpu') and stat.id != 0
}
logger.debug('Current bursting lves: %s', self._current_bursting_lves)
def _disable_bursting(self):
for _id in self._current_bursting_lves:
if _id == '0':
logger.debug('Skipping lve_id=0')
continue
self._disable_bursting_for_lve(_id)
logger.debug('Disabled bursting server-wide')
def _disable_bursting_for_lve(self, id_: str):
logger.debug('Disabling bursting for LVE %s', id_)
lve_limits = self._effective_normal_limits.get(id_, {})
default_limits = self._effective_normal_limits.get('0', {})
try:
lve_settings = self._pylve.liblve_settings()
lve_settings.ls_io = int(lve_limits.get('io', default_limits.get('io', LVE_DEFAULT['io'])))
lve_settings.ls_cpu = int(lve_limits.get('cpu', default_limits.get('cpu', LVE_DEFAULT['cpu'])))
lve_settings.ls_cpus = int(lve_limits.get('ncpu', default_limits.get('ncpu', LVE_DEFAULT['ncpu'])))
lve_settings.ls_memory = int(lve_limits.get('mem', default_limits.get('mem', LVE_DEFAULT['mem'])))
lve_settings.ls_enters = int(lve_limits.get('ep', default_limits.get('ep', LVE_DEFAULT['ep'])))
lve_settings.ls_memory_phy = int(lve_limits.get('pmem', default_limits.get('pmem', LVE_DEFAULT['pmem'])))
lve_settings.ls_nproc = int(lve_limits.get('nproc', default_limits.get('nproc', LVE_DEFAULT['nproc'])))
lve_settings.ls_iops = int(lve_limits.get('iops', default_limits.get('iops', LVE_DEFAULT['iops'])))
self._pylve.lve_setup(
int(id_), lve_settings, err_msg=f"Can't setup lve with id {id_}; error code {{code}}"
)
except RuntimeWarning as rw:
logging.exception(rw)
class _RawConfig(TypedDict, total=False):
bursting_enabled: bool
bursting_critical_load_threshold: str
bursting_critical_load_samples_num: str
class _Config(NamedTuple):
@classmethod
def read_from_config_file(cls) -> Self:
try:
raw_config = typing.cast(_RawConfig, read_raw_config(_logger=logger))
except FileNotFoundError:
logger.warning('Config file not found!')
raw_config = {}
default_params = {
"enabled": False,
"idle_time_threshold": 0.05,
"max_samples": 15,
}
external_params = {}
error_by_raw_key = {}
for raw_key, extractor, param in [
('bursting_enabled', get_boolean, 'enabled'),
('bursting_critical_load_threshold', float, 'idle_time_threshold'),
('bursting_critical_load_samples_num', int, 'max_samples'),
]:
try:
raw_value = raw_config[raw_key]
except KeyError:
continue
try:
value = extractor(raw_value)
except ValueError as e:
error_by_raw_key[raw_key] = str(e)
continue
external_params[param] = value
if error_by_raw_key:
logger.warning(
"Failed to parse some config keys: \n%s",
"\n".join(f"* {k}: {e}" for k, e in error_by_raw_key.items())
)
self = _Config(**ChainMap(external_params, default_params))
if (defaults_used := default_params.keys() - external_params.keys()):
logger.info('Using default values for: %s', defaults_used)
return self
enabled: bool
idle_time_threshold: float
max_samples: int
|