# coding=utf-8
#
# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2019 All Rights Reserved
#
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENSE.TXT
import json
import logging
import time
from typing import Callable # NOQA
import lvestats.lib.commons.decorators
from clcommon import FormattedException, cpapi
from clcommon.cpapi import NotSupported
from clcommon.cpapi.pluginlib import getuser
from clcommon.lib import GovernorStatus, MySQLGovernor, MySQLGovException
from lvestats.lib import dbengine
from lvestats.lib.commons import func, server_status
from lvestats.lib.commons.func import get_users_for_reseller, skip_user_by_maxuid
from lvestats.lib.commons.logsetup import setup_logging
from lvestats.lib.commons.users_manager import g_users_manager
from lvestats.lib.config import HIDE_MAX_UID_LVE_PARAMETER, str_to_bool
from lvestats.lib.parsers.cloudlinux_top_argparse import cloudlinux_top_parser
from lvestats.lib.uidconverter import uid_to_username
NOT_AVAILABLE = "N/A"
class CloudLinuxTop(object):
def __init__(self, cnf, engine=None):
self._log = setup_logging(
config=cnf, caller_name="CloudLinuxTop", file_level=logging.WARNING,
console_level=logging.FATAL)
self.fname = '/var/lve/cloudlinux_top.json'
self.fname_db = '/var/lve/cloudlinux_dbtop.json'
self.touch_fname = '/var/lve/governor.ts'
self.server_id = cnf.get('server_id', 'localhost')
self.hide_maxuid_lve = str_to_bool(cnf.get(HIDE_MAX_UID_LVE_PARAMETER, 'true'))
self.engine_error = None
if engine is not None:
self.engine = engine
else:
try:
self.engine = dbengine.make_db_engine(cnf)
except dbengine.MakeDbException as e:
self.engine_error = e
self.mysql_governor = MySQLGovernor()
self.governor_status, self.governor_error = \
self.mysql_governor.get_governor_status()
self.governor_mode = func.get_governor_mode(self.mysql_governor)
self._admins = self._get_admins() if getuser() == 'root' else set()
@staticmethod
def _get_admins():
try:
return cpapi.admins()
except (NotSupported, AttributeError):
return set()
def _get_username_by_id(self, uid):
return uid_to_username(
uid=uid,
local_server_id=self.server_id,
server_id=self.server_id,
db_engine=self.engine
)
def _get_order_key(self, order, cl_data):
# type: (str, dict) -> Callable[dict]
"""returns key function for sorted"""
def mysql_key(value):
def key(k):
username = self._get_username_by_id(k["id"])
if username not in cl_data["dbgov_data"]:
return 0
else:
dbgov_stats = cl_data["dbgov_data"][username]
return dbgov_stats[value]
return key
order_key_map = {
"cpu": lambda k: k["usage"]["cpu"]["all"],
"io": lambda k: k["usage"]["io"]["all"],
"mysql_cpu": mysql_key("cpu"),
"mysql_io": mysql_key("io"),
"iops": lambda k: k["usage"].get("iops", 0),
"ep": lambda k: k["usage"]["ep"],
"nproc": lambda k: k["usage"].get("pno", 0),
"pmem": lambda k: k["usage"].get("mem", 0),
}
return order_key_map[order]
def _load_from_json_file(self, fname, default=None):
"""
Try load and parse json data from file; return <default> if loading or
parsing wrong
:param fname: path to parsed file
:param None|dict default: default data if loaded file is wrong
:return None| dict: return loaded data
"""
try:
with open(fname, encoding='utf-8') as json_file:
cl_data = json.load(json_file)
except (IOError, OSError):
cl_data = default
return cl_data
def _touch_dbtop(self):
"""
Trying touch the file to signal new data
"""
try:
func.touch(self.touch_fname)
except (IOError, OSError):
self._log.error("An error occurred while touching file %s. "
"Database statistics may be not available", self.touch_fname)
def _load_data(self):
"""
Load and join data from separate files
:return dict: loaded data
"""
cl_data = self._load_from_json_file(self.fname, {"users": [], "resellers": []})
cl_data.update(self._load_from_json_file(self.fname_db, {"dbgov_data": {}}))
return cl_data
def _get_data(self,
username_filter=None,
domain_filter=None,
limit=None,
show_mysql=True,
order="cpu",
reseller=None):
"""
:type order: str
:type show_mysql: bool
:type limit: int|None
:type domain_filter: str|None
:type username_filter: str|None
:param None|str reseller: get information about given reseller
:rtype: tuple[list, list]
"""
users_result = []
resellers_result = []
users_list = []
current_user_name = getuser()
current_user_is_admin = current_user_name == 'root'
if current_user_is_admin or reseller:
self._touch_dbtop() # touch file as signal to save dbgov data to file
g_users_manager.build_users_cache(reseller)
if reseller:
users_list = get_users_for_reseller(reseller or getuser()) # show reseller information too
else:
users_list = [current_user_name]
cl_data = self._load_data()
cl_users_data_ordered = sorted(cl_data["users"], key=self._get_order_key(order, cl_data), reverse=True)
for user in cl_users_data_ordered:
# skip users with maxuid lve ids due to some client`s complaints (details: LVES-929)
if self.hide_maxuid_lve and skip_user_by_maxuid(user['id']):
continue
username = self._get_username_by_id(user["id"]) or NOT_AVAILABLE
# users must not be skipped if current user is root and when --for-reseller is not passed
should_skip_users = not current_user_is_admin or reseller is not None
if should_skip_users and username not in users_list or username in self._admins:
continue
if username == NOT_AVAILABLE:
user["domain"] = NOT_AVAILABLE
user["reseller"] = NOT_AVAILABLE
else:
user["reseller"] = g_users_manager.get_reseller(username, raise_exc=False) or NOT_AVAILABLE
user["domain"] = g_users_manager.get_domain(username, raise_exc=bool(domain_filter)) or NOT_AVAILABLE
if (domain_filter and (user["domain"] == NOT_AVAILABLE or domain_filter not in user["domain"])) or \
(username_filter and (username == NOT_AVAILABLE or username_filter not in username)):
continue
user["username"] = username
if show_mysql and self.governor_status == GovernorStatus.ENABLED:
self._add_mysql_data(user, cl_data, username)
users_result.append(user)
if limit is not None and 0 < limit <= len(users_result):
break
cl_resellers_data_ordered = \
sorted(cl_data.get("resellers", []), key=self._get_order_key(order, cl_data), reverse=True)
if current_user_is_admin:
for reseller_info in cl_resellers_data_ordered:
username = self._get_username_by_id(reseller_info['id'])
# get only reseller that we need (if needed)
if reseller and username != (reseller or getuser()):
continue
if username is None:
username = NOT_AVAILABLE
reseller_info["name"] = username
resellers_result.append(reseller_info)
return users_result, resellers_result
def _add_mysql_data(self, user, cl_data, username):
"""
add mysql statistics to the user dict
:type cl_data: dict
:type user: dict
:type username: str
"""
dbgov_stats = cl_data["dbgov_data"].get(
username,
{"cpu": 0, "io": 0, "cause_of_restrict": "-", "time_of_restrict": 0},
)
if self.governor_mode != "all":
try:
mysql_limits = self.mysql_governor.get_limits_by_user(username)
except MySQLGovException as e:
self.governor_status = GovernorStatus.ERROR
self.governor_error = e
else:
user["limit"]["cpu"]["mysql"] = mysql_limits[0]
user["limit"]["io"]["mysql"] = mysql_limits[1] * 1024 # in bytes
user["usage"]["cpu"]["mysql"] = dbgov_stats["cpu"]
user["usage"]["io"]["mysql"] = dbgov_stats["io"]
else:
user["limit"]["cpu"]["mysql"] = "-"
user["limit"]["io"]["mysql"] = "-"
user["usage"]["cpu"]["mysql"] = min(dbgov_stats["cpu"], user["usage"]["cpu"]["all"])
user["usage"]["io"]["mysql"] = min(dbgov_stats["io"], user["usage"]["io"]["all"])
if dbgov_stats["cause_of_restrict"] != "-":
user["restricted"] = True
user["restriction"] = {
"time": dbgov_stats["time_of_restrict"],
"reason": f"Violated the {dbgov_stats['cause_of_restrict']} limit"}
else:
user["restricted"] = False
@staticmethod
def to_json(value):
"""
:type value: dict
:rtype: str
"""
return json.dumps(value, indent=4, sort_keys=True)
@lvestats.lib.commons.decorators.no_sigpipe
def main(self, *args):
"""
:rtype: (str, int)
"""
parser = cloudlinux_top_parser()
result = {"timestamp": time.time()}
try:
namespace = parser.parse_args(args)
if not namespace.json:
return "Use --json option, other modes currently unsupported", 1
except ValueError as e:
result["result"] = e.args[0]
result["context"] = {}
return self.to_json(result), 0
if getuser() != 'root' and namespace.for_reseller is not None and namespace.for_reseller != getuser():
error_msg = 'Permission denied. Reseller can view ' \
'information only about himself. ' \
f'Option --for-reseller={namespace.for_reseller} is forbidden.'
result['result'] = error_msg
result["context"] = {}
return self.to_json(result), 1
lvestats_status = server_status.check_server_status()
if lvestats_status != server_status.RUNNING:
result["result"] = server_status.status_description[lvestats_status]
elif self.engine_error is not None:
result["result"] = self.engine_error.message
result["context"] = self.engine_error.context
else:
try:
users, resellers = self._get_data(
username_filter=namespace.username,
domain_filter=namespace.domain,
limit=namespace.max,
show_mysql=not namespace.hide_mysql,
order=namespace.order_by,
reseller=namespace.for_reseller)
except IOError as e:
result["result"] = str(e)
result["context"] = {}
except FormattedException as e:
result["result"] = e.message
result["context"] = e.context
else:
result.update({
"result": "success",
"users": users,
"resellers": resellers,
"mySqlGov": self.governor_status,
"mySqlGovMode": self.governor_mode})
# broken governor is a non-fatal problem
# and lve statistics should works even in that case
# so let's threat "error" status as "warning"
# and show "warning" message in web ui
# anyway, at the moment lvemanager ignores "errors" field
if self.governor_status == GovernorStatus.ERROR:
result["warning"] = self.governor_error.message
try:
result["context"] = self.governor_error.context
except AttributeError:
result["context"] = {}
return self.to_json(result), 0
|