ok
Mini Shell
# -*- coding: utf-8 -*-
# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2021 All Rights Reserved
#
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENSE.TXT
# wposctl.py - work code for clwposctl utility
from __future__ import absolute_import
import argparse
import datetime
import itertools
import json
import logging
import os
import pwd
import subprocess
import sys
import requests
from copy import deepcopy
from dataclasses import asdict
from typing import Dict, Iterator, Set, Tuple, List, Optional
from enum import Enum
from psutil import pid_exists
from clcommon.cpapi import cpusers, userdomains, is_admin, cpinfo, getCPName
from clwpos.billing import get_or_create_unique_identifier
from clwpos.migrations import migrate_configs
from clwpos.cron import install_cron_files, clean_clwpos_crons
from clwpos.feature_suites.configurations import FeatureStatus, FeatureStatusEnum, AdminSuitesConfig, \
any_suite_visible_on_server, is_module_visible_for_user, StatusSource, extract_suites
from clwpos.optimization_features import (
ALL_OPTIMIZATION_FEATURES,
OBJECT_CACHE_FEATURE,
CDN_FEATURE,
enable_without_config_affecting,
disable_without_config_affecting,
DocRootPath,
SITE_OPTIMIZATION_FEATURE,
Feature
)
from clwpos.feature_suites import (
ALL_SUITES,
any_suite_allowed_on_server,
get_suites_allowed_path,
get_admin_suites_config,
write_suites_allowed,
extract_features,
is_module_allowed_for_user,
PremiumSuite,
CDNSuitePro,
CDNSuite,
AWPSuite
)
from clcommon.clpwd import drop_privileges
from clwpos.cl_wpos_exceptions import WposError
from clwpos.user.config import UserConfig
from clwpos.constants import (
ALT_PHP_REDIS_ENABLE_UTILITY,
CLWPOS_UIDS_PATH,
CLWPOS_SOLO_PATH,
PHP_REDIS_ENABLE_UTILITY,
SUITES_MARKERS,
MIGRATION_NEEDED_MARKER,
SCAN_CACHE,
ADMIN_ENABLE_FEATURE_STATUS,
ADMIN_ENABLE_FEATURE_PID,
ADMIN_UPDATE_OBJECT_CACHE_BANNER_PID,
USERS_PLUGINS_SYNCING_PID,
CLN_URL,
SMART_ADVICE_ROOT_UTILITY,
ON_OFF_IDENTIFIERS,
XRAY_MANAGER_UTILITY
)
from clwpos.object_cache.redis_utils import reload_redis
from clwpos import gettext as _, billing
from clwpos.parse import ArgumentParser, CustomFormatter
from clwpos.logsetup import setup_logging, init_wpos_sentry_safely, ADMIN_LOGFILE_PATH
from clcommon.lib.cledition import is_cl_solo_edition
from clcommon.cpapi.cpapiexceptions import NoPackage
from clwpos.report_generator import ReportGenerator, ReportGeneratorError
from clwpos.utils import (
catch_error,
error_and_exit,
print_data,
check_license_decorator,
set_wpos_icon_visibility,
acquire_lock,
write_public_options,
get_pw,
is_redis_configuration_running,
install_monitoring_daemon,
get_server_wide_options,
is_ui_icon_hidden,
ServerWideOptions,
daemon_communicate,
ExtendedJSONEncoder,
is_shared_pro_safely,
get_supported_suites,
jwt_token_check,
should_xray_user_agent_enabled,
should_xray_user_agent_disabled
)
from clwpos.wpos_hooks import (
install_panel_hooks,
install_yum_universal_hook_alt_php,
_uninstall_hooks
)
from clcommon.clcagefs import setup_mount_dir_cagefs, _remount_cagefs
from clwpos.stats import fill_current_wpos_statistics
from clwpos.data_collector_utils import has_wps
from secureio import disable_quota
WPOS_SERVICE_ENABLE_ERR_MSG = _("Unable to run CL AccelerateWP daemon. Caching databases won't start and work. "
"You can find detailed information in log file")
REDIS_CONFIGURATION_WARNING_MSG = _("Configuration of PHP redis extension is running in background process. "
"This may take up to several minutes. Until the end of this process "
"functionality of CL AccelerateWP is limited.")
parser = ArgumentParser(
"/usr/bin/clwpos-admin",
"Utility for control CL AccelerateWP admin interface",
formatter_class=CustomFormatter,
allow_abbrev=False
)
_logger = setup_logging(__name__)
class CloudlinuxWposAdmin(object):
"""
Class for run cloudlinux-wpos-admin commands
"""
class EnablingStatus(Enum):
"""
Basic statuses while feature is enabling in background
"""
IDLE = 'idle'
PROGRESS = 'progress'
DONE = 'done'
def __init__(self):
self._is_json = False
self._opts: argparse.Namespace
self._logger = setup_logging(__name__)
init_wpos_sentry_safely(self._logger)
self.clwpos_path = "/var/clwpos"
self.modules_allowed_name = "modules_allowed.json"
# TODO: [unification] remove this attr once it is unification is done
self.is_solo = is_cl_solo_edition(skip_jwt_check=True)
self.wait_child_process = bool(os.environ.get('CL_WPOS_WAIT_CHILD_PROCESS'))
if self.wait_child_process:
self.exec_func = subprocess.run
else:
self.exec_func = subprocess.Popen
@catch_error
def run(self, argv):
"""
Run command action
:param argv: sys.argv[1:]
:return: clwpos-user utility retcode
"""
self._opts = self._parse_args(argv)
self._is_json = True
result = getattr(self, self._opts.command.replace("-", "_"))()
print_data(self._is_json, result)
def _parse_args(self, argv):
raise NotImplementedError
@staticmethod
def _create_markers(suites_list):
for suite in suites_list:
if SUITES_MARKERS.get(suite) and not os.path.isfile(SUITES_MARKERS.get(suite)):
open(SUITES_MARKERS.get(suite), 'w').close()
@staticmethod
def _clear_markers(suites_list):
for suite in suites_list:
if SUITES_MARKERS.get(suite) and os.path.isfile(
SUITES_MARKERS.get(suite)):
os.unlink(SUITES_MARKERS.get(suite))
@staticmethod
def _is_true(opt):
return opt == 'on'
@parser.command() # Uninstall cache for all domain during downgrade
def uninstall_cache_for_all_domains(self) -> dict:
"""
This command used during downgrade to lve-utils, which version does not support clwpos
:return:
"""
try:
users = cpusers()
except (OSError, IOError, IndexError, NoPackage) as e:
self._logger.warning("Can't get user list from panel: %s", str(e))
return {}
for username in users:
user_domains = userdomains(username)
with drop_privileges(username):
for doc_root, wp_path, module in _enabled_modules(username):
domain = _extract_domain(user_domains,
os.path.join(pwd.getpwnam(username).pw_dir, doc_root))
disable_without_config_affecting(DocRootPath(doc_root), wp_path, module=module, domain=domain)
return {}
@check_license_decorator
def _set_suite(self) -> dict:
"""
Write info related to module allowance into user file
"""
defaults = get_server_wide_options()
suites_list = [suite.strip() for suite in self._opts.suites.split(",")]
for suite in suites_list:
if suite not in ALL_SUITES:
error_and_exit(self._is_json, {'result': _(f'Unsupported suite: {suite}')})
if (self._opts.allowed
or self._opts.visible_for_all
or self._opts.allowed_for_all) and\
suite not in get_supported_suites():
error_and_exit(self._is_json, {'result': _(f'Suite is disabled externally: {suite}')})
valid_attributes = {}
allowed_attrs = ALL_SUITES[suite].allowed_attrubites
for attribute in allowed_attrs:
if self._opts.attrs \
and attribute.name not in self._opts.attrs \
and attribute.default is None:
error_and_exit(self._is_json, {'result': _(f'Attribute %s does not have default '
f'value and must be included: {attribute.name}')})
elif not self._opts.attrs and attribute.default is not None:
valid_attributes[attribute.name] = attribute.default
else:
if attribute.name in self._opts.attrs:
valid_attributes[attribute.name] = attribute.type(self._opts.attrs[attribute.name])
# what we check here is whether billing is allowed to enable the feature
# that is true in one cases: if the feature is already visible for users
if self._opts.allowed and self._opts.source and \
getattr(StatusSource, self._opts.source) == StatusSource.BILLING_OVERRIDE \
and suite not in defaults.visible_suites:
error_and_exit(self._is_json, {
'result': _(f'Suite %(suite)s is not visible for users and so '
f'cannot be allowed in billing. '
f'Activate the suite on server first. '
f'Contact your hoster if you don`t have an access to the server.'),
'context': dict(suite=suite)})
if self._opts.default and len(suites_list) > 1:
error_and_exit(self._is_json, {'result': _(f'Only one suite is possible with --default option')})
if self._opts.attrs and len(suites_list) > 1:
error_and_exit(self._is_json, {'result': _(f'Only one suite is possible with --attrs option')})
# TODO: allowed_for_all works in the way that runs --allowed command for each new user in hook
# accroding to Dennis, hoster's unlikely would enable more then 50GB free for all users
# or they would do that using packages in WHMCS, so I allow only default configuration config here
# and create new task to fix this later upon requests: https://cloudlinux.atlassian.net/browse/AWP-546
if self._opts.attrs and self._opts.allowed_for_all:
error_and_exit(self._is_json, {'result': _(f'Only default suite configuration'
f' can be activated for all users')})
modules_list = [module for suite in suites_list for module in ALL_SUITES[suite].feature_set]
if self._opts.allowed_for_all:
# async call here is fine, also no need to wait
# in most cases it should work fast
# if it works slower - scanning will be in process in UI
self._logger.info('Going to generate users report')
if not os.path.isfile(SCAN_CACHE):
ReportGenerator().scan()
# CL Shared (Pro)
if self._opts.allowed_for_all:
# Process all panel users
user_list_to_process = cpusers()
module_allowed = module_visible = True
self._create_markers(suites_list)
elif self._opts.disallowed_for_all:
# Process all panel users
user_list_to_process = cpusers()
module_allowed = module_visible = False
self._clear_markers(suites_list)
elif self._opts.visible_for_all:
# Process all panel users
user_list_to_process = cpusers()
module_allowed = False
module_visible = True
self._clear_markers(suites_list)
else:
if self._opts.default:
defaults = get_server_wide_options()
module_visible = suites_list[0] in defaults.visible_suites
module_allowed = suites_list[0] in defaults.allowed_suites
else:
module_allowed = module_visible = self._opts.allowed
# Process only specified users
user_arg_list = self._opts.users.split(",")
user_list_to_process = [user_arg_list[0].strip()] # in v1 only single user processing is supported
first_user_wpos_visible = module_visible and not any_suite_visible_on_server()
first_user_cdn_allowed = module_allowed and CDN_FEATURE in modules_list \
and not is_module_visible_for_user(CDN_FEATURE)
first_user_obj_cache_visible = module_visible and OBJECT_CACHE_FEATURE in modules_list \
and not is_module_visible_for_user(OBJECT_CACHE_FEATURE)
warning_dict = {}
if module_allowed:
retcode, stdout, stderr = install_monitoring_daemon(True)
if retcode:
self._logger.error("Starting service ended with error: %s, %s", stdout, stderr)
warning_dict.update({"warning": WPOS_SERVICE_ENABLE_ERR_MSG})
error_flag = False
is_one_user_processing = len(user_list_to_process) == 1
if self._opts.source:
source_override = getattr(StatusSource, self._opts.source)
else:
source_override = None
if self._opts.preserve_user_settings:
user_list_to_process = tuple()
for username in user_list_to_process:
# update modules only after daemon startup
try:
_error_flag, warning_d = self._process_user_suites(
username,
suites_list,
(
FeatureStatusEnum.ALLOWED
if module_allowed else
FeatureStatusEnum.VISIBLE if module_visible else
FeatureStatusEnum.DISABLED
),
source_override or (
StatusSource.DEFAULT if any((
self._opts.allowed_for_all,
self._opts.disallowed_for_all,
self._opts.visible_for_all,
self._opts.default)
) else StatusSource.COMMAND_LINE
),
self._opts.purchase_date,
valid_attributes,
is_one_user_processing)
except Exception as e:
# ignore all errors for users processing during
# bulk operations in order not to interrupt it
# and raise otherwise (for individual users processing)
if is_one_user_processing:
self._logger.error(
"Error while processing module for '%s': %s",
username, str(e))
raise
self._logger.exception(
"Error while processing module for '%s', processing other users will be continued: %s",
username, str(e))
_error_flag = True
warning_d = {}
# Skip further user processing if error
if _error_flag:
# Set global error flag
error_flag = True
continue
if self._opts.allowed:
self.exec_func(
["/usr/sbin/clwpos_collect_information.py", username],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL
)
if module_visible and is_ui_icon_hidden():
# toggle icon if allow is in progress and icon is hidden in UI
set_wpos_icon_visibility(hide=False)
with write_public_options() as options:
options.show_icon = True
if should_xray_user_agent_enabled(module_visible):
self.exec_func([XRAY_MANAGER_UTILITY, 'enable-user-agent'],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
if self._opts.allowed_for_all:
# /usr/sbin/clwpos_collect_information.py without args processes all users
self.exec_func(["/usr/sbin/clwpos_collect_information.py"], stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
_remount_cagefs()
if first_user_wpos_visible:
setup_mount_dir_cagefs(
CLWPOS_UIDS_PATH, prefix='*', remount_cagefs=True,
remount_in_background=not self.wait_child_process
)
install_panel_hooks()
if first_user_obj_cache_visible:
# This runs after object_cache module is allowed for any user
# and there were no users on server who are allowed object_cache module before
warning_dict.update({"warning": REDIS_CONFIGURATION_WARNING_MSG})
self.exec_func([ALT_PHP_REDIS_ENABLE_UTILITY], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
self.exec_func([PHP_REDIS_ENABLE_UTILITY], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
install_yum_universal_hook_alt_php()
elif module_allowed and OBJECT_CACHE_FEATURE in modules_list and is_redis_configuration_running():
warning_dict.update({"warning": REDIS_CONFIGURATION_WARNING_MSG})
if first_user_wpos_visible or first_user_obj_cache_visible:
install_cron_files(
itertools.compress(
[SITE_OPTIMIZATION_FEATURE, OBJECT_CACHE_FEATURE, CDN_FEATURE],
[first_user_wpos_visible, first_user_obj_cache_visible, first_user_cdn_allowed]
), wait_child_process=True)
if self._opts.disallowed_for_all:
_remount_cagefs()
# determine the case of all suites becoming disallowed
# after manipulations with users
if not module_visible and not any_suite_visible_on_server():
_uninstall_hooks()
clean_clwpos_crons()
set_wpos_icon_visibility(hide=True)
with write_public_options() as options:
options.show_icon = False
# TODO: [unification] once xray default value is same across all editions -> is_solo check will not be needed
if should_xray_user_agent_disabled(self.is_solo):
self.exec_func([XRAY_MANAGER_UTILITY, 'disable-user-agent'],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
# MIGRATION_NEEDED_MARKER is created in .spec only during 1 upgrade
# before_renaming_version -> renaming_version
if (self._opts.allowed_for_all or self._opts.visible_for_all) and os.path.isfile(MIGRATION_NEEDED_MARKER):
self._logger.info('set-suite for all was called, removing migration marker')
os.remove(MIGRATION_NEEDED_MARKER)
if error_flag:
error_and_exit(
self._is_json,
{
"result": _("User(s) process error. Please check log file %(logfile)s"),
"context": {"logfile": ADMIN_LOGFILE_PATH},
}
)
self._save_suites_list_to_public_settings(suites_list)
if self._opts.allowed_for_all or self._opts.allowed:
# synchronize features to cln so at the time when
# daemon tries to enable feature our remote service
# already knows that feature is purchased
self.cln_sync(False)
self._force_user_sync(user_list_to_process)
# notify daemon that we enabled some suites
daemon_communicate({
'command': "suite_allowed_callback",
'uid': 0
})
return warning_dict
def _force_user_sync(self, userlist: List[str]):
"""
Communicates with AWP provision server asking it to fetch data about user ASAP.
"""
if not os.path.exists(SMART_ADVICE_ROOT_UTILITY):
self._logger.warning('Force syncing is skipped, because alt-php-xray package is not installed')
return
account_ids = []
for username in userlist:
try:
account_ids.append(get_or_create_unique_identifier(username))
except Exception:
self._logger.exception('Unable to obtain account_id for user %s. ', username)
continue
if not account_ids:
self._logger.warning('No account_ids to be synced')
return
try:
subprocess.check_output([SMART_ADVICE_ROOT_UTILITY,
'awp-sync',
'--account_id',
','.join(account_ids)],
stderr=subprocess.PIPE, text=True)
except subprocess.CalledProcessError as e:
self._logger.warning('Unable to force synchronization of data for user %s. '
'stderr: %s; stdout: %s', str(userlist), e.stderr, e.stdout)
def _save_suites_list_to_public_settings(self, suites_list):
"""
Saves list of suites that are currently allowed/visible/disallowed
to config which each user on server can read.
"""
with write_public_options() as options:
for suite in [ALL_SUITES[suite] for suite in suites_list]:
if self._opts.allowed_for_all:
options.allowed_suites = list(set(options.allowed_suites) | {suite.name})
options.visible_suites = list(set(options.visible_suites) | {suite.name})
elif self._opts.visible_for_all:
options.allowed_suites = list(set(options.allowed_suites) - {suite.name})
options.visible_suites = list(set(options.visible_suites) | {suite.name})
elif self._opts.disallowed_for_all:
options.allowed_suites = list(set(options.allowed_suites) - {suite.name})
options.visible_suites = list(set(options.visible_suites) - {suite.name})
@check_license_decorator
def _set_options(self) -> dict:
"""
Set global options that affect all users.
For v1 it is only allowed to control WPOS icon visibility.
"""
if self._opts.icon_visible:
is_visible = self._is_true(self._opts.icon_visible)
retcode, stdout = set_wpos_icon_visibility(hide=not is_visible)
if retcode:
error_and_exit(
self._is_json,
{
"result": _("Error during changing of AccelerateWP icon visibility: \n%(error)s"),
"context": {"error": stdout}
},
)
with write_public_options() as options:
if self._opts.icon_visible is not None:
if self._is_true(self._opts.icon_visible):
options.show_icon = True
else:
options.show_icon = False
if self._opts.object_cache_banner_visible is not None:
if self._is_true(self._opts.object_cache_banner_visible):
options.disable_object_cache_banners = False
else:
options.disable_object_cache_banners = True
if self._opts.smart_advice_notifications is not None:
if self._is_true(self._opts.smart_advice_notifications):
options.disable_smart_advice_notifications = False
else:
options.disable_smart_advice_notifications = True
if self._opts.smart_advice_reminders is not None:
if self._is_true(self._opts.smart_advice_reminders):
options.disable_smart_advice_reminders = False
else:
options.disable_smart_advice_reminders = True
if self._opts.upgrade_url is not None:
if self._opts.suite == PremiumSuite.name:
options.upgrade_url = self._opts.upgrade_url or None
elif self._opts.suite == CDNSuitePro.name:
options.upgrade_url_cdn = self._opts.upgrade_url or None
if self._opts.feature_visible and self._opts.features:
self.set_feature(self._opts.features, self._is_true(self._opts.feature_visible))
return {}
def set_feature(self, features, is_visible):
features = [feature.strip() for feature in features.split(',')]
# accelerate_wp is base for many features (CDN, image optimization, critical css)
# do not see any sense to allow admin hide this feature
possible_to_hide = [f.to_interface_name() for f in ALL_OPTIMIZATION_FEATURES
if f != SITE_OPTIMIZATION_FEATURE.optimization_feature()]
for feature in features:
if feature not in possible_to_hide:
error_and_exit(
self._is_json,
{
"result": _("Invalid feature name passed: \n%(feature)s.\n"
" You may set only those features: %(supported)s"),
"context": {"feature": feature, "supported": possible_to_hide}
},
)
target_features = set([Feature(f).optimization_feature() for f in features])
already_hidden = set(get_server_wide_options().hidden_features)
if not is_visible:
features_to_hide = list(target_features.union(already_hidden))
all_premium_features = PremiumSuite.feature_set
all_free_features = AWPSuite.feature_set.union(CDNSuite.feature_set)
# suite_features: ['object_cache', 'critical_css', 'image_optimization']
# to hide: ['object_cache', 'critical_css', 'image_optimization', 'cdn']
# do not support case of hiding all features of suites, to prevent weird cases in UI
if all_premium_features.issubset(features_to_hide):
error_and_exit(
self._is_json,
{
"result": _("Hiding all features of AccelerateWP Premium is not permitted, "
"please use 'set-suite' command instead to disallow all "
"Premium features to users: \n%(feature)s"),
"context": {"feature": features_to_hide}
},
)
if all_free_features.issubset(features_to_hide):
error_and_exit(
self._is_json,
{
"result": _("Hiding all features of AccelerateWP is not permitted, "
"please use 'set-suite' command instead to disallow all "
"Premium features for users: \n%(feature)s"),
"context": {"feature": features_to_hide}
},
)
else:
# already_hidden: ['image_optimization', 'critical_cass']
# to show: ['critical_css']
# result: ['image_optimization']
features_to_hide = list(already_hidden.difference(target_features))
with write_public_options() as options:
options.hidden_features = features_to_hide
# just in case some users already enabled feature --> disable it on feature --hide
if not is_visible and features_to_hide:
for user in list(cpusers()):
try:
self._disable_feature(features_to_hide, user)
except Exception:
self._logger.exception('Unable to disable feature on set-feature --hide '
'for user %s', user)
continue
def _disable_feature(self, features, user):
user_domains = userdomains(user)
with drop_privileges(user):
enabled_features = UserConfig(user).enabled_modules()
for doc_root, wp_path, module in enabled_features:
domain = _extract_domain(user_domains,
os.path.join(pwd.getpwnam(user).pw_dir, doc_root))
if module not in features:
continue
command = ['/usr/bin/clwpos-user', '--user', user, 'disable', '--feature', module,
'--wp-path', wp_path, '--domain', domain]
result = subprocess.run(command, text=True, capture_output=True)
self._logger.info(result.stdout)
self._logger.info(result.stderr)
@catch_error
@check_license_decorator
def _get_options(self):
try:
return asdict(get_server_wide_options())
except json.decoder.JSONDecodeError as err:
raise WposError(
message=_(
"File is corrupted: Please, delete file mentioned in details or fix the corrupted line"),
details=str(err))
@catch_error
def _get_report(self) -> dict:
"""
Print report in stdout.
"""
report = {}
try:
users = self._opts.users.split(',') if self._opts.users else None
report = ReportGenerator().get(target_users=users)
except ReportGeneratorError as e:
error_and_exit(
self._is_json,
{
'result': e.message,
'context': e.context
}
)
except Exception as e:
error_and_exit(
self._is_json,
{
'result': _('Error during getting report: %(error)s'),
'context': {'error': e},
}
)
return report
def _generate_report(self) -> dict:
rg = ReportGenerator()
try:
if self._opts.status:
scan_status = rg.get_status()
else:
# TODO: implement --users support: send List[str] argument
scan_status = rg.scan() # initial status dict, like 0/10
return {
'result': 'success',
**scan_status,
}
except ReportGeneratorError as e:
error_and_exit(
self._is_json,
{
'result': e.message,
'context': e.context
}
)
except Exception as e:
error_and_exit(
self._is_json,
{
'result': _('Error during generating report: %(error)s'),
'context': {'error': str(e)},
}
)
@catch_error
def _get_stat(self) -> dict:
"""AccelerateWP statistics"""
return fill_current_wpos_statistics()
@catch_error
def _enable_feature(self):
"""
cli command for enabling optimization feature for all sites on the server
(where possible)
"""
# now only 1 feature is available to enable, will add parameter in case needed for other features as well
feature = 'accelerate_wp'
if self._opts.users:
users = self._opts.users.split(',')
else:
users = list(cpusers())
if self._opts.status:
return self._feature_enable_status()
if os.path.exists(ADMIN_ENABLE_FEATURE_PID):
error_and_exit(
self._is_json,
{
'result': _('Feature enabling is already in progress. '
'Process pid is stored in %(progress_marker)s.'
'If process with such pid does not exist - please, remove file "%(progress_marker)s" '
'and re-run command'),
'context': {'progress_marker': ADMIN_ENABLE_FEATURE_PID},
}
)
if os.path.exists(ADMIN_ENABLE_FEATURE_STATUS):
os.remove(ADMIN_ENABLE_FEATURE_STATUS)
enable_result = self.run_in_background(ADMIN_ENABLE_FEATURE_PID, self._enable_feature_for_users, users, feature)
if enable_result is None:
# parent
return {'result': 'success', 'status': self.EnablingStatus.PROGRESS.value, 'total_users_count': len(users)}
total_websites_to_enable, total_enabled_websites = enable_result
with open(ADMIN_ENABLE_FEATURE_STATUS, 'w') as f:
json.dump({
'wordpress_sites_to_enable_feature': total_websites_to_enable,
'wordpress_sites_with_enabled_feature': total_enabled_websites,
'total_users_count': len(users)
}, f, indent=4)
sys.exit(0)
def _feature_enable_status(self):
"""
Check current status of enabling feature process
- idle: if no pid file
- in progress: if pid file exists and such pid really exists in process list
- done: is enabling status file exists (which is created when process finishes)
"""
data = {'result': 'success'}
status = self.EnablingStatus.IDLE.value
if os.path.exists(ADMIN_ENABLE_FEATURE_PID):
with open(ADMIN_ENABLE_FEATURE_PID) as f:
pid = f.readline().strip()
if pid_exists(int(pid)):
status = self.EnablingStatus.PROGRESS.value
if os.path.exists(ADMIN_ENABLE_FEATURE_STATUS):
status = self.EnablingStatus.DONE.value
with open(ADMIN_ENABLE_FEATURE_STATUS) as f:
data.update(json.load(f))
if data.get('wordpress_sites_to_enable_feature') != data.get('wordpress_sites_with_enabled_feature'):
data['warning'] = 'Feature was enabled not for all sites on the server, due to some errors. ' \
f'Please, take a look at {ADMIN_LOGFILE_PATH}'
data['status'] = status
return data
def run_in_background(self, pidfile, func, *args, **kwargs):
"""
Forks child process in background if needed and created pid file with forked pid
"""
# dummy func result if func returns nothing
dummy_result = True
fp = os.fork()
if fp:
# parent process
return
# redirect everything to devnull in order not to block parent
si = open(os.devnull, 'r')
so = open(os.devnull, 'a+')
se = open(os.devnull, 'a+')
os.dup2(si.fileno(), sys.stdin.fileno())
os.dup2(so.fileno(), sys.stdout.fileno())
os.dup2(se.fileno(), sys.stderr.fileno())
# child process
child_pid = os.getpid()
with open(pidfile, 'w') as f:
f.write(str(child_pid))
try:
func_result = func(*args, **kwargs)
return func_result if func_result is not None else dummy_result
finally:
if os.path.exists(pidfile):
os.remove(pidfile)
def _enable_feature_for_users(self, usernames, feature):
"""
Enables passed feature for passed username`s sites
domains and target sites are obtained via user cli utility get (which knows all
about incompatibilities and able to detect sites/domains correctly)
for those sites which do not have any incompatibilities - run user cli command enable feature
returns <how many sites we should enable> and <how many sites were in fact enabled w/o errors>
"""
total_websites_to_enable, total_enabled_websites = 0, 0
for username in usernames:
try:
target_wps = self._get_target_websites(username, feature)
except Exception:
self._logger.exception('Unable to get websites info for enabling feature "%s"',
feature)
continue
total_websites_to_enable += sum(len(websites) for websites in target_wps.values())
for domain, websites in target_wps.items():
for wp_site in websites:
self._logger.info('Enabling optimization feature "%s" for user "%s" website "%s"',
feature,
username,
wp_site)
try:
enable_result = self._enable_for_site(username, feature, wp_site, domain,
self._opts.ignore_errors,
self._opts.skip_dns_check)
enabled_status = enable_result['feature']['enabled']
except Exception:
self._logger.exception('Failed to enable feature "%s" for website "%s". ',
feature,
wp_site)
continue
if enabled_status:
total_enabled_websites += 1
return total_websites_to_enable, total_enabled_websites
def _get_target_websites(self, username, feature):
"""
Gets target websites via user command GET
"""
target_wps = {}
# skip heavy `get` once we detect no WP sites for user
if has_wps(username):
result = subprocess.run(
['/usr/bin/clwpos-user', '--user', username, 'get'],
capture_output=True, text=True, timeout=600)
docroots_info = json.loads(result.stdout)['docroots']
for docroot in docroots_info:
domain = docroot['domains'][0]
target_wps[domain] = [
wp['path']
for wp in docroot['wps']
if not wp['features'][feature]['enabled'] and not wp['features'][feature].get('issues')
]
return target_wps
def _enable_for_site(self, username, feature, wp_site, domain, ignore_errors, skip_dns_check):
"""
Enables feature via user cli command ENABLE
"""
command = ['/usr/bin/clwpos-user', '--user', username, 'enable', '--feature', feature,
'--wp-path', wp_site, '--domain', domain]
if ignore_errors:
command.append('--ignore-errors')
elif skip_dns_check:
command.append('--skip-dns-check')
result = subprocess.run(command, text=True, capture_output=True, timeout=180)
self._logger.info(result.stdout)
self._logger.info(result.stderr)
return json.loads(result.stdout)
@catch_error
@parser.command()
def billing_check(self):
"""
Automatically turn off suites that are not suppoted on this server.
"""
supported_suites = get_supported_suites()
suites_to_turn_off = set(ALL_SUITES.keys()) - set(supported_suites)
if not suites_to_turn_off:
return
logging.info('Suites %s need to be turned off on the server', suites_to_turn_off)
result = subprocess.run(['cloudlinux-awp-admin', 'set-suite',
'--suites', ','.join(sorted(suites_to_turn_off)),
'--disallowed-for-all'],
text=True, capture_output=True, timeout=3600)
self._logger.info(result.stdout)
self._logger.info(result.stderr)
@catch_error
@parser.command() # Synchronize billing usage statistics with CLN
def cln_sync(self, exit=True) -> dict:
"""AccelerateWP cln integration"""
report = billing.get_report()
cln_report_url = f'{CLN_URL}/cln/api/clos/server/addons/v2'
success, err, jwt_str = jwt_token_check()
if not success:
self._logger.warning('JWT error: %s, report to CLN will not be sent', err)
else:
r = requests.post(cln_report_url,
data=json.dumps(report, cls=ExtendedJSONEncoder),
headers={'Authorization': f'JWT {jwt_str.decode("utf-8")}',
'Content-Type': 'application/json'})
if r.status_code != 200:
self._logger.error('CLN report sending failed with status: %s, http response: %s',
str(r.status_code), r.text)
return report
@catch_error
@parser.command() # migrate configs
def migrate_configs(self):
migrate_configs()
return True
@catch_error
@parser.command() # Synchronize cron files with current status
def sync_cron_files(self):
"""
Install cron files based on current status of wpos.
This code is executed after updates to add new
cron files that might be missing.
"""
install_cron_files(
itertools.compress(
[
SITE_OPTIMIZATION_FEATURE,
CDN_FEATURE,
OBJECT_CACHE_FEATURE
],
[
any_suite_visible_on_server(),
is_module_allowed_for_user(CDN_FEATURE),
is_module_allowed_for_user(OBJECT_CACHE_FEATURE)
]
), wait_child_process=True)
return True
@staticmethod
def all_suites_disabled(suites_admin_config: AdminSuitesConfig,
default_config: ServerWideOptions) -> bool:
"""
Check if all feature suites are disabled.
"""
for suite, status in suites_admin_config.suites.items():
if status in [FeatureStatusEnum.VISIBLE, FeatureStatusEnum.ALLOWED]:
return False
if status == FeatureStatusEnum.DEFAULT:
any_feature_enabled_by_default = \
set(ALL_SUITES[suite].feature_set) & set(default_config.allowed_features)
if any_feature_enabled_by_default:
return False
return True
def _is_supported_for_user(self, username):
user_owner = cpinfo(cpuser=username, keyls=('reseller',))[0][0]
# admin is owned by "root", but "root" is not is_admin()
should_be_whitelisted = getCPName() == 'DirectAdmin' and username == 'admin'
return any([is_admin(user_owner), should_be_whitelisted])
def _process_user_suites(self, user_name: str, suites: List[str],
allowed_state: FeatureStatusEnum,
state_source: StatusSource,
purchase_date: datetime.date,
attributes: Dict,
is_one_user: bool) -> Tuple[bool, Optional[dict]]:
"""
Enable/disable modules for user.
- write admin config for user with new state
- install/uninstall WP plugin
- reload deamon to start/stop redis
:param user_name: username
:param suites: Suites list to process
:param allowed_state: Suite state
:param purchase_date: Date when user last payed for the product
:param is_one_user: True - utility processes one user, False - some users
For messages backward compatibility
:return: Tuple: (error_flag, warning_flag)
"""
# Get modules_allowed.json for user
try:
pw_info = get_pw(username=user_name)
uid, gid = pw_info.pw_uid, pw_info.pw_gid
except KeyError:
if is_one_user:
error_and_exit(
self._is_json,
{
"result": _("User %(username)s does not exist."),
"context": {"username": user_name},
},
)
self._logger.error("User %s does not exist.", user_name)
return True, None
if allowed_state != FeatureStatusEnum.DISABLED and \
not self._is_supported_for_user(user_name):
self._logger.warning('User username="%s" is owned by reseller, suites="%s" cannot be allowed',
user_name,
str(suites))
if is_one_user:
error_and_exit(
self._is_json,
{
"result": _("User %(username)s is owned by reseller, "
"--visible or --allowed states cannot be set"),
"context": {"username": user_name},
},
)
# pid file is cleaned up after finishing syncing plugins in background,
# until process is running - block command
if os.path.exists(USERS_PLUGINS_SYNCING_PID.format(uid=str(uid))):
if is_one_user:
error_and_exit(
self._is_json,
{
"result": _("Plugins syncing in currently running for user %(username)s, "
"please wait and try again. If issue persists - check file presence %(sync_pid)s "
"and try to remove it"),
"context": {"username": user_name,
'sync_pid': USERS_PLUGINS_SYNCING_PID.format(uid=str(uid))},
},
)
self._logger.error("Plugins syncing in currently running for user %s", user_name)
return True, None
suites_allowed_path = get_suites_allowed_path(uid)
warning_dict = {}
try:
os.makedirs(os.path.dirname(suites_allowed_path), 0o755, exist_ok=False)
except OSError:
pass
else:
if is_one_user:
_remount_cagefs(user_name)
# automatically create user identifier when
# he is allowed to use feature
if allowed_state == FeatureStatusEnum.ALLOWED:
get_or_create_unique_identifier(user_name)
with acquire_lock(suites_allowed_path):
default_config = get_server_wide_options()
config_contents = get_admin_suites_config(uid)
features_old_state = extract_features(
deepcopy(config_contents),
default_config,
allowed_state
)
unsupported_suites_for_reseller = [PremiumSuite.name, CDNSuitePro.name, CDNSuite.name]
for suite in suites:
if suite in unsupported_suites_for_reseller and \
allowed_state != FeatureStatusEnum.DISABLED and \
not self._is_supported_for_user(user_name):
self._logger.info('Silently disallow %s user because it is owned by reseller', user_name)
config_contents.suites[suite] = FeatureStatusEnum.DISABLED
else:
config_contents.suites[suite] = allowed_state
config_contents.sources[suite] = state_source
if attributes:
config_contents.attributes[suite] = attributes
if purchase_date:
config_contents.purchase_dates[suite] = str(purchase_date)
features_new_state = extract_features(
deepcopy(config_contents),
default_config,
allowed_state
)
try:
write_suites_allowed(uid, gid, config_contents)
except (IOError, OSError) as err:
if is_one_user:
raise WposError(
message=_("Configuration file '%(path)s' update failed."),
details=str(err),
context=dict(path=suites_allowed_path)
)
self._logger.error("Configuration file %s update failed. Error is %s",
suites_allowed_path, str(err))
return True, None
self.synchronize_plugins_if_needed(allowed_state, state_source,
user_name, uid, features_old_state, features_new_state)
return False, warning_dict
def synchronize_plugins_if_needed(self, state, source, username, uid, features_old_state, features_new_state):
"""
1. does not sync plugins if source == BILLING (WHMCS CALL) and state == allowed
2. start syncing plugins in background if source == BILLING (WHMCS CALL) and state != allowed
3. it syncs plugins in regular regime for both allowed/non-allowed states if source != BILLING
"""
# set-suite command execution time is limited to 25s by our billing
# implementation -> call uninstalling/installing plugin in background
if source == StatusSource.BILLING_OVERRIDE:
_logger.info('Syncing plugins in background for user: %s', username)
# start in background
sync_result = self.run_in_background(USERS_PLUGINS_SYNCING_PID.format(uid=str(uid)),
synchronize_plugins_status_for_user,
username,
uid,
features_old_state,
features_new_state)
if sync_result is not None:
self._logger.info('Forked syncing plugins for user %s, '
'features old states: %s, features new states: %s',
username,
str(features_old_state),
str(features_new_state))
# ATTENTION!!!!! BECAUSE HERE WE ARE IN CHILD PROCESS
sys.exit(0)
else:
_logger.info('Start syncing plugins for user: %s, source: %s, state: %s',
username,
str(source),
str(state))
synchronize_plugins_status_for_user(username, uid, features_old_state, features_new_state)
def _object_cache_banner(self):
if self._opts.disable:
disable_object_cache_banners = True
else:
disable_object_cache_banners = False
if self._opts.all:
users = cpusers()
if not users:
error_and_exit(
self._is_json,
{"result": _("There are no users in the control panel.")},
)
user_list_to_process = users
else:
user_list_to_process = self._opts.users.split(",")
if os.path.exists(ADMIN_UPDATE_OBJECT_CACHE_BANNER_PID):
error_and_exit(
self._is_json,
{
'result': _('Updating visibility of Object Cache PRO banners is already in progress. '
'Process pid is stored in %(progress_marker)s.'
'If process with such pid does not exist - please, remove file "%(progress_marker)s" '
'and re-run command'),
'context': {'progress_marker': ADMIN_UPDATE_OBJECT_CACHE_BANNER_PID},
}
)
self.run_in_background(ADMIN_UPDATE_OBJECT_CACHE_BANNER_PID, self._update_object_cache_banner_for_users,
user_list_to_process, disable_object_cache_banners)
return {}
def _update_object_cache_banner_for_users(self, usernames, disable_object_cache_banners: bool):
if disable_object_cache_banners:
status = '--disable'
else:
status = '--enable'
for username in usernames:
command = ['/usr/bin/clwpos-user', '--user', username, 'object-cache-banner', '--all', status]
result = subprocess.run(command, text=True, capture_output=True, timeout=180)
self._logger.info(result.stdout)
self._logger.info(result.stderr)
def synchronize_plugins_status_for_user(username: str, uid: int,
old_state: Dict[str, FeatureStatus],
new_state: Dict[str, FeatureStatus]):
"""
Compare old and new states of modules in admin's wpos config,
determine what modules should be enabled and disabled
and synchronize new state for each panel's user.
"""
old_state = {key for key, value in old_state.items() if value.status == FeatureStatusEnum.ALLOWED}
new_state = {key for key, value in new_state.items() if value.status == FeatureStatusEnum.ALLOWED}
enabled_modules = new_state - old_state
disabled_modules = old_state - new_state
synchronize_plugins_for_user(username, uid, enabled_modules, disabled_modules)
def _extract_domain(all_domains, docroot):
for domain_info in all_domains:
if docroot == domain_info[1]:
return domain_info[0]
def synchronize_plugins_for_user(username: str, uid: int, enabled_modules: Set[str], disabled_modules: Set[str]):
"""
Iterate through user's docroots and wp_paths
and enable/disable modules with wp-cli
not modifying user's wpos config.
"""
user_domains = userdomains(username)
# TODO: [pain point] look all places (xray/ssa/accelerate-wp) which use drop_privileges + writing something
# create unified contextmanager for bypassing quota as well
# https://cloudlinux.atlassian.net/browse/CLPRO-864
with drop_privileges(username), disable_quota():
user_config = UserConfig(username=username)
for doc_root, wp_path, module in user_config.enabled_modules():
domain = _extract_domain(user_domains,
os.path.join(pwd.getpwnam(username).pw_dir, doc_root))
if module in enabled_modules:
enable_without_config_affecting(
DocRootPath(doc_root),
wp_path,
module=module,
domain=domain
)
if module in disabled_modules:
disable_without_config_affecting(
DocRootPath(doc_root),
wp_path,
module=module,
domain=domain
)
# we don't hardcode object cache here in case we will
# add some other modules that will require redis running
modules_that_require_redis = [
str(module) for module in ALL_OPTIMIZATION_FEATURES
if module.redis_daemon_required()
]
# reload redis only if we need
if not user_config.is_default_config() \
and set(modules_that_require_redis) & (enabled_modules | disabled_modules):
try:
# Reload redis for user
reload_redis(uid)
except WposError as e:
_logger.exception("CL AccelerateWP daemon error: '%s'; details: '%s'; context: '%s'", e.message, e.details, e.context)
except Exception as e:
_logger.exception(e)
def _enabled_modules(username: str) -> Iterator[Tuple[str, str, str]]:
return UserConfig(username=username).enabled_modules()
Zerion Mini Shell 1.0