| Dir : /opt/sharedrads/oldrads/ |
| Server: Linux ngx353.inmotionhosting.com 4.18.0-553.22.1.lve.1.el8.x86_64 #1 SMP Tue Oct 8 15:52:54 UTC 2024 x86_64 IP: 209.182.202.254 |
| Dir : //opt/sharedrads/oldrads/autosuspend.py |
#!/opt/imh-python/bin/python3
"""Automatic resource overage suspension script"""
import datetime
import socket
import sys
import re
import sh
import time
import yaml
import os
import configparser
import pp_api
import logging
import pwd
from collections import defaultdict
from multiprocessing import cpu_count
from functools import partial
from rads.shared import (
is_suspended,
is_cpanel_user,
get_secure_username,
SYS_USERS,
)
class Autosuspend:
"""
Gathers current and historic user cp usage data and determines
whether or not to enact an account suspension, send a warning
or pass over system users
"""
config_files = [
'/opt/sharedrads/etc/autosuspend.cfg',
'/opt/sharedrads/etc/autosuspend.cfg.local',
]
brand = ('imh', 'hub')['hub' in socket.gethostname()]
def __init__(self):
"""
Initializes an instance of Autosuspend, including a logging
object, parsed config from Autosuspend.config_files and
various information on system users
"""
self.config = configparser.ConfigParser(allow_no_value=False)
self.config.read(self.config_files)
logging.basicConfig(
level=logging.INFO,
format=f'%(asctime)s {sys.argv[0]}: %(message)s',
datefmt='%Y-%m-%d:%H:%M:%S %Z',
filename=self.suspension_log,
)
self.logger = logging.getLogger('suspension_logger')
self.priors = prior_events(
data_file=self.data_file,
log=self.suspension_log,
log_offset=self.suspension_log_offset,
)
self.suspensions_enabled = self.config.getboolean(
'suspensions',
'enabled',
)
self.warnings_enabled = self.config.getboolean(
'warnings',
'enabled',
)
self.freepass_enabled = self.config.getboolean(
'freepass',
'enabled',
)
self.actions = {
'suspension': enact_suspension,
'warning': send_warning,
'freepass': give_free_pass,
}
self.server_overloaded = server_overloaded()
_users = top_users(
interval_file=self.sa_interval_file,
max_age=self.sa_interval_file_max_age,
)
self.users = {
name: User(
name=name,
delta=delta,
suspensions=self.priors.get(name, {}).get('suspensions', []),
warnings=self.priors.get(name, {}).get('warnings', []),
freepasses=self.priors.get(name, {}).get('freepasses', []),
)
for name, delta in _users
if not is_suspended(name)
}
def __repr__(self):
"""
Returns a representation of an Autosuspend object
"""
repr_data = [
'brand',
'disruptive_action_interval',
'server_load_factor',
'server_overloaded',
'suspensions_enabled',
'warnings_enabled',
'freepass_enabled',
]
repr_str = '<Autosuspend {}>'.format(
' '.join([f'{i}:{getattr(self, i)}' for i in repr_data])
)
return repr_str
def suspension_critera_met(self, user):
"""
Tests a User object to see if it meets suspension criteria
"""
if not user.warned_within(self.disruptive_action_interval):
self.logger.debug(
f'{user.name} not warned within {self.disruptive_action_interval}, not suspending'
)
return False
# double check this logic - if user was suspended longer ago than action_interval they should be elligible
if not user.suspended_longer_ago(self.disruptive_action_interval):
self.logger.debug(
f'{user.name} not suspended within {self.disruptive_action_interval}, not suspending'
)
return False
if user.num_warnings <= self.warning_count:
self.logger.debug(
f'Not suspended; only {user.num_warnings} warnings, need {self.warning_count}'
)
return False
if float(user.delta) >= float(self.suspensions['max_delta']):
return True
return False
def warning_critera_met(self, user):
"""
Tests a User object to see if it meets warning criteria
"""
if user.warned_within(self.disruptive_action_interval):
self.logger.debug(
f'{user.name} warned within {self.disruptive_action_interval}, not warning'
)
return False
if float(user.delta) >= float(self.warnings['max_delta']):
return True
else:
self.logger.debug(
'{} has not consumed more than {}cp in the last {}'.format(
user.name,
self.warnings['max_delta'],
self.disruptive_action_interval,
)
)
return False
def freepass_criteria_met(self, user):
"""
Tests a user to see if it meets freepass criteria
"""
self.logger.debug(f'Testing {user.name} for freepass...')
if float(user.delta) >= float(self.freepass['max_delta']):
self.logger.debug(
f'{user.name} has a delta of {user.delta}, which is above the threshold.'
)
if not user.given_freepass_within(self.time_between_free_passes):
self.logger.debug(
f'{user.name} was not given a free pass within {self.time_between_free_passes} so they get one'
)
return True
else:
self.logger.debug(
f'{user.name} got a free pass within the last {self.time_between_free_passes} days, not sending another'
)
return False
def run(self):
"""
Loops through Autosuspend.users, calling Autosuspend.test for each
"""
self.logger.info(f'Autosuspend run starting {repr(self)}')
if not self.users:
return
for user in self.users.values():
action = self.test(user)
action_func = self.actions.get(
action,
lambda *x, **y: None,
)
wrapper = partial(
action_func, email_template=getattr(self, f'{action}_template')
)
wrapper(user=user.name, comment=user.note)
self.logger.info('Autosuspend run complete')
def test(self, user):
"""
Determines what action, if any to take against an individual
User object
"""
user.suspend = self.suspension_critera_met(user)
user.warn = self.warning_critera_met(user)
user.freepass = self.freepass_criteria_met(user)
if user.suspend and self.suspensions_enabled and self.server_overloaded:
user.note = (
'AUTO SUSPENSION: Consumed {:.2f}cp within '
'the last measured interval.'.format(user.delta)
)
self.logger.debug(f'Suspending {user}')
self.logger.info(
f'{user.delta} [AUTO_SUSPENSION] ra - root "{user.note}"'
)
return 'suspension'
elif user.freepass and self.freepass_enabled:
user.note = (
'AUTO RA FREEPASS: Consumed {:.2f}cp within '
'the last measured interval.'.format(user.delta)
)
self.logger.debug(f'Freepassing {user}')
self.logger.info(f'{user.name} [FREEPASS] ra - root "{user.note}"')
return 'freepass'
elif user.warn and self.warnings_enabled:
user.note = (
'AUTO RA WARNING: Consumed {:.2f}cp within '
'the last measured interval.'.format(user.delta)
)
self.logger.debug(f'Warning {user}')
self.logger.info(f'{user.name} [WARNING] ra - root "{user.note}"')
return 'warning'
else:
self.logger.debug(f'Skipping {user}')
return 'skip'
def __getattr__(self, item):
"""
Returns items as strings from settings and brand-specific settings
sections or entire config sections as a dict
e.g. <Autosuspend instance>.suspension_log;
<Autosuspend instance>.settings['suspension_log']
"""
if item in self.config.sections():
return dict(self.config.items(item))
# See if a given key is present in the settings section
for section in (f'settings_{self.brand}', 'settings'):
if self.config.has_option(section, item):
return self.config.get(section, item)
class User:
"""
Instantiated to represent a system user.
"""
def __init__(self, **args):
"""
Initializes the User object
"""
self.data_dict = args
self.warn = False
self.suspend = False
self.freepass = False
self.num_suspensions = len(args['suspensions'])
self.num_warnings = len(args['warnings'])
self.num_freepasses = len(args['freepasses'])
def __getattr__(self, item):
"""
Returns an item from self.data_dict or None in the event of a KeyError
"""
try:
return self.data_dict[item]
except KeyError:
pass
def __repr__(self):
"""
Returns a useful representation of a User object
"""
repr_data = [
'name',
'delta',
'last_suspension',
'last_warning',
'last_freepass',
'num_suspensions',
'num_warnings',
'num_freepasses',
'suspend',
'warn',
'freepass',
'note',
]
repr_str = '<User {}>'.format(
' '.join([f'{i}:{getattr(self, i)}' for i in repr_data])
)
return repr_str
def warned_within(self, delta):
"""
Returns True if the user's last warning was sent within the current
time - delta, False otherwise
"""
if not isinstance(delta, datetime.timedelta):
delta = str_to_timedelta(delta)
try:
return datetime.datetime.now() < (self.last_warning + delta)
except TypeError:
return False
def suspended_longer_ago(self, delta):
"""
Returns True if the user's last suspension was longer ago
than the current time - delta, False otherwise
"""
if not isinstance(delta, datetime.timedelta):
delta = str_to_timedelta(delta)
try:
return datetime.datetime.now() > (self.last_suspension + delta)
except TypeError:
return True
def given_freepass_within(self, delta):
"""
In the case self.last_freepass is None, we return false.
"""
if not self.last_freepass:
return False
if not isinstance(delta, datetime.timedelta):
delta = str_to_timedelta(delta)
try:
return datetime.datetime.now() < (self.last_freepass + delta)
except TypeError:
return True
@property
def last_suspension(self):
"""
returns a datetime object which represents the last time
the user was suspended or None
"""
return self._last_suspension
@last_suspension.getter
def last_suspension(self):
"""
returns a datetime object which represents the last time
the user was suspended or None
"""
return self._nth_date('suspensions', -1)
@property
def last_warning(self):
"""
returns a datetime object which represents the last time
the user was warned or None
"""
return self._last_warning
@last_warning.getter
def last_warning(self):
"""
returns a datetime object which represents the last time
the user was warned or None
"""
return self._nth_date('warnings', -1)
@property
def last_freepass(self):
return self._last_freepass
@last_freepass.getter
def last_freepass(self):
return self._nth_date('freepasses', -1)
def _nth_date(self, attr, index):
"""
Return a datetime object representation of a date from
suspension or warning lists
"""
items = getattr(self, attr)
try:
return datetime.datetime.fromtimestamp(
sorted(map(float, items))[index]
)
except (TypeError, IndexError):
pass
def str_to_timedelta(time_str):
"""
Munges strings into timedelta objects
"""
match = re.search(
r"""(:?
(:?(?P<hours>\d+)[Hh])?
(:?(?P<minutes>\d+)[Mm])?
(:?(?P<days>\d+)[Dd])?
(:?(?P<seconds>\d+)[Ss])?
)+""",
''.join(time_str.split()),
re.VERBOSE,
)
groups = {k: float(v) for k, v in match.groupdict().items() if v}
return datetime.timedelta(**groups)
def server_overloaded(factor=1.5):
"""
Determines whether or not the sever is unduly stressed by comparing the
15-minute load average and the product of number of cores and 'factor'.
"""
return (cpu_count() * factor) <= os.getloadavg()[-1]
def try_open_yaml(yaml_path):
"""Try to read a yaml file. If impossible, return an empty dict"""
try:
data = yaml.load(file(yaml_path, 'r'))
except (OSError, yaml.error.MarkedYAMLError):
return {}
if not isinstance(data, dict):
return {}
return data
def get_log_data(logfile, offsetfile, ignore_offset=False):
"""
Reads and offset from the offset file, returns data from the offset to
the end of the file
"""
# try to read the offset from the offset file
try:
with open(offsetfile) as offset_fh:
offset = int(offset_fh.readline())
# Set offset to 0 if the offset can't be converted to an integer or the
# file is missing
except (OSError, ValueError):
offset = 0
if ignore_offset:
offset = 0
try:
with open(logfile) as logfile_fh:
logfile_fh.seek(0, 2)
logfile_length = logfile_fh.tell()
if offset > logfile_length:
logfile_fh.seek(0)
else:
logfile_fh.seek(offset)
output = logfile_fh.readlines()
newoffset = logfile_fh.tell()
# If the file can't be opened return an empty string
# and set newoffset to 0
except OSError:
output = ""
newoffset = 0
# Write the new offset to the offset file
with open(offsetfile, 'w') as offset_fh:
offset_fh.write(str(newoffset))
return output
def prior_events(log=None, log_offset=None, data_file=None):
'''Returns a dict that contains account suspension times'''
suspension_re = re.compile(
r"""(?P<time>\d{4}-\d{2}-\d{2}:\d{2}:\d{2}:\d{2})
\s\w+\s+[\w/\.-]+:\s+(?P<user>\w+)\s+\[
(:?
(?P<suspensions>(:?AUTO_)?SUSPENSION)|
(?P<warnings>WARNING)|
(?P<freepasses>FREEPASS)
)
\]\s+ra""",
re.VERBOSE,
)
priors = defaultdict(
lambda: {'suspensions': [], 'warnings': [], 'freepasses': []}
)
# Get prior suspensions
if data_file is not None:
priors.update(try_open_yaml(data_file))
# If past_suspensions wasn't populated with any data, skip using the offset
# when reading log data
if len(priors) < 1:
skip_offset = True
else:
skip_offset = False
# Lolast_freepassop through new lines from the suspension log and add to the suspension
# data dict
for line in get_log_data(log, log_offset, skip_offset):
match = suspension_re.search(line)
if match:
user = match.group('user')
event_time = int(
time.mktime(
time.strptime(match.group('time'), "%Y-%m-%d:%H:%M:%S")
)
)
for event in ('suspensions', 'warnings', 'freepasses'):
if match.group(event):
try:
priors[user][event].append(event_time)
except (KeyError, AttributeError):
priors[user][event] = [event_time]
yaml.dump(dict(list(priors.items())), file(data_file, 'w+'))
return priors
def prepare_str_content(user):
"""Runs various commands, returns string output"""
output = []
rads_cmds = (
('/opt/sharedrads/check_user', dict(plaintext=True)),
(
'/opt/sharedrads/nlp',
dict(w=80, p=True, _err_to_out=True, today=True),
),
('/opt/sharedrads/recent-cp', dict(b=True)),
)
for script, kwargs in rads_cmds:
# these RADS scripts all accept a user name as the first
# positional argument so it can just be baked in
try:
cmd = sh.Command(script).bake(user)
except sh.CommandNotFound:
continue
try:
result = cmd(**kwargs)
except sh.ErrorReturnCode:
continue
output.append(
(
f'>>> {result.ran}',
result.stdout,
)
)
output.append(
(
'>>> Running processes prior to suspension',
sh.awk(sh.ps('auwwx', _piped=True), "$1 ~ /%s|USER/" % user).stdout,
)
)
output = "\n\n".join(['\n'.join(items) for items in output])
return output
def save_str(user, content):
"""
Saves STR data in the user's '.imh' folder
"""
imhdir = os.path.join(pwd.getpwnam(user).pw_dir, '.imh')
if not os.path.isdir(imhdir):
try:
os.mkdir(imhdir)
except OSError:
return
date_string = datetime.datetime.strftime(
datetime.datetime.now(), '%Y-%m-%d_%H:%M:%S'
)
strfile = os.path.join(imhdir, f'str_{date_string}')
with open(strfile, 'w') as str_fh:
str_fh.write(content)
def send_str(user, content):
"""Sends an STR to the STR queue"""
strmailer_cmd = sh.Command('/opt/sharedrads/mailers/strmailer-suspend.py')
strmailer_cmd(user, socket.gethostname().split('.')[0], _in=content)
def enact_suspension(user, comment, email_template):
"""Suspends the user, notes the account and sends an STR"""
ppa = pp_api.PowerPanel()
ppa.call('notification.send', template=email_template, cpanelUser=user)
ppa.call(
'hosting.insert-note',
admin_user='auto',
user=user,
note='AUTO SUSPENSION: %s' % comment,
flagged=True,
type='Suspension',
)
message = 'RADS suspended {} for resource abuse.\n\n{}\n\n' '{}'.format(
user, comment, prepare_str_content(user)
)
send_str(user, message)
save_str(user, message)
suspend_cmd = sh.Command('/opt/sharedrads/suspend_user')
suspend_cmd(
user, invoked_by=sys.argv[0], ra=True, c=comment, _tty_out=False
)
def send_warning(user, comment, email_template):
"""Sends a warning to the user, notes the customer's account"""
ppa = pp_api.PowerPanel()
cmd = sh.Command(
'/opt/sharedrads/python/send_customer_str/send_customer_str'
)
run = cmd('-u', user)
if run.exit_code != 0:
send_str(
user, "Could not generate STR for customer with send_customer_str"
)
ppa.call('notification.send', template=email_template, cpanelUser=user)
ppa.call(
'hosting.insert-note',
admin_user='auto',
user=user,
note='AUTO RA NOTICE: %s' % comment,
flagged=True,
type='Resource Management',
)
save_str(user, prepare_str_content(user))
def give_free_pass(user, comment, email_template):
save_str(user, prepare_str_content(user))
def top_users(interval_file, max_age, num_users=None):
"""
Return sorted top users in the last time interval. If num_users specified,
only return that number of users. Otherwise, return all of them. This
function also handles saving the current sa data for the next run
"""
this_sa = get_sa_dict()
user_deltas = []
if check_interval_file_age(interval_file, max_age):
last_sa = try_open_yaml(interval_file)
for user in this_sa.keys():
try:
delta = float(this_sa[user]) - float(last_sa[user])
except (KeyError, ValueError):
continue
user_deltas.append((user, delta))
# sort by usage, descending
user_deltas.sort(key=lambda tup: tup[1], reverse=True)
# save this run's sa data for the next run
with open(interval_file, 'w') as data_file:
yaml.dump(
this_sa,
data_file,
)
return user_deltas[:num_users]
def get_sa_dict():
"""get info from 'sa -m' in the form of a dict"""
sec_usr = get_secure_username()
skip_user_re = re.compile(
r'(?:{}{})\b'.format(
'' if sec_usr is None else '%s|' % sec_usr, '|'.join(SYS_USERS)
)
)
sa_re = re.compile(
r'^(?P<user>\w+) +[0-9]+ +[0-9]+\.[0-9]{2}'
r're +(?P<cp>[0-9]+\.[0-9]{2})cp '
)
sa_output = [
x for x in sh.sa(m=True).splitlines() if not skip_user_re.match(x)
]
sa_dict = {}
for line in sa_output:
match = sa_re.search(line)
if match is not None:
user, cp_total = match.group('user'), match.group('cp')
if not is_cpanel_user(user):
continue
sa_dict[user] = float(cp_total)
return sa_dict
def check_interval_file_age(interval_file_path, max_age):
"""
Check how old the sa_interval_file is. Return true if the age is what
is expected from autosuspend.cfg. If false, the file's contents cannot
be used to decide whether users should be suspended
"""
if not isinstance(max_age, datetime.timedelta):
max_age = str_to_timedelta(max_age)
try:
mtime = datetime.datetime.fromtimestamp(
os.path.getmtime(interval_file_path)
)
except OSError: # file not found
return False
return (datetime.datetime.now() - mtime) < max_age
if __name__ == '__main__':
Autosuspend().run()