| Dir : /opt/sharedrads/ |
| Server: Linux ngx353.inmotionhosting.com 4.18.0-553.22.1.lve.1.el8.x86_64 #1 SMP Tue Oct 8 15:52:54 UTC 2024 x86_64 IP: 209.182.202.254 |
| Dir : //opt/sharedrads/move_generator.py |
#!/opt/imh-python/bin/python3
"""Disk Move Generator - generates disk move tickets
according to arguments and exclusions"""
from operator import itemgetter
from platform import node
import datetime
import argparse
import sys
from pathlib import Path
from concurrent.futures import ThreadPoolExecutor, as_completed
from typing import Union
import arrow
import yaml
from tabulate import tabulate
import rads
EXCLUSION_LIST = Path('/var/log/disk_exclude')
TIMER = 30 # days before a user is removed from the exclusion list
def get_args():
"""Parse arguments"""
parser = argparse.ArgumentParser(description=__doc__)
group = parser.add_mutually_exclusive_group()
parser.add_argument(
'-a',
'--add',
type=str,
dest='add_user',
nargs='+',
default=[],
help="Add user to exclusion list. Entries survive 30 days",
)
parser.add_argument(
'-m',
'--min',
type=int,
default=8,
help="Minimum size of account to migrate in GB, default 8",
)
parser.add_argument(
'-x', '--max', type=int, help="Maximum size of account to migrate in GB"
)
parser.add_argument(
'-t',
'--total',
type=int,
help="Lists several eligible accounts whose size totals up to X GB",
)
group.add_argument(
'-e',
'--exclude',
type=str,
dest='exc_user',
nargs='+',
default=[],
help="List of users to exclude alongside exclusion list",
)
group.add_argument(
'-n',
'--noexclude',
action="store_true",
help="Do not use exclusion list",
)
parser.add_argument(
'-l',
'--listaccount',
action="store_true",
help="Print list of eligible accounts",
)
parser.add_argument(
'-d',
'--ticket',
action="store_true",
help="Email eligible accounts to the disk moves queue",
)
args = parser.parse_args()
# one of l, d, or a must be picked
if args.ticket is False and args.listaccount is False and not args.add_user:
print(
"--ticket (-d), --listaccount (-l),",
"or --add (-a) [user] is required",
)
sys.exit(1)
return args
def get_user_owners():
"""Parse /etc/trueuserowners"""
with open('/etc/trueuserowners', encoding='utf-8') as handle:
user_owners = yaml.load(handle, rads.DumbYamlLoader)
if user_owners is None:
return {}
return user_owners
def main():
args = get_args()
refresh_exclusion_list()
# If user to be added to exclusion list
if args.add_user:
# and list or email also selected
if args.listaccount or args.ticket:
print("Adding to exclusion list first...")
for user in args.add_user:
if not rads.is_cpuser(user):
print(f"{user} is not a valid cpanel user")
args.add_user.remove(user)
add_excluded_users(args.add_user)
if args.listaccount or args.ticket:
# collect a list of lists containing the eligible users
# and the total size
accounts = collect_accounts(
args.min, args.max, args.total, args.noexclude, args.exc_user
)
if args.listaccount:
list_accounts(accounts)
if args.ticket:
email_accounts(accounts)
return args
def get_exclusion_list() -> dict:
'''Read from the exclusion list and return it as a dict'''
data = {}
try:
with open(EXCLUSION_LIST, encoding='ascii') as exclusionlist:
data: dict = yaml.load(exclusionlist)
if not isinstance(data, dict):
print("Error in exclusion list, rebuilding")
data = {}
write_exclusion_list(data)
except (yaml.YAMLError, OSError) as exc:
print(type(exc).__name__, exc, sep=': ')
print('Recreating', EXCLUSION_LIST)
write_exclusion_list(data)
return data
def add_excluded_users(users: list[str]):
'''Format user information and timestamp for the exclusion list'''
for user in users:
exclusion_list = get_exclusion_list()
exclusion_list[user] = arrow.now().int_timestamp
write_exclusion_list(exclusion_list)
print(f"{user} added to exclusion list")
def write_exclusion_list(exclusion_list: dict[str, int]) -> None:
'''Write to the exclusion list'''
try:
with open(EXCLUSION_LIST, 'w', encoding='ascii') as outfile:
yaml.dump(exclusion_list, outfile, indent=4)
except Exception:
pass
def refresh_exclusion_list() -> None:
'''If a timeout has expired, remove the user'''
try:
timeouts = get_exclusion_list()
new_dict = {}
for user in timeouts:
if arrow.now().int_timestamp - timeouts[user] < int(
datetime.timedelta(days=TIMER).total_seconds()
):
new_dict[user] = timeouts[user]
write_exclusion_list(new_dict)
except Exception:
pass
def initial_disqualify(
user: str,
*,
min_size: int,
max_size: int,
noexclude: bool,
exclusion_list: list[str],
) -> tuple[str, Union[float, None]]:
'''Run the user through the first gamut to determine if
eligible for a move'''
try:
# knock out ineligible accounts
if rads.cpuser_safe(user):
return user, None
if Path('/var/cpanel/suspended', user).is_file():
return None
if not noexclude and user in exclusion_list:
return user, None
# get size
size_gb: float = rads.QuotaCtl().getquota(user) / 2**30
# check for eligibility based on size
if size_gb < min_size:
return user, None
if max_size and size_gb > max_size:
return user, None
# whatever's left after that, add to accounts list
return user, size_gb
except KeyboardInterrupt:
# drop child proc if killed
return user, None
def collect_accounts(
min_size: int,
max_size: int,
total_gb: int,
noexclude: bool,
exclude: list[str],
) -> list[tuple[str, str, float]]:
'''Get a list of users, and then eliminate them based on suspension
status, size, and eligibility based on options provided'''
# initializing everything
size_total = 0
accounts = []
eligible_accounts = []
final_list = []
# gather exclusion lists
try:
exclusion_list = list(get_exclusion_list().keys())
except Exception as exc:
print(f"Skipping exception file - {type(exc).__name__}: {exc}")
exclusion_list = []
exclusion_list += exclude
# create child processes to run through the eligibility checks
kwargs = dict(
min_size=min_size,
max_size=max_size,
noexclude=noexclude,
exclusion_list=exclusion_list,
)
accounts = []
user_owners = get_user_owners()
with ThreadPoolExecutor(max_workers=4) as pool:
try:
jobs = []
for user, owner in user_owners.items():
jobs.append(pool.submit(initial_disqualify, user, **kwargs))
for future in as_completed(jobs):
user, size_gb = future.result()
if size_gb is not None:
owner = user_owners[user]
accounts.append((user, owner, size_gb))
except KeyboardInterrupt:
print("Caught KeyboardInterrupt.")
pool.shutdown(wait=False, cancel_futures=True)
return []
if not accounts:
return final_list
# if anything survived those criteria...
accounts.sort(key=itemgetter(2), reverse=True) # sort by size, descending
# get a list of accounts of size > total
if total_gb:
size_total = 0
for account in accounts:
if len(eligible_accounts) < 3 or size_total < total_gb:
eligible_accounts.append(account)
size_total += account[2]
else:
break
accounts = eligible_accounts
final_list = accounts[:25]
return final_list
def list_accounts(accounts):
'''Print the list of eligible accounts in a pretty table'''
if not accounts:
print("No accounts match criteria.")
return
print(
tabulate(
reversed(accounts),
headers=["User", "Owner", "Size (GB)"],
floatfmt=".1f",
)
)
if total := sum(x[2] for x in accounts):
print(f"Total size of matching accounts: {total:.2f} GB")
def email_accounts(accounts: list[tuple[str, str, float]]):
'''Send an email for each user in accounts'''
server = node().split(".")[0]
exclude = []
for user, _, size_gb in accounts:
mail_disk_move(user, server, size_gb)
exclude.append(user)
add_excluded_users(exclude)
def mail_disk_move(username: str, server: str, size_gb: float):
'''Sends email to the disk moves queue'''
to_addr = "moves@imhadmin.net"
subject = f"DISK MOVE: {username} @ {server}"
body = f"""
A new server disk move is required for {username} @ {server}
Move Username: {username}
Account Size: {size_gb} GiB
Please review the account to determine if they are eligible for a migration:
* hasn't been moved recently (e.g. in the past year/no current move scheduled)
* is not storing ToS content
* is not large to the point of absurdity
* other reasons left to the discretion of the administrator
If the account is not eligible for a migration, please add them to the
exception list to prevent further tickets being generated for their account:
move_generator.py -a {username}
If the account is not eligible for reasons of ToS content, please respond
to this ticket with the relevant information and leave it open to address
further. For convenience, you may also update the subject line with the
date on which this should be addressed again and/or notice count.
"""
if rads.send_email(to_addr, subject, body):
print(f"Disk move tickets sent for {username}.")
else:
print(
"Sending of disk_move ticket failed. You can use the -l option to",
"view eligible accounts to move",
)
if __name__ == "__main__":
main()