| Dir : /opt/sharedrads/oldrads/ |
| Server: Linux ngx353.inmotionhosting.com 4.18.0-553.22.1.lve.1.el8.x86_64 #1 SMP Tue Oct 8 15:52:54 UTC 2024 x86_64 IP: 209.182.202.254 |
| Dir : //opt/sharedrads/oldrads/scrape_fails.py |
#!/opt/imh-python/bin/python3
from collections import Counter
import pika
import platform
import re
from pathlib import Path
FQDN = platform.node()
def scrape_http():
# 28/Aug/2017:13:07:21
"""Check for modsec hits"""
hits_list = []
log_file = Path('/usr/local/apache/logs/modsec_audit.log')
line_number = 0
inode = log_file.stat().st_ino
try:
prev_inode = get_previous('modsec')[0]
prev_line = get_previous('modsec')[1]
except Exception:
# Files dont exist, or were removed. Assume starting from scratch
prev_inode = int(0)
prev_line = int(0)
if int(inode) != int(prev_inode):
# Inode changed. Logfile was most likely rotated. Start over
with open(log_file, encoding='ascii') as file:
for line in file.readlines():
line_number += 1 # Line number counter
try:
if line.split()[-1] in ('80', '443'):
remote_ip = line.split()[3]
hits_list.append(remote_ip)
except Exception:
pass
else:
with open(log_file, encoding='ascii') as file:
line_number = int(prev_line)
for _ in range(int(prev_line)):
next(file) # Skip ahead past the lines we already did
for line in file:
line_number += 1 # Line number counter
try:
if line.split()[-1] in ('80', '443'):
remote_ip = line.split()[3]
hits_list.append(remote_ip)
except Exception:
pass
# update the cache file so we know where to pick up next time
update_marker('modsec', inode, line_number)
return hits_list
def scrape_ftp():
"""Check /var/log/messages for failed FTP logins"""
hits_list = []
log_file = Path('/var/log/messages')
line_number = 0
inode = log_file.stat().st_ino
try:
prev_inode = get_previous('ftp')[0]
prev_line = get_previous('ftp')[1]
except Exception: # No cache files, start over
prev_inode = int(0)
prev_line = int(0)
if int(inode) != int(prev_inode): # log rotated, start from beginning
with open(log_file, encoding='ascii') as file:
for line in file.readlines():
line_number += 1 # Increase line counter
if 'pure-ftpd' in line and 'Authentication failed' in line:
ip = line.split()[5].split("@")[1][:-1]
hits_list.append(ip)
else:
with open(log_file, encoding='ascii') as file:
line_number = int(prev_line)
for _ in range(int(prev_line)):
next(file) # Skip past lines we already parsed
for line in file:
line_number += 1 # Increase line counter
if 'pure-ftpd' in line and 'Authentication failed' in line:
ip = line.split()[5].split("@")[1][:-1]
hits_list.append(ip)
# update the cache file so we know where to pick up next time
update_marker('ftp', inode, line_number)
return hits_list
def scrape_mysql():
"""check /var/lib/mysql/$server.err for failed logins"""
hits_list = []
log_file = Path(f'/var/lib/mysql/{FQDN}.err')
line_number = 0
inode = log_file.stat().st_ino
try:
prev_inode = get_previous('mysql')[0]
prev_line = get_previous('mysql')[1]
except Exception: # No cache files, start over
prev_inode = int(0)
prev_line = int(0)
if int(inode) != int(prev_inode): # log rotated, start from beginning
with open(log_file, encoding='ascii') as file:
for line in file.readlines():
line_number += 1 # increase line counter
if "Access denied" in line:
ip = re.findall(r'[0-9]+(?:\.[0-9]+){3}', line)
hits_list = hits_list + ip
print(len(hits_list))
else:
with open(log_file, encoding='ascii') as file:
line_number = int(prev_line)
for _ in range(int(prev_line)):
next(file) # skip through
for line in file:
line_number += 1 # increase line counter
if 'Access denied' in line:
ip = re.findall(r'[0-9]+(?:\.[0-9]+){3}', line)
hits_list = hits_list + ip
update_marker('mysql', inode, line_number) # update the cache file
return hits_list
def update_marker(file_name, inode, line_number):
"""Update the file in /var/cache with the inode and line number"""
try:
store_folder = Path('/var/cache/iptrack')
store_folder.mkdir(mode=0o755, parents=True, exist_ok=True)
store_file = store_folder / file_name
with open(store_file, 'w+', encoding='ascii') as file:
file.write(str(inode) + ':' + str(line_number))
file.close()
except Exception as e: # If this fails, something is seriously wrong
print(e)
def get_previous(log_name):
"""Get the previous line number we were on"""
with Path('/var/cache/iptrack', log_name).open(encoding='ascii') as file:
for line in file:
inode = line.split(":")[0]
line_no = line.split(":")[1]
return [inode, line_no]
def send_message(channel, service, server, ip, hits):
"""Send a message to rabbitmq"""
channel.basic_publish(
exchange='',
routing_key='iptrack',
properties=pika.BasicProperties(
headers=({'service': service, 'server': server, 'hits': hits})
),
body=ip,
)
def main():
"""Scrape the log files, get some ips, and sent it to rabbitmq"""
with open('/opt/sharedrads/etc/rabbit_pass', encoding='ascii') as file:
for line in file.readlines():
rabbit_pass = line.strip()
host = 'ash-sys-pro-iptrack1.imhadmin.net'
credentials = pika.PlainCredentials('iptrack', rabbit_pass)
connection = pika.BlockingConnection(
pika.ConnectionParameters(host, credentials=credentials)
)
channel = connection.channel()
channel.queue_declare(queue='iptrack')
try:
for key, val in Counter(scrape_http()).items():
send_message(channel, 'http', FQDN, key, val)
for key, val in Counter(scrape_ftp()).items():
send_message(channel, 'ftp', FQDN, key, val)
for key, val in Counter(scrape_mysql()).items():
send_message(channel, 'mysql', FQDN, key, val)
except Exception as e:
print(e)
connection.close()
if __name__ == '__main__':
main()