#Necessary imports
import string
import pandas as pd
import regex as re
import datetime
import datefinder
import sys
import subprocess
import linecache
from itertools import islice
import time
import gzip
import os
# Declaring Files
agent_file='/Analytics/venv/Jup/CAPE_Apache_Beam/agent.properties'
bookmark_file='/Analytics/venv/Jup/CAPE_Apache_Beam/bookmark_log.properties'
# Get VM List from agent.properties file
def get_VMList():
with open(agent_file,'rt') as file:
for line in file:
if 'VMList' in line:
vm_list = str(line.split('='))
vm_list = re.findall(r'\d+', vm_list)
VMList = list(map(int, vm_list))
return VMList
print("The VM List is : ",get_VMList())
VMList = get_VMList()
#Get number of lines to be copied
def get_log_block_size():
with open(agent_file,'rt') as file:
for line in file:
if 'vmmetricskpi.log_block' in line:
print("Getting the Log Block size")
log_block_size=line.split('=')
#print(log_block_size,type(log_block_size))
no_of_lines=log_block_size[1]
return no_of_lines
print("Number of Lines : ",get_log_block_size())
print("Getting Current Timestamp")
# Checking Current Timestamp
current_time = time.strftime("%Y-%m-%d-%H-%M-%S")
print(current_time)
# Getting All VM Details
def get_vm_details():
All_VM_Details = []
for vm in VMList:
vm_id=vm
with open(agent_file,'rt') as file:
for line in file:
if str(vm) + ".privatip" in line:
VM_Details = {vm_id:{'vm_ip':[],'vm_output_log_filename':[],'vm_log_enable':[],'vm_log_filename':[],'bookmark':[],'bookmark_ip':[],'bookmark_log':[]}};
vm_ip = str(line.split('='))
vm_ip = re.findall( r'[0-9]+(?:\.[0-9]+){3}', vm_ip)
vm_ip = list(map(str, vm_ip))
vm_ip = vm_ip[0]
VM_Details[vm_id]['vm_ip'].append(vm_ip)
bookmark_ip = vm_ip
VM_Details[vm_id]['bookmark_ip'].append(bookmark_ip)
bookmark_log=bookmark_ip + "_bookmark_log="
VM_Details[vm_id]['bookmark_log'].append(bookmark_log)
if str(vm)+".vmmetricskpi.output_log.filename" in line:
vm_output_log_filename =line.split('=')
vm_output_log_filename= list(map(str,vm_output_log_filename))
vm_output_log_filename = vm_output_log_filename[1].strip()
VM_Details[vm_id]['vm_output_log_filename'].append(vm_output_log_filename)
if str(vm)+".vmmetricskpi.log_enable" in line:
vm_log_enable =line.split('=')
vm_log_enable= list(map(str,vm_log_enable))
vm_log_enable = vm_log_enable[1].strip()
VM_Details[vm_id]['vm_log_enable'].append(vm_log_enable)
if str(vm)+".vmmetricskpi.log.filename" in line:
vm_log_filename =line.split('=')
vm_log_filename= list(map(str,vm_log_filename))
vm_log_filename = vm_log_filename[1].strip()
bookmark=vm_log_filename.split("-")
bookmark= list(map(str,bookmark))
bookmark = bookmark[1].strip()
VM_Details[vm_id]['vm_log_filename'].append(vm_log_filename)
VM_Details[vm_id]['bookmark'].append(bookmark)
All_VM_Details.append(VM_Details)
all_vm_details =dict()
all_vm_details = zip(VMList, All_VM_Details)
all_vm_details = dict(all_vm_details)
return all_vm_details
print("VM Details : \n ",get_vm_details())
def filter_log_data(input_file):
print(f"Filtering data in {input_file}")
# Define the list of important words
important_words = ['unauthorized', 'error', 'kernel error', 'OS error', 'rejected', 'warning', "error", "fail", "exception", "critical",
"security", "authentication", "intrusion", "attack", "status", "performance", "uptime", "load", "config", "setting", "permission", "firewall",
"debug", "trace", "stack"]
# Define the output filename
output_file = f"{input_file.replace('.txt', '-filtered.txt')}"
# Open the input and output files
with open(input_file, 'r') as input_file, open(output_file, 'w') as output_file:
# Loop through each line in the input file
for line in input_file:
# Check if any of the important words appear in the line
if any(word in line.lower() for word in important_words):
# If yes, write the line to the output file
output_file.write(line)
# Set permissions on the output file
os.chmod(output_file.name, 0o777)
# Compress the output file and remove the uncompressed file
with open(output_file.name, 'rb') as input_file, gzip.open(output_file.name + '.gz', 'wb') as output_file_gz:
input_data = input_file.read()
output_file_gz.write(input_data)
os.remove(output_file.name)
#os.remove(input_file)
# Return the output filename
return output_file.name + '.gz'
def copying_loglines_and_updating_bookmark(bookmark_record, syslog_record_no, i, log_block_size):
print("The function with searching_copying_logs is getting called.")
vm_id = i
print("VM ID :", vm_id)
bookmark_line = bookmark_record
number_of_lines = log_block_size
print("No of Lines to copy : ", number_of_lines)
print("Bookmark Record", bookmark_record)
old_line_number = syslog_record_no
print('Old line number : ', old_line_number)
last_line_no = int(old_line_number) + int(number_of_lines)
print('New Line Number :', last_line_no)
all_vm_details = get_vm_details()
vm_ip = all_vm_details[vm_id][vm_id]['vm_ip'][0]
vm_log_filename = all_vm_details[vm_id][vm_id]['vm_log_filename'][0]
syslog_record = str(vm_ip) + "_bookmark_log="
print(syslog_record)
print("VM Log FIle : ", vm_log_filename)
# Finding the line with index as line number
last_line_output = linecache.getline(vm_log_filename, last_line_no)
print("Content for nth line :", last_line_output)
filename = f"{vm_ip}-T{current_time}.txt"
print(filename)
import itertools
X = int(old_line_number) # Starting line number
print(f"Old Value is {X}")
Y = int(last_line_no) # Ending line number
print(f'New Value is {Y}')
with open('/Analytics/venv/Jup/CAPE_Apache_Beam/Data/LONSTBYRDEV02-10.72.209.183.txt', 'r') as input_file, open(filename, 'w') as output_file:
output_file.writelines(itertools.islice(input_file, X - 1, Y))
# Updating the last index in a Bookmark File
print("Updating the last index in a Bookmark File")
bookmark_record = syslog_record
print(f"Bookmark Record for {i} is {bookmark_record}")
print(f"Updating Bookmark record for {bookmark_record}")
updated_bookmark_rec = f"{bookmark_record}{last_line_no}\n"
print(updated_bookmark_rec)
with open(bookmark_file, 'r+') as file:
lines = file.readlines()
file.seek(0)
for line_no, line in enumerate(lines):
if bookmark_record in line:
lines[line_no] = updated_bookmark_rec
file.seek(0)
file.writelines(lines)
file.truncate()
# Call filter_log_data to filter the copied data
filtered_filename = filter_log_data(filename)
# Remove the uncompressed file
os.remove(filename)
# Return the filtered filename
return filtered_filename
#Performing Operations if Bookmark Record is found
def perform_functions_with_bookmark_record(bookmark_line,i):
print("The function with perform_functions_with_bookmark_record is getting called.")
vm_id=i
print("VM ID :",vm_id)
bookmark_record=bookmark_line
number_of_lines=get_log_block_size()
print("No of Lines to copy : ",number_of_lines)
print("Bookmark Record",bookmark_record)
all_vm_details = get_vm_details()
vm_ip = all_vm_details[vm_id][vm_id]['vm_ip'][0]
vm_log_filename=all_vm_details[vm_id][vm_id]['vm_log_filename'][0]
print("VM Log FIle : ",vm_log_filename)
vm_output_directory_loc=all_vm_details[vm_id][vm_id]['vm_output_log_filename'][0]
print("VM Output Directory : ",vm_output_directory_loc)
bookmark_log=all_vm_details[vm_id][vm_id]['bookmark_log'][0]
syslog_record = bookmark_record.split(str(vm_ip)+'_bookmark_log=')
syslog_record_no=syslog_record[1]
print("Syslog Record number for VM {} is {}" .format(i,syslog_record_no))
copying_loglines_and_updating_bookmark(bookmark_record,syslog_record_no,i,number_of_lines)
# Checking if Bookmark Record is found in RUntime_log.properties
def check_bookmark_record(i):
vm_id=i
all_vm_details = get_vm_details()
vm_ip = all_vm_details[vm_id][vm_id]['vm_ip'][0] ##Access List Element without Quotes
print(vm_ip)
line_to_search = str(vm_ip)+"_bookmark_log="
print("Bookmark Log : ",line_to_search)
with open(bookmark_file, "r") as f:
lines = f.readlines()
filtered_lines = filter(lambda x: line_to_search in x, lines)
filtered_lines = [line for line in filtered_lines]
if filtered_lines:
for bookmark_line in filtered_lines:
print(f"Bookmark Log Record found: {bookmark_line}")
print("Redirecting to perform_functions_with_bookmark_record")
perform_functions_with_bookmark_record(bookmark_line,i)
else:
## Making changes here
bookmark_index=1
print("Line to search",line_to_search)
bookmark_line =str(line_to_search) + str(bookmark_index)
print("Bookmark_log : ", bookmark_line)
print("No line found containing the specified string: ")
print("Bookmark Log Record not found.Creating a record and redirecting to syslog File Location")
with open(bookmark_file, 'a') as file:
file.write("\n")
file.write(bookmark_line)
perform_functions_with_bookmark_record(bookmark_line,i)
return None
#Checking log enable condition is Yes or No
def check_log_enable():
VMList = get_VMList()
#print(VMList)
#print("VM Details : ",get_vm_details())
for i in VMList:
vm_id=i
all_vm_details = get_vm_details()
if all_vm_details[vm_id][vm_id]['vm_log_enable']==['y']:
print(f"Log enable condition is yes.Lets proceed further to check Bookmark Function",vm_id)
check_bookmark_record(vm_id) ## Call Function
else:
print(f"Log enable condition is no. No action Needed." ,vm_id)
continue
check_log_enable()