Untitled

#Necessary imports
import string
import pandas as pd
import regex as re
import datetime
import datefinder
import sys
import subprocess
import linecache
from itertools import islice
import time
import gzip
import os
import itertools


# Declaring Files
agent_file='/Analytics/venv/Jup/CAPE_Apache_Beam/agent.properties'
bookmark_file='/Analytics/venv/Jup/CAPE_Apache_Beam/bookmark_log.properties'

#Program Execution
import time
start_time = time.time()


##Caching the VMList so that even if subsequent calls, it returns the cached VM list without needing to read the file again. 
vm_list_cache = None

def get_VMList():
    global vm_list_cache
    if vm_list_cache is None:
        try:
            with open(agent_file,'rt') as file:
                for line in file:
                    if 'VMList' in line:
                        vm_list = str(line.split('='))
                        vm_list = re.findall(r'\d+', vm_list)
                        vm_list_cache = list(map(int, vm_list))
                        break
        except FileNotFoundError:
            print(f"{agent_file} not found")                
        except Exception as e:
            print(f"An error occurred while reading the file: {str(e)}")
            vm_list_cache = []
    return vm_list_cache

print("The VM List is : ",get_VMList())


#Get number of lines to be copied
def get_log_block_size():
    try:
        with open(agent_file,'rt') as file:
            for line in file:
                if 'vmmetricskpi.log_block' in line:
                    print("Getting the Log Block size")
                    log_block_size=line.split('=')
                    #print(log_block_size,type(log_block_size))
                    no_of_lines=log_block_size[1]
                    break
    except Exception as e:
        print(f"An error occurred while reading the file: {str(e)}")                
                    
    return no_of_lines

print("Number of Lines : ",get_log_block_size())


# Checking Current Timestamp
print("Getting Current Timestamp")
current_time = time.strftime("%Y-%m-%d-%H-%M-%S")
print("Current Time: ",current_time)

def get_vm_details():
    VMList=get_VMList()
    All_VM_Details = []
    for vm in VMList:
        vm_id=vm
        try:
            VM_Details = {vm_id:{'vm_ip':[],'vm_output_log_filename':[],'vm_log_enable':[],'vm_log_filename':[],'bookmark':[],'bookmark_ip':[],'bookmark_log':[]}};
            with open(agent_file,'rt') as file:
                for line in file:
                    if str(vm) + ".privatip" in line:
                        vm_ip = str(line.split('='))
                        vm_ip = re.findall( r'[0-9]+(?:\.[0-9]+){3}', vm_ip)
                        vm_ip = list(map(str, vm_ip))
                        vm_ip = vm_ip[0]
                        VM_Details[vm_id]['vm_ip'].append(vm_ip)
                        bookmark_ip = vm_ip
                        VM_Details[vm_id]['bookmark_ip'].append(bookmark_ip)
                        bookmark_log=bookmark_ip + "_bookmark_log="
                        VM_Details[vm_id]['bookmark_log'].append(bookmark_log)

                    if str(vm)+".vmmetricskpi.output_log.filename" in line:
                        vm_output_log_filename =line.split('=')
                        vm_output_log_filename= list(map(str,vm_output_log_filename))
                        vm_output_log_filename = vm_output_log_filename[1].strip()
                        VM_Details[vm_id]['vm_output_log_filename'].append(vm_output_log_filename)

                    if str(vm)+".vmmetricskpi.log_enable" in line:
                        vm_log_enable =line.split('=')
                        vm_log_enable= list(map(str,vm_log_enable))
                        vm_log_enable = vm_log_enable[1].strip()
                        VM_Details[vm_id]['vm_log_enable'].append(vm_log_enable)

                    if str(vm)+".vmmetricskpi.log.filename" in line:
                        vm_log_filename =line.split('=')
                        vm_log_filename= list(map(str,vm_log_filename))
                        vm_log_filename = vm_log_filename[1].strip()
                        bookmark=vm_log_filename.split("-")
                        bookmark= list(map(str,bookmark))
                        bookmark = bookmark[1].strip()
                        VM_Details[vm_id]['vm_log_filename'].append(vm_log_filename)
                        VM_Details[vm_id]['bookmark'].append(bookmark)
        except FileNotFoundError:
            print(f"{agent_file} not found")
        except Exception as e:
            print(f"Failed to get details for VM {vm}: {e}")
            continue
                
        All_VM_Details.append(VM_Details)
        all_vm_details =dict()
        all_vm_details = zip(VMList, All_VM_Details)
        all_vm_details = dict(all_vm_details)
    return all_vm_details

print("VM Details : \n ",get_vm_details())  


# Checking log enable condition is Yes or No  
def check_log_enable():
    VMList = get_VMList()
    all_vm_details = get_vm_details() ## Moved Outside the Loop (change from Original Code)

    for vm_id in VMList:
        try:
            vm_log_enable = all_vm_details[vm_id][vm_id]['vm_log_enable'][0]
            if vm_log_enable.lower() == 'y':
                print(f"Log enable condition is yes. Let's proceed further to check Bookmark Function for VM {vm_id}")
                check_bookmark_record(vm_id)
            else:
                print(f"Log enable condition is no. No action needed for VM {vm_id}")
        except KeyError:
            print(f"No VM details found for VM {vm_id}")
            continue
            

# Checking if Bookmark Record is found in Runtime_log.properties
def check_bookmark_record(i):
    vm_id = i
    all_vm_details = get_vm_details()
    vm_ip = all_vm_details[vm_id][vm_id]['vm_ip'][0]  # Access List Element without Quotes
    print(vm_ip)
    line_to_search = f"{vm_ip}_bookmark_log="
    print("Bookmark Log : ", line_to_search)
    
    try:
        with open(bookmark_file, "r") as f:
            lines = f.readlines()
            filtered_lines = [line for line in lines if line_to_search in line]
            if filtered_lines:
                for bookmark_line in filtered_lines:
                    print(f"Bookmark Log Record found: {bookmark_line}")
                    print("Redirecting to perform_functions_with_bookmark_record")
                    perform_functions_with_bookmark_record(bookmark_line, i)
            else:
                bookmark_index = 1
                print("Line to search", line_to_search)
                bookmark_line = f"{line_to_search}{bookmark_index}"
                print("Bookmark_log : ", bookmark_line)
                print("No line found containing the specified string: ")
                print("Bookmark Log Record not found.Creating a record and redirecting to syslog File Location")
                with open(bookmark_file, 'a') as file:
                    file.write("\n")
                    file.write(bookmark_line)
                perform_functions_with_bookmark_record(bookmark_line, i)
    except FileNotFoundError:
        print(f"Bookmark file '{bookmark_file}' not found.")
        
    return None    
    

def perform_functions_with_bookmark_record(bookmark_line, i):
    try:
        print("The function with perform_functions_with_bookmark_record is getting called.")
        vm_id = i
        print("VM ID :", vm_id)
        bookmark_record = bookmark_line
        number_of_lines = get_log_block_size()
        print("No of Lines to copy : ", number_of_lines)
        print("Bookmark Record", bookmark_record)
        all_vm_details = get_vm_details()
        vm_ip = all_vm_details[vm_id][vm_id]['vm_ip'][0]
        vm_log_filename = all_vm_details[vm_id][vm_id]['vm_log_filename'][0]
        print("VM Log FIle : ", vm_log_filename)
        vm_output_directory_loc = all_vm_details[vm_id][vm_id]['vm_output_log_filename'][0]
        print("VM Output Directory : ", vm_output_directory_loc)
        bookmark_log = all_vm_details[vm_id][vm_id]['bookmark_log'][0]
        syslog_record = bookmark_record.split(str(vm_ip) + '_bookmark_log=')
        syslog_record_no = syslog_record[1]
        print("Syslog Record number for VM {} is {}".format(i, syslog_record_no))
        copying_loglines_and_updating_bookmark(bookmark_record, syslog_record_no, i, number_of_lines)
    except Exception as e:
        print("Error in perform_functions_with_bookmark_record function:", e)
        

def copying_loglines_and_updating_bookmark(bookmark_record, syslog_record_no, i, log_block_size):
    print("The function with searching_copying_logs is getting called.")
    vm_id = i
    print("VM ID :", vm_id)
    bookmark_line = bookmark_record
    number_of_lines = log_block_size
    print("No of Lines to copy : ", number_of_lines)
    print("Bookmark Record", bookmark_record)
    old_line_number = syslog_record_no
    print('Old line number from bookmark file is : ', old_line_number)
    last_line_no = int(old_line_number) + int(number_of_lines)
    print('Value of New Line Number is :', last_line_no)
    all_vm_details = get_vm_details()
    vm_ip = all_vm_details[vm_id][vm_id]['vm_ip'][0]
    vm_log_filename = all_vm_details[vm_id][vm_id]['vm_log_filename'][0]
    syslog_record = str(vm_ip) + "_bookmark_log="
    print(syslog_record)
    print("VM Log FIle : ", vm_log_filename)
    
    # Finding the line with index as line number
    try:
        last_line_output = linecache.getline(vm_log_filename, last_line_no)
        print("Content for nth line :", last_line_output)
    except Exception as e:
        print("Exception while reading line from file:", e)
        return None
    
    filename = f"{vm_ip}-T{current_time}.txt"
    print(filename)
    
    X = int(old_line_number)  # Starting line number
    #print(f"Old Value is {X}")
    Y = int(last_line_no)  # Ending line number
    print(f'New Value is {Y}')
    
    try:
        with open(vm_log_filename, 'r') as input_file, open(filename, 'w') as output_file:
            # Using itertools.islice to copy lines between start and end indexes
            output_file.writelines(itertools.islice(input_file, X - 1, Y))
    except Exception as e:
        print("Exception while copying lines to file:", e)
        return None
    
    # Updating the last index in a Bookmark File
    print("Updating the last index in a Bookmark File")
    bookmark_record = syslog_record
    print(f"Bookmark Record for {i} is {bookmark_record}")
    print(f"Updating Bookmark record for {bookmark_record}")
    updated_bookmark_rec = f"{bookmark_record}{last_line_no}\n"
    print(updated_bookmark_rec)
    
    try:
        with open(bookmark_file, 'r+') as file:
            lines = file.readlines()
            file.seek(0)
            for line_no, line in enumerate(lines):
                if bookmark_record in line:
                    lines[line_no] = updated_bookmark_rec
            file.seek(0)
            file.writelines(lines)
            file.truncate()
    except Exception as e:
        print("Exception while updating bookmark file:", e)
        return None

    # Call filter_log_data to filter the copied data
    filtered_filename = filter_log_data(filename)
    
    # Compress the output file and remove the uncompressed file
    output_file_name = output_file.name
    new_output_file_name = output_file_name.replace('.txt', '-cassandra.txt')
    os.rename(output_file_name, new_output_file_name)
    try:
        with open(new_output_file_name, 'rb') as input_file, gzip.open(new_output_file_name + '.gz', 'wb') as output_file_gz: 
            input_data = input_file.read()
            output_file_gz.write(input_data)
            os.remove(new_output_file_name)
            # Set permissions on the output file
            os.chmod(output_file_gz.name, 0o777)
            
    except Exception as e:
        print("Exception while compressing file:", e)
    
    # Return the filtered filename
    return filtered_filename


def filter_log_data(input_file):
    print(f"Filtering data in {input_file} file.")
    
    # Define the list of important words
    important_words = ['unauthorized', 'error','cron','kernel error', 'OS error', 'rejected', 'warning', "error", "fail", "exception", "critical",
            "security", "authentication", "intrusion", "attack", "status", "performance", "uptime", "load", "config", "setting", "permission", "firewall",
            "debug", "trace", "stack"]
    
    # Define the output filename
    output_file = f"{input_file.replace('.txt', '-postgre.txt')}"
    
    try:
        # Open the input and output files
        with open(input_file, 'r') as input_file, open(output_file, 'w') as output_file:
            # Loop through each line in the input file
            for line in input_file:
                # Check if any of the important words appear in the line
                if any(word in line.lower() for word in important_words):
                    # If yes, write the line to the output file
                    output_file.write(line)
                    
        # Set permissions on the output file
        os.chmod(output_file.name, 0o777)
        
        # Compress the output file and remove the uncompressed file
        with open(output_file.name, 'rb') as input_file, gzip.open(output_file.name + '.gz', 'wb') as output_file_gz:
            input_data = input_file.read()
            output_file_gz.write(input_data)
            
        os.remove(output_file.name)
        
        # Return the output filename
        return output_file.name + '.gz'
    
    except Exception as e:
        print(f"Error occurred: {str(e)}")
        return None
    
            
check_log_enable()    

## Execution Time    
end_time = time.time()
print("Execution time: ", end_time - start_time, "seconds")