Untitled
unknown
plain_text
7 months ago
12 kB
1
Indexable
Never
Getting Current Timestamp 2023-04-28-08-06-53 WARNING:root:Make sure that locally built Python SDK docker image has Python 3.6 interpreter. /var/log/EventLog/Processed_logs_cassandra-T2023-03-30-06-21-02.log /var/log/EventLog/Processed_logs_cassandra-T2023-03-29-07-34-44.log /var/log/EventLog/Processed_logs_cassandra-T2023-04-25-08-00-56.log /var/log/EventLog/Processed_logs_cassandra-T2023-04-25-07-59-19.log /var/log/EventLog/Processed_logs_cassandra-T2023-04-25-08-01-58.log /var/log/EventLog/Processed_logs_cassandra-T2023-03-29-07-35-54.log /var/log/EventLog/Processed_logs_cassandra-T2023-03-29-07-37-24.log /var/log/EventLog/Processed_logs_cassandra-T2023-03-29-07-38-49.log /var/log/EventLog/Processed_logs_cassandra-T2023-03-30-06-17-14.log /var/log/EventLog/Processed_logs_cassandra-T2023-03-30-06-18-03.log /var/log/EventLog/Processed_logs_cassandra-T2023-04-25-08-03-24.log Received dataframe with shape (36147, 5) Filtering important logs using keywords: ['unauthorized, error, cron, kernel error, OS error, rejected, warning, error, fail, exception, critical, security, authentication, intrusion, attack, status, performance, uptime, load, config, setting, permission, firewall, debug, trace, stack, hosts, /etc/passwd, /etc/sudoers, config_change, /etc/group , Root Shell Access on,fstab_edited'] Number of important logs found: 0 ['/var/log/EventLog/Processed_logs_cassandra-T2023-03-30-06-21-02.log', '/var/log/EventLog/Processed_logs_cassandra-T2023-03-29-07-34-44.log', '/var/log/EventLog/Processed_logs_cassandra-T2023-04-25-08-00-56.log', '/var/log/EventLog/Processed_logs_cassandra-T2023-04-25-07-59-19.log', '/var/log/EventLog/Processed_logs_cassandra-T2023-04-25-08-01-58.log', '/var/log/EventLog/Processed_logs_cassandra-T2023-03-29-07-35-54.log', '/var/log/EventLog/Processed_logs_cassandra-T2023-03-29-07-37-24.log', '/var/log/EventLog/Processed_logs_cassandra-T2023-03-29-07-38-49.log', '/var/log/EventLog/Processed_logs_cassandra-T2023-03-30-06-17-14.log', '/var/log/EventLog/Processed_logs_cassandra-T2023-03-30-06-18-03.log', '/var/log/EventLog/Processed_logs_cassandra-T2023-04-25-08-03-24.log'] Traceback (most recent call last): File "apache_beam/runners/common.py", line 1198, in apache_beam.runners.common.DoFnRunner.process File "apache_beam/runners/common.py", line 537, in apache_beam.runners.common.SimpleInvoker.invoke_process File "windows_postgre_event_log.py", line 88, in process for hostname, group in df.groupby('hostname'): File "/home/TCSCAPEAGENT/Python/venv/CAPEANALYTICS/lib64/python3.6/site-packages/pandas/core/frame.py", line 6525, in groupby dropna=dropna, File "/home/TCSCAPEAGENT/Python/venv/CAPEANALYTICS/lib64/python3.6/site-packages/pandas/core/groupby/groupby.py", line 533, in __init__ dropna=self.dropna, File "/home/TCSCAPEAGENT/Python/venv/CAPEANALYTICS/lib64/python3.6/site-packages/pandas/core/groupby/grouper.py", line 786, in get_grouper raise KeyError(gpr) KeyError: 'hostname' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "windows_postgre_event_log.py", line 130, in <module> | "Split and Save Files" >> beam.ParDo(SplitAndSaveFilesDoFn(folder_path))) File "/home/TCSCAPEAGENT/Python/venv/CAPEANALYTICS/lib64/python3.6/site-packages/apache_beam/pipeline.py", line 596, in __exit__ self.result = self.run() File "/home/TCSCAPEAGENT/Python/venv/CAPEANALYTICS/lib64/python3.6/site-packages/apache_beam/pipeline.py", line 573, in run return self.runner.run_pipeline(self, self._options) File "/home/TCSCAPEAGENT/Python/venv/CAPEANALYTICS/lib64/python3.6/site-packages/apache_beam/runners/direct/direct_runner.py", line 131, in run_pipeline return runner.run_pipeline(pipeline, options) File "/home/TCSCAPEAGENT/Python/venv/CAPEANALYTICS/lib64/python3.6/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py", line 200, in run_pipeline pipeline.to_runner_api(default_environment=self._default_environment)) File "/home/TCSCAPEAGENT/Python/venv/CAPEANALYTICS/lib64/python3.6/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py", line 210, in run_via_runner_api return self.run_stages(stage_context, stages) File "/home/TCSCAPEAGENT/Python/venv/CAPEANALYTICS/lib64/python3.6/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py", line 396, in run_stages runner_execution_context, bundle_context_manager) File "/home/TCSCAPEAGENT/Python/venv/CAPEANALYTICS/lib64/python3.6/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py", line 667, in _run_stage bundle_manager)) File "/home/TCSCAPEAGENT/Python/venv/CAPEANALYTICS/lib64/python3.6/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py", line 784, in _run_bundle data_input, data_output, input_timers, expected_timer_output) File "/home/TCSCAPEAGENT/Python/venv/CAPEANALYTICS/lib64/python3.6/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py", line 1094, in process_bundle result_future = self._worker_handler.control_conn.push(process_bundle_req) File "/home/TCSCAPEAGENT/Python/venv/CAPEANALYTICS/lib64/python3.6/site-packages/apache_beam/runners/portability/fn_api_runner/worker_handlers.py", line 378, in push response = self.worker.do_instruction(request) File "/home/TCSCAPEAGENT/Python/venv/CAPEANALYTICS/lib64/python3.6/site-packages/apache_beam/runners/worker/sdk_worker.py", line 581, in do_instruction getattr(request, request_type), request.instruction_id) File "/home/TCSCAPEAGENT/Python/venv/CAPEANALYTICS/lib64/python3.6/site-packages/apache_beam/runners/worker/sdk_worker.py", line 618, in process_bundle bundle_processor.process_bundle(instruction_id)) File "/home/TCSCAPEAGENT/Python/venv/CAPEANALYTICS/lib64/python3.6/site-packages/apache_beam/runners/worker/bundle_processor.py", line 996, in process_bundle element.data) File "/home/TCSCAPEAGENT/Python/venv/CAPEANALYTICS/lib64/python3.6/site-packages/apache_beam/runners/worker/bundle_processor.py", line 221, in process_encoded self.output(decoded_value) File "apache_beam/runners/worker/operations.py", line 346, in apache_beam.runners.worker.operations.Operation.output File "apache_beam/runners/worker/operations.py", line 348, in apache_beam.runners.worker.operations.Operation.output File "apache_beam/runners/worker/operations.py", line 215, in apache_beam.runners.worker.operations.SingletonConsumerSet.receive File "apache_beam/runners/worker/operations.py", line 707, in apache_beam.runners.worker.operations.DoOperation.process File "apache_beam/runners/worker/operations.py", line 708, in apache_beam.runners.worker.operations.DoOperation.process File "apache_beam/runners/common.py", line 1200, in apache_beam.runners.common.DoFnRunner.process File "apache_beam/runners/common.py", line 1265, in apache_beam.runners.common.DoFnRunner._reraise_augmented File "apache_beam/runners/common.py", line 1198, in apache_beam.runners.common.DoFnRunner.process File "apache_beam/runners/common.py", line 536, in apache_beam.runners.common.SimpleInvoker.invoke_process File "apache_beam/runners/common.py", line 1361, in apache_beam.runners.common._OutputProcessor.process_outputs File "apache_beam/runners/worker/operations.py", line 215, in apache_beam.runners.worker.operations.SingletonConsumerSet.receive File "apache_beam/runners/worker/operations.py", line 707, in apache_beam.runners.worker.operations.DoOperation.process File "apache_beam/runners/worker/operations.py", line 708, in apache_beam.runners.worker.operations.DoOperation.process File "apache_beam/runners/common.py", line 1200, in apache_beam.runners.common.DoFnRunner.process File "apache_beam/runners/common.py", line 1265, in apache_beam.runners.common.DoFnRunner._reraise_augmented File "apache_beam/runners/common.py", line 1198, in apache_beam.runners.common.DoFnRunner.process File "apache_beam/runners/common.py", line 536, in apache_beam.runners.common.SimpleInvoker.invoke_process File "apache_beam/runners/common.py", line 1361, in apache_beam.runners.common._OutputProcessor.process_outputs File "apache_beam/runners/worker/operations.py", line 215, in apache_beam.runners.worker.operations.SingletonConsumerSet.receive File "apache_beam/runners/worker/operations.py", line 707, in apache_beam.runners.worker.operations.DoOperation.process File "apache_beam/runners/worker/operations.py", line 708, in apache_beam.runners.worker.operations.DoOperation.process File "apache_beam/runners/common.py", line 1200, in apache_beam.runners.common.DoFnRunner.process File "apache_beam/runners/common.py", line 1265, in apache_beam.runners.common.DoFnRunner._reraise_augmented File "apache_beam/runners/common.py", line 1198, in apache_beam.runners.common.DoFnRunner.process File "apache_beam/runners/common.py", line 718, in apache_beam.runners.common.PerWindowInvoker.invoke_process File "apache_beam/runners/common.py", line 841, in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window File "apache_beam/runners/common.py", line 1361, in apache_beam.runners.common._OutputProcessor.process_outputs File "apache_beam/runners/worker/operations.py", line 215, in apache_beam.runners.worker.operations.SingletonConsumerSet.receive File "apache_beam/runners/worker/operations.py", line 707, in apache_beam.runners.worker.operations.DoOperation.process File "apache_beam/runners/worker/operations.py", line 708, in apache_beam.runners.worker.operations.DoOperation.process File "apache_beam/runners/common.py", line 1200, in apache_beam.runners.common.DoFnRunner.process File "apache_beam/runners/common.py", line 1265, in apache_beam.runners.common.DoFnRunner._reraise_augmented File "apache_beam/runners/common.py", line 1198, in apache_beam.runners.common.DoFnRunner.process File "apache_beam/runners/common.py", line 536, in apache_beam.runners.common.SimpleInvoker.invoke_process File "apache_beam/runners/common.py", line 1361, in apache_beam.runners.common._OutputProcessor.process_outputs File "apache_beam/runners/worker/operations.py", line 215, in apache_beam.runners.worker.operations.SingletonConsumerSet.receive File "apache_beam/runners/worker/operations.py", line 707, in apache_beam.runners.worker.operations.DoOperation.process File "apache_beam/runners/worker/operations.py", line 708, in apache_beam.runners.worker.operations.DoOperation.process File "apache_beam/runners/common.py", line 1200, in apache_beam.runners.common.DoFnRunner.process File "apache_beam/runners/common.py", line 1281, in apache_beam.runners.common.DoFnRunner._reraise_augmented File "apache_beam/runners/common.py", line 1198, in apache_beam.runners.common.DoFnRunner.process File "apache_beam/runners/common.py", line 537, in apache_beam.runners.common.SimpleInvoker.invoke_process File "windows_postgre_event_log.py", line 88, in process for hostname, group in df.groupby('hostname'): File "/home/TCSCAPEAGENT/Python/venv/CAPEANALYTICS/lib64/python3.6/site-packages/pandas/core/frame.py", line 6525, in groupby dropna=dropna, File "/home/TCSCAPEAGENT/Python/venv/CAPEANALYTICS/lib64/python3.6/site-packages/pandas/core/groupby/groupby.py", line 533, in __init__ dropna=self.dropna, File "/home/TCSCAPEAGENT/Python/venv/CAPEANALYTICS/lib64/python3.6/site-packages/pandas/core/groupby/grouper.py", line 786, in get_grouper raise KeyError(gpr) KeyError: "hostname [while running 'Split and Save Files']"