CG fetch

require 'socket'
require 'date'
require 'rubygems'
require 'json'
require 'net/http'
require 'uri'

CHITRAGUPTA_HOST = "http://chitraguptaes-use-cluster-001-internal.browserstack.com:9200"


def process_response(response)
  scroll_id = response['_scroll_id']
  hits = response['hits']['hits']
  puts("hits = #{hits.length}")
  hits.reverse.each do |hit|
    begin
      parse_event(hit)
    # rescue StandardError => e
    #   Rails.logger.error "exception occured while parsing event: #{hit} #{e.message}"
    #   @errors << e.message
    end
  end
  return scroll_id, hits
end

def parse_event(data)
  zombie_data = {}
  #data = data["_source"]
  #puts data["fields"].keys
  begin
    # print(data["fields"]["data.request.id"])
    hostname = (data["fields"]["meta.hostname"][0])
    timestamp = (data["fields"]["meta.timestamp"][0])
    dynamic_data = (data["fields"]["log.dynamic_data"][0])
    # duration = (data["fields"]["data.response.duration"][0])
    # status = (data["fields"]["data.response.status"][0])
    # ip = (data["fields"]["data.request.ip"][0])
    endpoint = (data["fields"]["data.request.endpoint"][0])
    # rails_id = (data["fields"]["data.request.id"][0])
  rescue StandardError => e
    puts "#{e.backtrace} "
    # puts data
  end

  line = {
    "meta" => {
      "timestamp" => timestamp,
      # "rails_id" => rails_id,
      # "hostname" => hostname
    },
    "log" => {
      "dynamic_data" => dynamic_data
    }
  }
  if dynamic_data.include? "2893077"
    open('/Users/ritvik/bscode/snippets/cgnginxB.log', 'a') { |f|
      f.puts "#{timestamp},#{dynamic_data}"
    }
  end
end


def get_data_from_es(scroll_id, start_time, end_time)
  if scroll_id
    request_url = CHITRAGUPTA_HOST + "/_search/scroll"
    request_body = scroll_request_body(scroll_id)
  else
    indexes = [
      "chitragupta-auto*-serverv*",
      "chitragupta_usw_001:chitragupta-auto*-serverv*",
      "chitragupta_euw_001:chitragupta-auto*-serverv*",
      "chitragupta_euwdc_001:chitragupta-auto*-serverv*"
    ]
    index = indexes.join(",")
    request_url = CHITRAGUPTA_HOST + "/#{index}/_search"
    request_body = new_start_request_body(start_time, end_time)
  end
  puts "Request URL: #{request_url} #{request_body}"
  begin
    response = make_requests(request_url, request_body)
    # puts "Response: #{response}"
    parsed_response = JSON.parse(response.body) #if response.body.present?
    # puts "parsed_response: #{parsed_response}"
    # puts parsed_response
    # parses and builds a dictionary from the response json
    result = process_response(parsed_response)
  rescue JSON::ParserError => e
    puts "Exception occured while parsing the response from es with #{e.message}"
    @errors << e.message
  rescue StandardError => e
    puts "Exception occurred while getting data from es with #{e.message}"
    @errors << e.message
  end
  return result
end

def make_requests(request_url, request_body)
  url = URI(request_url)
  http = Net::HTTP.new(url.host, url.port)
  http.read_timeout = 10
  http.open_timeout = 500
  req = Net::HTTP::Get.new(url.request_uri)
  req["Content-Type"] = "application/json"
  req.body = request_body.to_json
  req.basic_auth("chitragupta_automate_user", "23d80e979d65269b12da1f0f881548eb8b7271b7")
  result = http.request(req)
  return result
  #rescue *ALL_NET_HTTP_ERRORS => e
  #puts "NET_HTTP exception for #{request_url} with #{request_body} #{e.message}"
  #@retries += 1
  ##retry unless @retries > MAX_RETRIES # retrying for network related issues
  #@errors << e.message
end

def new_start_request_body(start_time, end_time)
  return {
    "track_total_hits": false,
    "sort": [
      {
        "meta.timestamp": {
          "order": "desc",
          "unmapped_type": "boolean"
        }
      }
    ],
    "fields": [
      {
        "field": "*",
        "include_unmapped": "true"
      },
      {
        "field": "@timestamp",
        "format": "strict_date_optional_time"
      },
      {
        "field": "meta.timestamp",
        "format": "strict_date_optional_time"
      }
    ],
    "size": 5000,
    "version": true,
    "script_fields": {},
    "stored_fields": [
      "*"
    ],
    "runtime_mappings": {},
    "_source": false,
    "query": {
      "bool": {
        "must": [],
        "filter": [
          {
            "range": {
              "meta.timestamp": {
                "format": "strict_date_optional_time",
                "gte":start_time,
                "lte": end_time,
              }
            }
          },
          {
            "match_phrase": {
              "meta.component": "railssel"
            }
          },
          {
            "match_phrase": {
              "meta.application": "rails"
            }
          },
          # {
          #   "match_phrase": {
          #     "meta.hostname": "nginx-auto-003-dcp-euw1a-prod.browserstack.com"
          #   }
          # },
          {
            "match_phrase": {
              "meta.file": "/ebs/railssel-auto/applog/production.log"
            }
          },
          {
            "match_phrase": {
              "log.kind": "Blocking*"
            }
          }
        ],
        "should": [],
        "must_not": []
      }
    },
    "highlight": {
      "pre_tags": [
        "@kibana-highlighted-field@"
      ],
      "post_tags": [
        "@/kibana-highlighted-field@"
      ],
      "fields": {
        "*": {}
      },
      "fragment_size": 2147483647
    }
  }
end


def start
  delta = 60 * 5
  # previous_hour = (Time.now.utc - (86400 * 7))
  # start_time = previous_hour.strftime("%Y-%m-%dT%H:00:00.000Z")
  # end_time = Time.now.utc.strftime("%Y-%m-%dT%H:00:00.000Z")

  # st = Time.new(2023, 03, 13, 12, 0, 0)
  # et = Time.new(2023, 03, 13, 12, 0, 10)

  # st should be from 5 to 6 hour UTC time, 2023-03-08

  st = Time.new(2023, 03, 8, 5, 0, 0)
  et = Time.new(2023, 03, 8, 7, 0, 0)
  
  
  while (st <= et)
    start_time = st.strftime("%Y-%m-%dT%H:%M:%S.000Z")
    end_time = (st+delta).strftime("%Y-%m-%dT%H:%M:%S.000Z")
    puts "#{start_time} - #{end_time}"
    get_data_from_es(nil, start_time, end_time)
    st += delta
  end

  system('ruby parsecg.rb euw-003-noon.csv')
end

data = start