Untitled
unknown
plain_text
2 years ago
32 kB
4
Indexable
# -*- coding: utf-8 -*- import copy import json import os from bson import ObjectId from datetime import date from datetime import datetime from pymongo import MongoClient from common.config import config from common.log import setup_logging from common.secure import generate_secrets from constants.alert_methods import * from constants.alert_state import * from constants.resource_types import * ENV = os.environ.get('ENV', 'development') CONF = config[ENV] LOG = setup_logging(__name__) conn = MongoClient(CONF.MONGO_URI) db = conn[CONF.MONGO_DATABASE] def get_all_alarms(resource_type=None): alarms = db[CONF.MONGO_ALARMS] if not resource_type: return alarms.find({"_deleted": False}) else: return alarms.find({ "_deleted": False, 'resource_type': resource_type}) def get_all_enabled_alarms(resource_type=None): alarms = db[CONF.MONGO_ALARMS] if not resource_type: return alarms.find({'enable': True, "_deleted": False}) else: return alarms.find({ 'enable': True, "_deleted": False, 'resource_type': resource_type}) def get_all_enabled_alarms_for_autoscaling(cluster_id): alarms = db[CONF.MONGO_ALARMS] alarm_ids = { "scale_out": [], "scale_in": [], } if not cluster_id: return {} else: for alarm in alarms.find({ 'enable': True, "_deleted": False, 'resource_type': AUTOSCALE_GROUP, "cluster_id": cluster_id, }): if alarm['comparison']['compare_type'] in ["<", "<="]: alarm_ids["scale_in"].append(str(alarm["_id"])) else: alarm_ids["scale_out"].append(str(alarm["_id"])) return alarm_ids def get_alarm(alarm_id): try: alarms = db[CONF.MONGO_ALARMS] alarms = list(alarms.find( {'_id': ObjectId(alarm_id), "_deleted": False})) if len(alarms) == 1: return alarms[0] except: return False def set_delete_alarm(alarm_id): try: alarms = db[CONF.MONGO_ALARMS] alarms.updateMany( {"_id": alarm_id}, { "$set": { "_deleted": True } } ) except: pass def get_receiver(receiver_id): try: receiver = db[CONF.MONGO_RECEIVERS] receiver = list(receiver.find( {'_id': ObjectId(receiver_id), "_deleted": False})) if len(receiver) == 1: return receiver[0] except: return False def is_deleted_receivers(receiver_id): try: receiver = db[CONF.MONGO_RECEIVERS] receiver = list(receiver.find( {'_id': ObjectId(receiver_id), "_deleted": True})) if len(receiver) == 1: return True except: return False def save_history(alarm_name, alarm_id, state, user_id, project_id, resource_type, measurement=None, resource=None, value=None): now = datetime.now() history = db[CONF.MONGO_HISTORIES] try: history.insert_one({ 'name': alarm_name, 'alarm_id': str(alarm_id), 'resource_type': resource_type, 'state': state, 'user_id': user_id, 'project_id': project_id, 'resource': resource, 'measurement': measurement, '_created': now, 'value': value }) except Exception as e: LOG.error("Can't write history for alert. Error {}".format(str(e))) def save_message_audit(alarm_id, receiver_id, method_type, method_address, message=None, state=None, method=None, user_id=None, project_id=None, resource=None, resource_type=None): """Luu lich su canh bao Arguments: alarm_id {[type]} -- [description] receiver_id {[type]} -- [description] method_type {[type]} -- [description] method_address {[type]} -- [description] Keyword Arguments: message {[type]} -- [description] (default: {None}) state {[type]} -- [description] (default: {None}) method {[type]} -- [description] (default: {None}) user_id {[type]} -- [description] (default: {None}) project_id {[type]} -- [description] (default: {None}) resource {[type]} -- [description] (default: {None}) resource_type {[type]} -- [description] (default: {None}) """ now = datetime.now() try: message_audit = db[CONF.MONGO_MESSAGE_AUDIT] message_audit.insert_one({ '_created': now, 'alarm_id': alarm_id, 'message': message, 'method': method, 'project_id': project_id, 'receiver_id': receiver_id, 'resource': resource, 'resource_type': resource_type, 'state': state, 'user_id': user_id, method_type: method_address, }) except Exception as e: LOG.error( "Can't write message audit for user " "{}/ project {}. Error {}".format( user_id, project_id, str(e)) ) def get_latest_trigger_alert(alarm_id, resource=None, measurement=None): history = db[CONF.MONGO_HISTORIES] h = history.find({ 'alarm_id': str(alarm_id), 'resource': resource, 'measurement': measurement }).limit(1).sort('_id', -1) return h def list_projects(): alarms = db[CONF.MONGO_ALARMS] return alarms.distinct('project_id') def count_message(project_id, start_date, end_date, method=None): try: query = { 'project_id': project_id, '_created': { '$gte': start_date, '$lte': end_date } } if method: query.update({'method': method}) audit = db[CONF.MONGO_MESSAGE_AUDIT] return audit.count(query) except Exception: return 0 def save_statistic(body): try: is_exist = db.statistics.count({ 'project_id': body.get('project_id'), 'datetime': body.get('datetime') }) if not is_exist: db.statistics.insert_one(body) return True except Exception: return False def save_service_states(hostname, service_name): states = db[CONF.MONGO_SERVICE_STATES] try: recent_object = list(states.find({"service_name": service_name})) now = datetime.now() if len(recent_object) > 0: states.update({ "service_name": service_name }, { "$set": { "hostname": hostname, "service_name": service_name, "last_updated": now, "_updated": now } }) else: states.insert_one({ "hostname": hostname, "service_name": service_name, "last_updated": now, "_created": now }) except Exception as e: LOG.error("Can't write state for worker. Error {}".format(str(e))) def find_to_update_objects_alarms(**payload): """ Dùng để cập nhật lại thông tin alarm khi mà tài nguyên bị xóa Sử dụng cho cuckoo_aphora dùng để tự động cập nhật lại alarms resource_type là loại alarms cho cái nào: instance, volume, ... payload: là cái dict bao gồm id và name và project_id, resource_type """ alarms = db[CONF.MONGO_ALARMS] if payload['resource_type'] != 'load_balancer': resources = list(alarms.find({ "_deleted": False, "project_id": payload['project_id'], "resource_type": { "$in": [payload['resource_type']] }, payload['resource_type'] + 's': { "$elemMatch": { "id": { "$eq": payload['id'] }, "name": { "$eq": payload['name'] } } } })) if payload['resource_type'] == 'load_balancer': if payload['event_type'] == "delete_load_balancer": resources = list(alarms.find({ "_deleted": False, "resource_type": { "$in": [payload['resource_type']] }, payload['resource_type'] + 's': { "$elemMatch": { "load_balancer_id": { "$eq": payload['id'] } } } })) if payload['event_type'] == "delete_listener": resources = list(alarms.find({ "_deleted": False, "resource_type": { "$in": [payload['resource_type']] }, payload['resource_type'] + 's': { "$elemMatch": { "target_id": { "$eq": payload['id'] }, "target_type": "frontend" } } })) if payload['event_type'] == "delete_pool": resources = list(alarms.find({ "_deleted": False, "resource_type": { "$in": [payload['resource_type']] }, payload['resource_type'] + 's': { "$elemMatch": { "target_id": { "$eq": payload['id'] }, "target_type": "backend" } } })) for resource in resources: # danh sách resources được sử dụng để giám sát. alarm_targets = resource[payload['resource_type'] + 's'] if payload['resource_type'] != 'load_balancer': # Xóa resource đã bị xóa đi for alarm_target in alarm_targets: if alarm_target.get("id") == payload['id']: alarm_targets.remove(alarm_target) if payload['resource_type'] == 'load_balancer': for lb in alarm_targets: if payload['event_type'] == "delete_load_balancer": if lb['load_balancer_id'] == payload['id']: alarm_targets.remove(lb) elif payload['event_type'] in [ "delete_listener", "delete_pool" ]: if lb['target_id'] == payload['id']: alarm_targets.remove(lb) LOG.debug("Updating alarm {}".format(resource['_id'])) alarms.update_one({ "_id": resource['_id'] }, { "$set": { payload['resource_type'] + 's': alarm_targets } }) def find_to_update_methods_receivers_alarms(receiver_id, **kwargs): """ Dùng để cập nhật lại thông tin alarm khi mà receivers bị thay đổi thông tin: phương thức nhận """ alarms = db[CONF.MONGO_ALARMS] resources = list(alarms.find({ "_deleted": False, "project_id": kwargs.get('project_id'), "receivers": { "$elemMatch": { "receiver_id": { "$eq": str(receiver_id) } } } })) for resource in resources: need_update = False for receiver in resource['receivers']: if receiver['receiver_id'] == str(receiver_id): methods = copy.copy(receiver['methods']) for method in methods: if method not in kwargs.get('methods'): receiver['methods'].remove(method) need_update = True if need_update: LOG.debug("Updating alarm {}".format(resource['_id'])) alarms.update_one({ "_id": resource['_id'], "_deleted": False }, { "$set": { "receivers": resource['receivers'] } }) def find_to_update_list_receivers_alarms(receiver_id, **kwargs): """ Dùng để cập nhật lại danh sách các receivers khi receivers bị force delete. """ alarms = db[CONF.MONGO_ALARMS] resources = list(alarms.find({ "_deleted": False, "project_id": kwargs.get('project_id'), "receivers": { "$elemMatch": { "receiver_id": { "$eq": str(receiver_id) } } } })) for resource in resources: need_update = False receivers = copy.copy(resource['receivers']) for receiver in resource['receivers']: if receiver['receiver_id'] == str(receiver_id): # Xoa receiver se bi xoa ra khoi danh sach receivers.remove(receiver) need_update = True if need_update: LOG.debug("Updating alarm {}".format(resource['_id'])) alarms.update_one({ "_id": resource['_id'], "_deleted": False }, { "$set": { "receivers": receivers } }) # ---------------------------------------------------------------------------- # PROCESS FOR SMS RESOURCES # ---------------------------------------------------------------------------- def bought_sms_packages(user_id): # Kiem tra nguoi dung da tung mua sms package sms_packages = db[CONF.MONGO_SMS_PACKAGES] resources = list(sms_packages.find({ 'user_id': user_id })) if len(resources) == 1: return True return False def update_bought_sms_packages( amount_sms, auto_renew=False, package_id=None, user_id=None): # Luu so sms co the dung sau khi tu dong mua goi sms # Cap nhat thong tin sms packages sms_packages = db[CONF.MONGO_SMS_PACKAGES] if package_id: resources = list(sms_packages.find({ '_id': package_id })) elif user_id: resources = list(sms_packages.find({ 'user_id': user_id })) if len(resources) == 1: package = resources[0] sms_packages.update_one({ '_id': package.get('_id') }, { '$inc': { 'pending_sms': amount_sms, } }) def get_sms_packages(package_id=None, user_id=None): sms_packages = db[CONF.MONGO_SMS_PACKAGES] if package_id: resources = list(sms_packages.find({ '_id': package_id })) elif user_id: resources = list(sms_packages.find({ 'user_id': user_id })) if len(resources) == 1: return resources[0] return {} def decrease_sms_packages_by_one(package_id=None, user_id=None): """Giam so luong sms cua user di 1 Khi gui sms thanh cong can giam so luong sms hien co cua user di 1 Keyword Arguments: package_id {[type]} -- [description] (default: {None}) user_id {[type]} -- [description] (default: {None}) """ sms_packages = db[CONF.MONGO_SMS_PACKAGES] if package_id: sms_packages.update_one({ '_id': package_id }, { '$inc': { 'pending_sms': -1 } }) elif user_id: sms_packages.update_one({ 'user_id': user_id }, { '$inc': { 'pending_sms': -1 } }) def receiver_infor_verified(method_type, method_address, project_id): # Tim thong tin receivers da duoc xac nhan receiver = db[CONF.MONGO_RECEIVERS] if method_type == 'sms_number': method_type_verify = 'verified_sms_number' if method_type == 'email_address': method_type_verify = 'verified_email_address' if method_type == 'webhook_url': method_type_verify = 'verified_webhook_url' if method_type == 'telegram_chat_id': method_type_verify = 'verified_telegram_chat_id' receiver = list(receiver.find({ 'project_id': project_id, "_deleted": False, method_type: method_address, method_type_verify: True, })) if len(receiver) != 0: return True else: return False def save_message_alarms(receiver_id, method, resource, message, resource_type=None): """Save message of alarms with same receiver_id resource_type [description] Arguments: receiver_id -- receiver_id using to sent message method -- method of receiver_id [email, slack, telegram] message -- content of messages alarm_id: id of alarm """ tmp = db['tmp'] if method not in ['email', 'slack', 'telegram', 'webhook', 'sms']: return False try: # When resource have dot, we need convert resource = resource.replace('.', '-') except: if resource_type == BIZFLY_METRICS_AGENTS: resource["agent_id"] = str(resource["agent_id"]) resource = json.dumps(resource).replace('.', '-') queues = list(tmp.find({ 'receiver_id': receiver_id, 'resource_type': resource_type, 'method': method })) # Have message will update its if len(queues) != 0: tmp.update_one({ 'receiver_id': receiver_id, 'resource_type': resource_type, 'method': method }, { '$set': { 'messages.' + resource: message } }) return True tmp.insert_one({ 'receiver_id': receiver_id, 'resource_type': resource_type, 'method': method, 'messages': {resource: message}, }) return True def delete_message_alarms(receiver_id, method, resource_type=None): """Delete message was saved before [description] Arguments: receiver_id -- receiver_id using to sent message method -- method of receiver_id [email, slack, telegram] """ tmp = db['tmp'] if method not in ['email', 'slack', 'telegram', 'webhook', 'sms']: return False tmp.delete_many({ 'receiver_id': receiver_id, 'resource_type': resource_type, 'method': method }) return True def get_message_alarms(receiver_id, method, resource_type=None): """Get message was saved before [description] Arguments: receiver_id -- receiver_id using to sent message method -- method of receiver_id [email, slack, telegram] """ tmp = db['tmp'] if method not in ['email', 'slack', 'telegram', 'webhook', 'sms']: return [] messages = list(tmp.find({ 'receiver_id': receiver_id, 'resource_type': resource_type, 'method': method })) if len(messages) > 0: delete_message_alarms( receiver_id=receiver_id, method=method, resource_type=resource_type) return messages[0].get('messages').values() return [] def get_methods_receivers(receiver_id): # Return methods of receivers receiver = db[CONF.MONGO_RECEIVERS] receiver = list(receiver.find( {'_id': ObjectId(receiver_id), "_deleted": False})) methods = [] if len(receiver) == 1: receiver = receiver[0] if 'sms_number' in receiver: methods.append(SMS) if 'email_address' in receiver: methods.append(EMAIL) if 'telegram_chat_id' in receiver: methods.append(TELEGRAM) if 'webhook_url' in receiver: methods.append(WEB_HOOK) # if 'autoscale_cluster_name' in receiver: # methods.append(AUTOSCALE_GROUP) if 'slack_channel_name' in receiver: methods.append(SLACK) return methods def save_do_alert(alarm_id, project_id, user_id, message, state): """Save when called alert function [description] Arguments: alarm_id {str} -- id of alarms project_id {str} -- id of project user_id {str} -- id of user """ audit = db[CONF.MONGO_MESSAGE_AUDIT] now = datetime.now() audit.insert_one({ '_created': now, 'alarm_id': alarm_id, 'called_alert': True, 'message': message, 'project_id': project_id, 'state': state, 'user_id': user_id, }) def count_message_by_alarms(alarm_id, start_date, end_date, resource_type, resource=None): """ Do count record in audit collection with state of resource_type with range time Arguments: alarm_id {[type]} -- [description] start {[type]} -- [description] end {[type]} -- [description] Returns: [type] -- [description] """ if resource_type in [HOST, HTTP]: state = [DOWN] else: state = [ALARM, CRITICAL] try: query = { 'alarm_id': alarm_id, 'resource': resource, 'state': {'$in': state}, '_created': { '$gte': start_date, '$lte': end_date } } history = db[CONF.MONGO_HISTORIES] return history.count_documents(query) except Exception: return 0 def get_latest_sent_sms(alarm_id, resource, resource_type, receiver_id, state): """Tim thoi gian gui sms gan nhat [description] Arguments: alarm_id {[type]} -- [description] resource {[type]} -- [description] resource_type {[type]} -- [description] receiver_id {[type]} -- [description] state {[type]} -- [description] Returns: [type] -- [description] """ histories_sms = db[CONF.MONGO_HISTORIES_SMS] h = histories_sms.find({ 'alarm_id': alarm_id, 'resource': resource, 'resource_type': resource_type, 'receiver_id': receiver_id, 'state': state, }).limit(1).sort('$natural', -1) return h def get_latest_good_alarms(alarm_id, resource, resource_type, receiver_id): """Tim thoi gian gui canh bao binh thuong gan nhat [description] Arguments: alarm_id {[type]} -- [description] resource {[type]} -- [description] resource_type {[type]} -- [description] receiver_id {[type]} -- [description] Returns: [type] -- [description] """ audit = db[CONF.MONGO_MESSAGE_AUDIT] unexpect_state = [ALARM, CRITICAL, DOWN] h = audit.find({ 'alarm_id': alarm_id, 'resource': resource, 'resource_type': resource_type, 'receiver_id': receiver_id, 'state': {'$nin': unexpect_state}, }).limit(1).sort('$natural', -1) return h def save_history_sms(alarm_id, receiver_id, message, user_id, project_id, state=None, resource=None, resource_type=None): """Save history sms Save information when was sent a sms to receiver_id Arguments: alarm_id {[type]} -- [description] receiver_id {[type]} -- [description] message {[type]} -- [description] """ histories_sms = db[CONF.MONGO_HISTORIES_SMS] now = datetime.now() histories_sms.insert_one({ '_created': now, 'alarm_id': alarm_id, 'receiver_id': receiver_id, 'message': message, 'user_id': user_id, 'project_id': project_id, 'resource': resource, 'resource_type': resource_type, 'state': state, }) def save_log_sms(sms_number, context): """ Save response from API using sent sms """ log_sms = db[CONF.MONGO_LOG_SMS] now = datetime.now() log_sms.insert_one({ '_created': now, 'sms_number': sms_number, 'context_log': context, }) def get_log_sms_count_perday(): # Return log count that sms was sent successful log_sms = db[CONF.MONGO_LOG_SMS] today = date.today().ctime() try: count = log_sms.find({ "_created": { "$gt": datetime.strptime(today, "%a %b %d %H:%M:%S %Y") } }).count() return count except Exception as e: LOG.error("Failed to get sms count was sent: %s" % e) return 0 def get_settings(config_key): # Return value of setting service """ Data struct in settings collection: { "config": name_of_config, "value": value_of_config } """ settings = db[CONF.MONGO_SETTINGS] try: settings = list(settings.find({ "config": config_key })) if len(settings) == 1: return settings[0].get("value") except: return {} return {} def get_quotas_sms_system(): # Return quotas sms of sytems per day sms_perday = get_settings("quotas_sms_perday") threshold = get_settings("quotas_sms_threshold_to_alert") return round(sms_perday * threshold / 100) def save_history_buy_sms(amount_sms, project_id, invoices, is_automation=True): """ Save history buy sms successful Pararms: amount_sms : Amount SMS that `who` was bought is_automation: Is automation buy or mananual buy? """ histories_buy_sms = db[CONF.MONGO_HISTORIES_BUY_SMS] now = datetime.now() histories_buy_sms.insert_one({ '_created': now, 'amount_sms': amount_sms, 'project_id': project_id, 'invoices': invoices, 'is_automation': is_automation }) def update_latest_send_verify_sms(receiver_id): # set last time was sent verify sms method of receiver receivers = db[CONF.MONGO_RECEIVERS] try: receiver_id = ObjectId(receiver_id) except Exception as e: LOG.debug(e) receivers.update_one({ "_id": receiver_id }, { "$set": { "last_time_send_verify_sms": datetime.now() } }) # ---------------------------------------------------------------------------- # PROCESS FOR AGENT RESOURCES # ---------------------------------------------------------------------------- def get_all_agents(): # Return all agents record agents = db['agents'] return agents.find({'_deleted': False}) def get_agents(agent_id, project_id): # Return a agent record try: agents = db['agents'] agents = list(agents.find({ '_id': ObjectId(agent_id), 'project_id': project_id, '_deleted': False })) if len(agents) == 1: return agents[0] except: return False def get_latest_agents(limit_by, project_id): # Return latest agents was register in system latest_agents = [] try: agents = db['agents'] latest_agents = list(agents.find({ 'project_id': project_id, '_deleted': False }).limit(limit_by).sort("_id", -1)) except Exception as e: LOG.error(e) return latest_agents def save_history_alert_healthcheck_agents(agent_id): healthcheck_agents = db['healthcheck_agents'] healthcheck_agents.insert_one({ "_id": ObjectId(), "_created": datetime.now(), "agent_id": agent_id }) def get_latest_healthcheck_reported(agent_id): healthcheck_agents = db['healthcheck_agents'] last_reported = list(healthcheck_agents.find({ "agent_id": agent_id }).limit(1).sort("_id", -1)) if len(last_reported) > 0: return last_reported[0] return {} def get_alarm_included_agent(agent_id, project_id): # Return a agent record try: alarms = db[CONF.MONGO_ALARMS] return alarms.find({ 'project_id': project_id, '_deleted': False, 'agents': {"$elemMatch": { "agent_id": agent_id }} }) except: return [] # ---------------------------------------------------------------------------- # PROCESS FOR SECRET RESOURCES # ---------------------------------------------------------------------------- def create_secret(data): now = datetime.now() secrets = db['secrets'] try: data["_created"] = now secrets.insert_one(data) return True except Exception as e: LOG.error("Can't create new secret. Error {}".format(str(e))) return False def get_secret(project_id, secret_id=None, secret=None): # Return a secret record try: secrets = db['secrets'] queries = { "project_id": project_id } if secret_id: queries["_id"] = secret_id secrets = list(secrets.find(queries)) if len(secrets) >= 1: if secret and generate_secrets( data=secret, salt=secrets[0]["salt"] ) != secrets[0]["hashes_secret"]: return False return secrets[0] except: return False def get_all_secrets(project_id, user_id, **kwargs): secrets = db['secrets'] filters = kwargs.get("where", {}) filters["project_id"] = project_id filters["user_id"] = user_id return list(secrets.find(filters)) def delete_secret(secret_id, project_id, user_id): secrets = db['secrets'] try: secrets.delete_many({ "_id": secret_id, "project_id": project_id, 'user_id': user_id }) except: return False return True def save_alarm_state(alarm_id, state, measurement=None, resource=None): # Save state of an alarm now = datetime.now() states = db[CONF.MONGO_ALARMS_STATES] try: states.insert_one({ '_created': now, 'alarm_id': alarm_id, 'measurement': measurement, 'resource': resource, 'state': state, }) except Exception as e: LOG.error("Can't write states for alarm: {} - {}".format( alarm_id, str(e))) def delete_alarm_state(alarm_id): # Set _deleted for state of an alarm now = datetime.now() states = db[CONF.MONGO_ALARMS_STATES] try: states.update({ 'alarm_id': alarm_id, }, { "$set": { '_deleted': now, } }) except Exception as e: LOG.error("Can't delete states for alarm: {} - {}".format( alarm_id, str(e))) def update_alarm_state(alarm_id, state, resource, measurement=None): # Update state of an alarm states = db[CONF.MONGO_ALARMS_STATES] try: states.update({ 'alarm_id': alarm_id, 'measurement': measurement, 'resource': resource, }, { "$set": { 'state': state, } }) except Exception as e: LOG.error("Can't update states for alarm: {} - {}".format( alarm_id, str(e))) def get_latest_alarm_state(alarm_id, measurement=None, resource=None): # Return state of an alarm states = db[CONF.MONGO_ALARMS_STATES] if measurement: s = states.find({ 'alarm_id': alarm_id, 'resource': resource, 'measurement': measurement }).limit(1) else: s = states.find({ 'alarm_id': alarm_id, 'resource': resource }).limit(1) return s
Editor is loading...