Untitled

mail@pastecode.io avatar
unknown
plain_text
a year ago
12 kB
1
Indexable
Never
"""
#######################################################################################################################################################################
API Name 	: cacheStore
Description 	: This api stores and fetches the cache details stored in the cassandra table cache_store table in hobs_ods_staging keyspace.
		  Paramters are required to fetch the cache details.valid parameters are valueType,key
 		  Columns that needs to be returned in the get response can be configured in validColumns property in config/dataservice/cacheStore.config file
		  Columns that needs to be passed in the post request can be configured in validKeys property in config/dataservice/cacneStore.config file
		  valis
Parameters that can be passed are customerId,interactionId,interactionDate,startDate,endDate. And the parameters are configurable in config/dataservice/getInteractions.config


Revision History
----------------

Version         Date                    Author                          Description
0.1             29-Nov-2021             senthilCR                       Initial version of the api to store and fetch cache details in the cache_store
0.2		09-Feb-2022		senthilCR			modified the script to handle value input as dictionaty or list of dictionary


#######################################################################################################################################################################
"""

from models.cache_store import cache_store
from models.cache_store_by_username import cache_store_by_username
from models.cache_store_by_date import cache_store_by_date
from models.indexes import indexes
from models.columns import columns as tablecolumns
from datetime import datetime, timedelta, timezone
from dateutil.relativedelta import relativedelta
from pandas.io.json import json_normalize
import json
from config.logger import configlogfile

from flask import jsonify, make_response
import pandas as pd
import sys
import gc
import uuid
import pytz

"""
This function is used to fetch cache details based on valueType and key
"""


def getData(payLoad, configvalues):
    from config.logger import configlogfile
    logging = configlogfile()
    logging.info("Started to fetch the records for " + str(payLoad))

    cacheStore = []
    try:
        payLoad = payLoad.to_dict()
        columnMapping = json.loads(configvalues.get('apiconfiguration', 'columnMapping', raw=True))
        reversecolumnMapping = {y: x for x, y in columnMapping.items()}
        validColumns = json.loads(configvalues.get('apiconfiguration', 'validColumns', raw=True))
        datecolumns = json.loads(configvalues.get('apiconfiguration', 'dateColumns', raw=True))
        datecolumns.append('createdtime')  # Add 'createdtime' column
        validArguments = json.loads(configvalues.get('apiconfiguration', 'validArguments', raw=True))
        logging.debug('Arguments -' + str(payLoad))

        for k, v in reversecolumnMapping.items():
            if k in list(payLoad.keys()):
                payLoad[v] = payLoad.pop(k)

        recs = indexes.objects().filter(keyspace_name=cache_store.__keyspace__)
        recs = recs.filter(table_name='cache_store')
        indexedcolumns = [row.options['target'] for row in recs]

        recs = tablecolumns.objects().filter(keyspace_name=cache_store.__keyspace__)
        recs = recs.filter(table_name='cache_store')

        partitioncolumns = [row.column_name for row in recs if row.kind in ["partition_key"]]
        partitioncolumns = partitioncolumns + [row.column_name for row in recs if row.kind in ["primary_key", "clustering"]]
        parametercolumns = partitioncolumns + indexedcolumns

        if not bool(payLoad):
            return (("200", "Parameters need to be passed to fetch values from the cache store"))

        if set(list(payLoad.keys())).issubset(validArguments):
            for i in range(len(partitioncolumns)):
                if partitioncolumns[i] in list(payLoad.keys()):
                    if partitioncolumns[i] in datecolumns:
                        query = 'global cacheStoreRecords;cacheStoreRecords=cache_store.objects().filter(' + \
                            partitioncolumns[i] + '=datetime.strptime(\'' + \
                            str(payLoad[partitioncolumns[i]]) + '\',\'%Y-%m-%d\'));'
                    else:
                        query = 'global cacheStoreRecords;cacheStoreRecords=cache_store.objects().filter(' + \
                            partitioncolumns[i] + '=\'' + str(payLoad[partitioncolumns[i]]) + '\');'
                    exec(query)
        else:
            return (("200", ("9003", "Invalid arguments passed to the API. Valid arguments are " + ', '.join(validArguments))))

        if len(cacheStoreRecords) == 0:
            return (("200", {}))

        cacheStore = [row.__json__() for row in cacheStoreRecords]
        cacheStore = pd.DataFrame.from_dict(cacheStore, orient='columns')

        if len(cacheStore) > 0:
            cacheStore = cacheStore.applymap(lambda x: '' if pd.isnull(x) else x)
            cacheStore = cacheStore[validColumns]

        return (("200", cacheStore))

    except Exception as e:
        logging.error(str(sys.exc_info()[0]) + " - " + str(e))
        return (("400", ("9001", "Unable to fetch values from the cache store")))

    finally:
        cacheStoreRecords = None
        cacheStore = None
        recs = None
        partitioncolumns = None
        indexedcolumns = None
        gc.collect()
        logging.error("Error - {} . Line No - {} ".format(str(e), str(sys.exc_info()[-1].tb_lineno)))
        return (("500", "Technical exception"))


"""
This function is used to store the request details in cache_store table

"""


def postData(payLoad, configvalues):
    logging = configlogfile()
    logging.info("Preparing to save records")

    try:
        dateColumns = json.loads(configvalues.get('apiconfiguration', 'dateColumns', raw=True))
        keysToBeIgnored = json.loads(configvalues.get('apiconfiguration', 'keysToBeIgnored', raw=True))
        columnMapping = json.loads(configvalues.get('apiconfiguration', 'columnMapping', raw=True))
        validColumns = json.loads(configvalues.get('apiconfiguration', 'validKeys', raw=True))
        timeZone = configvalues.get('apiconfiguration', 'timeZone', raw=True)
        logging.debug("arguments-" + str(payLoad))
        logging.debug("validColumns-" + str(validColumns))
        logging.debug("payLoad[before]-" + str(payLoad))

        if payLoad == None or not isinstance(payLoad, dict) or len(payLoad) == 0:
            return (("422", ("9005", "Missing required attribute in payLoad. " + ','.join(validColumns) + ".")))
        if not set(list(payLoad.keys())).issubset(set(list(columnMapping.values()))):
            return (("422", ("9006", "Field rule violation. Payload having non mapped attribute. " + ','.join(
                list(set(list(payLoad.keys())) - set(columnMapping.values()))) + ".")))
            # return ((["9006"], ("Field rule violation. Payload having non mapped attribute. " + ','.join(list(set(list(payLoad.keys())) - set(columnMapping.values()))) + ".")))
        if not set(validColumns).issubset(set(list(payLoad.keys()))):
            return (("422", ("9005", "Field rule violation or missing required attribute. " + ",".join(
                list(set(validColumns) - set(list(payLoad.keys())))))))
            # return ((["9005"], "Field rule violation or missing required attribute. " + ",".join(
            # list(set(validColumns) - set(list(payLoad.keys()))))))
        logging.info(
            "saving records for cart details " + payLoad['businessId'] + "," + payLoad['valueType'] + "," + payLoad[
                'key'])

        # if 'value' in payLoad.keys() and not (isinstance(payLoad['value'], list) or isinstance(payLoad['value'],dict)):
        #    return (("422", ("9005", "Field rule violation. value field needs to be in list or dictionary format")))

        data = {}
        data = payLoad.copy()
        if 'createdTime' not in data.keys():
            data['createdTime'] = datetime.strftime(datetime.now(pytz.timezone(timeZone)), '%Y-%m-%dT%H:%M:%S.%f%z')
        if 'updatedTime' not in data.keys():
            data['updatedTime'] = datetime.strftime(datetime.now(pytz.timezone(timeZone)), '%Y-%m-%dT%H:%M:%S.%f%z')

        data = {k: v for k, v in data.items() if v is not None}  # removes the fields with null values
        data = {k: v for k, v in data.items() if k not in keysToBeIgnored}
        data = {k: v for k, v in data.items() if k in columnMapping.values()}  # remove fields not mapped

        for k, v in columnMapping.items():
            if (v in data):
                data[k] = data.pop(v)

        insertSql = ""
        for k, v in data.items():
            logging.debug(k)
            logging.debug(v)
            logging.debug(type(v));
            if k in dateColumns:  # formats the date columns in the request
                logging.debug("processing date columns")
                try:
                    datetime.strptime(v, '%Y-%m-%dT%H:%M:%S.%f%z')
                    insertSql = insertSql + k + "=datetime.strptime('" + v + "','%Y-%m-%dT%H:%M:%S.%f%z'),"
                except ValueError:
                    return (("422", ("9005",
                                     "Field rule violation. " + k + " field needs to be in 2020-01-01T00:00:00.000000+0530 format")))

            elif type(v) == str:
                if k == 'value':
                    logging.debug('value-' + str(v))

                    insertSql = insertSql + str(k) + "='" + str(
                        str(v)) + "'.encode('utf-8'),"  # convert the string to bytes to store data in blob column

                else:
                    if not v.isalnum():  # logic included to check for alphanumeric string and enclose string with double quotes

                        insertSql = insertSql + str(k) + "=\"\"\"" + v + "\"\"\","
                        # insertSql = insertSql + str(k) + "='" + str(v) + "',"

                    else:
                        insertSql = insertSql + str(k) + "='" + str(v) + "',"
            elif isinstance(v, list) or isinstance(v, dict):
                if k == 'value':
                    insertSql = insertSql + str(k) + "=b'" + json.dumps(v) + "',"
                else:
                    insertSql = insertSql + str(k) + "=" + str(v) + ","

            elif type(b) == blob:
                insertSql = insertSql + str(k) + "=textAsBlob('" + str(v) + "),"
            else:
                insertSql = insertSql + str(k) + "='" + str(v) + "',"

        insertSql = insertSql[0:len(insertSql) - 1]
        insertSql_cache = "global cacheStoreData; cacheStoreData =  cache_store(" + insertSql + ")"
        insertSql_usercache = "global cacheUserData; cacheUserData =  cache_store_by_username(" + insertSql + ")"
        insertSql_cacheData = "global cacheData; cacheData =  cache_store_by_date(" + insertSql + ")"
        logging.debug('insertSql')
        logging.debug(insertSql_cache)
        logging.debug(insertSql_usercache)
        logging.debug(insertSql_cacheData)
        # return ((["0000"],insertSql))
        exec(insertSql_cache)
        exec(insertSql_usercache)
        exec(insertSql_cacheData)

        cacheStoreData.save()
        cacheUserData.save()
        cacheData.save()
        gc.collect()
        logging.info("Successfully saved the cache details")
        return (("200", "Successfully saved the cache details"))
        return (("200", {k: v for k, v in payLoad.items()}))
    except Exception as e:
        gc.collect()
        logging.error("Error - {} . Line No - {} ".format(str(e), str(sys.exc_info()[-1].tb_lineno)))
        return (("500", "Technical exception"))