Untitled

 avatar
unknown
plain_text
2 years ago
13 kB
4
Indexable
from models.cache_store import cache_store
from models.indexes import indexes
from models.columns import columns as tablecolumns
from datetime import datetime, timedelta, timezone
from dateutil.relativedelta import relativedelta
from pandas.io.json import json_normalize
import json
from config.logger import configlogfile

from flask import jsonify, make_response
import pandas as pd
import sys
import gc
import uuid
import pytz

def getData(payLoad, configvalues):
    from config.logger import configlogfile
    logging = configlogfile()
    logging.info("Started to fetch the records for " + str(payLoad) )

    cacheStore = []
    try:
        payLoad = payLoad.to_dict()
        columnMapping = json.loads(configvalues.get('apiconfiguration', 'columnMapping', raw=True))
        reversecolumnMapping = {y: x for x, y in columnMapping.items()}
        validColumns = json.loads(configvalues.get('apiconfiguration', 'validColumns', raw=True))
        datecolumns = json.loads(configvalues.get('apiconfiguration', 'dateColumns', raw=True))
        logging.debug('Arguments -' + str(payLoad))
        # return ((["0000"],str(payLoad['valueType'])))

        for k, v in reversecolumnMapping.items():
            if k in list(payLoad.keys()):
                payLoad[v] = payLoad.pop(k)
        # modified the code on 18-Feb to fetch data based on keyspace and then table name 
        recs = indexes.objects().filter(keyspace_name =cache_store.__keyspace__)
        recs = recs.filter(table_name='cache_store')
        indexedcolumns = [row.options['target'] for row in recs]

        recs =  tablecolumns.objects().filter(keyspace_name = cache_store.__keyspace__)
        recs = recs.filter(table_name='cache_store')
       
        partitioncolumns = [row.column_name for row in recs if
                            row.kind in ["partition_key"]]
        partitioncolumns = partitioncolumns + [row.column_name for row in recs if
                                               row.kind in ["primary_key", "clustering"]]
        parametercolumns = partitioncolumns + indexedcolumns
        partitioncolstofilter = [parametercolumn for parametercolumn in parametercolumns if
                            parametercolumn in list(payLoad.keys())]

        if (bool(payLoad) == False):
            return (("200","parameters needs to be passed to fetch values from the cache store"))
            #query = 'global cacheStoreRecords;cacheStoreRecords=cache_store.objects().all();'
        else:
            #return ((["0000"], str(range(len(partitioncolstofilter)))))
            if set(list(payLoad.keys())).issubset(parametercolumns):

                for i in range(len(partitioncolstofilter)):
                    if i == 0:
                        if partitioncolstofilter[i] in datecolumns:
                            query = 'global cacheStoreRecords;cacheStoreRecords=cache_store.objects().filter(' + \
                                    partitioncolstofilter[i] + '=datetime.strptime(\'' + \
                                    str((payLoad[partitioncolstofilter[i]])) + '\',\'%Y-%m-%dT%H:%M:%S%z\'));'
                            # return ((["0000"],query))
                        else:
                            query = 'global cacheStoreRecords;cacheStoreRecords=cache_store.objects().filter(' + \
                                    partitioncolstofilter[i] + '=\'' + str(payLoad[partitioncolstofilter[i]]) + '\');'
                    else:
                        if partitioncolstofilter[i] in datecolumns:
                            query = 'global cacheStoreRecords;cacheStoreRecords=cacheStoreRecords.filter(' + \
                                    partitioncolstofilter[i] + \
                                    '=datetime.strptime(\'' + str(
                                (payLoad[partitioncolstofilter[i]])) + '\',\'%Y-%m-%dT%H:%M:%S%z\'));'
                            # return ((["0000"],query))
                        else:
                            query = 'global cacheStoreRecords;cacheStoreRecords=cacheStoreRecords.filter(' + \
                                    partitioncolstofilter[i] + \
                                    '=\'' + str(payLoad[partitioncolstofilter[i]]) + '\');'

                    #return ((["0000"],query))
                    exec(query)
            else:
                for i in range(len(parametercolumns)):
                    for k, v in reversecolumnMapping.items():
                        if v == parametercolumns[i]:
                            parametercolumns[i] = k
                return (("200",("9003", "Invalid Arguments  passed to the API. Valid Arguments are " + ','.join(parametercolumns))))
        if len(cacheStoreRecords) == 0:
            return (("200",{}))  
            #return (("404", ("9007", "cache details could not be found.")))
            #return ((["9007"], "Details could not be found"))
        cacheStore = [row.__json__() for row in cacheStoreRecords]
        #return ((200,cacheStore))
        cacheStore = pd.DataFrame.from_dict((cacheStore), orient='columns')
        cacheStore.fillna('',inplace = True)
        if len(cacheStore)>0:
            for column in datecolumns:
                if column in cacheStore.columns.tolist():
                    cacheStore[column] = pd.to_datetime(cacheStore[column],unit='ns')
                    cacheStore[column] = cacheStore[column].dt.tz_localize('UTC').dt.tz_convert(configvalues.get('apiconfiguration','timeZone')).dt.strftime('%Y-%m-%dT%H:%M:%S.%f%z')
                #return (("404",("0000",cacheStore[column].to_list())))
                cacheStore[column] = [ "" if columnValue == "NaT" else columnValue for columnValue in cacheStore[column].to_list()]
                #return (("404",("0000",cacheStore[column].to_list())))
        cacheStore = cacheStore[validColumns]
        validColumns = {k: v for k, v in columnMapping.items() if k in validColumns}
        cacheStore = cacheStore.rename(columns=(validColumns))
        cacheStore = cacheStore.to_dict(orient='records')
        import unicodedata
        if len(cacheStore) > 0:
            #cacheStore = [ {k : ( unicodedata.normalize('NFC',v) if k == 'value' else v ) for k,v in x.items()} for x in cacheStore]
            cacheStore = [ {k : (v.encode().decode() if k == 'value' else v ) for k,v in x.items()} for x in cacheStore]
        logging.debug('cacheStore-'+str(cacheStore))
        response = {}
        response = response if len(cacheStore) == 0 else cacheStore[0] if len(cacheStore) == 1 else cacheStore

        logging.info("Completed fetching the records")
        gc.collect()
        return ((200,(response)))

    except Exception as e:
        gc.collect()
        logging.error("Error - {} . Line No - {} ".format(str(e), str(sys.exc_info()[-1].tb_lineno)))
        return (("500", "Technical exception"))


"""
This function is used to store the request details in cache_store table

"""
def postData(payLoad, configvalues):
    logging = configlogfile()
    logging.info("Preparing to save records" )

    try:
        dateColumns = json.loads(configvalues.get('apiconfiguration', 'dateColumns', raw=True))
        keysToBeIgnored = json.loads(configvalues.get('apiconfiguration', 'keysToBeIgnored', raw=True))
        columnMapping = json.loads(configvalues.get('apiconfiguration', 'columnMapping', raw=True))
        validColumns = json.loads(configvalues.get('apiconfiguration', 'validKeys', raw=True))
        timeZone = configvalues.get('apiconfiguration', 'timeZone', raw=True)
        logging.debug("arguments-" + str(payLoad))
        logging.debug("validColumns-" + str(validColumns))
        logging.debug("payLoad[before]-" + str(payLoad))

        if payLoad == None or not isinstance(payLoad, dict) or len(payLoad) == 0:
            return (("422", ("9005", "Missing required attribute in payLoad. " + ','.join(validColumns) + ".")))
        if not set(list(payLoad.keys())).issubset(set(list(columnMapping.values()))):
            return (("422", ("9006", "Field rule violation. Payload having non mapped attribute. " + ','.join(list(set(list(payLoad.keys()))-set(columnMapping.values()))) + ".")))
            #return ((["9006"], ("Field rule violation. Payload having non mapped attribute. " + ','.join(list(set(list(payLoad.keys())) - set(columnMapping.values()))) + ".")))
        if not set(validColumns).issubset(set(list(payLoad.keys()))):
            return (("422", ("9005", "Field rule violation or missing required attribute. "+",".join(list(set(validColumns)-set(list(payLoad.keys())))))))
            #return ((["9005"], "Field rule violation or missing required attribute. " + ",".join(
                #list(set(validColumns) - set(list(payLoad.keys()))))))
        logging.info("saving records for cart details " + payLoad['businessId'] + "," +  payLoad['valueType'] + "," + payLoad['key'])

        #if 'value' in payLoad.keys() and not (isinstance(payLoad['value'], list) or isinstance(payLoad['value'],dict)):
        #    return (("422", ("9005", "Field rule violation. value field needs to be in list or dictionary format")))


        data = {}
        data = payLoad.copy()
        if 'createdTime' not in data.keys():
            data['createdTime'] = datetime.strftime(datetime.now(pytz.timezone(timeZone)),'%Y-%m-%dT%H:%M:%S.%f%z') 
        if 'updatedTime' not in data.keys():
            data['updatedTime'] = datetime.strftime(datetime.now(pytz.timezone(timeZone)),'%Y-%m-%dT%H:%M:%S.%f%z')

        data = {k: v for k, v in data.items() if v is not None}  # removes the fields with null values
        data = {k: v for k, v in data.items() if k not in keysToBeIgnored}
        data = {k: v for k, v in data.items() if k in columnMapping.values()}  # remove fields not mapped

        for k, v in columnMapping.items():
            if (v in data):
                data[k] = data.pop(v)

        insertSql = ""
        for k, v in data.items():
            logging.debug(k)
            logging.debug(v)
            logging.debug(type(v));
            if k in dateColumns:  # formats the date columns in the request
                logging.debug("processing date columns")
                try: 
                    datetime.strptime(v,'%Y-%m-%dT%H:%M:%S.%f%z')
                    insertSql = insertSql + k + "=datetime.strptime('" + v + "','%Y-%m-%dT%H:%M:%S.%f%z'),"
                except ValueError:
                    return (("422", ("9005", "Field rule violation. " + k +  " field needs to be in 2020-01-01T00:00:00.000000+0530 format")))
              
            elif type(v) == str:
                if k == 'value':
                    logging.debug('value-' + str(v))
                    
                    insertSql = insertSql + str(k) + "='" + str( str(v))+ "'.encode('utf-8'),"  # convert the string to bytes to store data in blob column

                else:
                    if not v.isalnum():  # logic included to check for alphanumeric string and enclose string with double quotes

                        insertSql = insertSql + str(k) + "=\"\"\"" + v + "\"\"\","
                        #insertSql = insertSql + str(k) + "='" + str(v) + "',"

                    else:
                        insertSql = insertSql + str(k) + "='" + str(v) + "',"
            elif isinstance(v, list) or isinstance(v, dict):
                if k == 'value':
                    insertSql = insertSql + str(k) + "=b'" + json.dumps(v) + "',"
                else:
                    insertSql = insertSql + str(k) + "=" + str(v) + ","

            elif type(b) == blob:
                insertSql = insertSql + str(k) + "=textAsBlob('" + str(v) + "),"
            else:
                insertSql = insertSql + str(k) + "='" + str(v) + "',"

        insertSql = insertSql[0:len(insertSql) - 1]
        insertSql = "global cacheStoreData; cacheStoreData =  cache_store(" + insertSql + ")"
        logging.debug('insertSql')
        logging.debug(insertSql)
        #return ((["0000"],insertSql))
        exec(insertSql)

        cacheStoreData.save()
        gc.collect()
        logging.info("Successfully saved the cache details")
        return (("200","Successfully saved the cache details"))
        return (("200", {k: v for k, v in payLoad.items()}))
    except Exception as e:
        gc.collect()
        logging.error("Error - {} . Line No - {} ".format(str(e), str(sys.exc_info()[-1].tb_lineno)))
        return (("500", "Technical exception"))


Editor is loading...