Untitled
unknown
plain_text
a year ago
6.7 kB
2
Indexable
Never
"""
#######################################################################################################################################################################
API Name : cacheStore
Description : This api stores and fetches the cache details stored in the cassandra table cache_store table in hobs_ods_staging keyspace.
Parameters are required to fetch the cache details. Valid parameters are valueType and key.
Columns that need to be returned in the GET response can be configured in validColumns property in config/dataservice/cacheStore.config file.
Columns that need to be passed in the POST request can be configured in validKeys property in config/dataservice/cacheStore.config file.
Revision History
----------------
Version Date Author Description
0.1 29-Nov-2021 senthilCR Initial version of the API to store and fetch cache details in the cache_store
0.2 09-Feb-2022 senthilCR Modified the script to handle value input as a dictionary or list of dictionaries
#######################################################################################################################################################################
"""
from models.cache_store import cache_store
from models.cache_store_by_username import cache_store_by_username
from models.cache_store_by_date import cache_store_by_date
from models.indexes import indexes
from models.columns import columns as tablecolumns
from datetime import datetime, timedelta, timezone
from dateutil.relativedelta import relativedelta
from pandas.io.json import json_normalize
import json
from config.logger import configlogfile
from flask import jsonify, make_response
import pandas as pd
import sys
import gc
import uuid
import pytz
"""
This function is used to fetch cache details based on valueType and key
"""
def getData(payLoad, configvalues):
from config.logger import configlogfile
logging = configlogfile()
logging.info("Started to fetch the records for " + str(payLoad))
cacheStore = []
try:
payLoad = payLoad.to_dict()
columnMapping = json.loads(configvalues.get('apiconfiguration', 'columnMapping', raw=True))
reversecolumnMapping = {y: x for x, y in columnMapping.items()}
validColumns = json.loads(configvalues.get('apiconfiguration', 'validColumns', raw=True))
datecolumns = json.loads(configvalues.get('apiconfiguration', 'dateColumns', raw=True))
datecolumns.append('createdtime') # Add 'createdtime' column
validArguments = json.loads(configvalues.get('apiconfiguration', 'validArguments', raw=True))
logging.debug('Arguments -' + str(payLoad))
for k, v in reversecolumnMapping.items():
if k in list(payLoad.keys()):
payLoad[v] = payLoad.pop(k)
recs = indexes.objects().filter(keyspace_name=cache_store.__keyspace__)
recs = recs.filter(table_name='cache_store')
indexedcolumns = [row.options['target'] for row in recs]
recs = tablecolumns.objects().filter(keyspace_name=cache_store.__keyspace__)
recs = recs.filter(table_name='cache_store')
partitioncolumns = [row.column_name for row in recs if row.kind in ["partition_key"]]
partitioncolumns = partitioncolumns + [row.column_name for row in recs if row.kind in ["primary_key", "clustering"]]
parametercolumns = partitioncolumns + indexedcolumns
if not bool(payLoad):
return ("200", "Parameters need to be passed to fetch values from the cache store")
if set(list(payLoad.keys())).issubset(validArguments):
if 'valueType' in payLoad.keys() and 'key' in payLoad.keys():
if len(set(partitioncolumns).intersection(set(payLoad.keys()))) > 0:
df = pd.DataFrame(list(cache_store.objects().filter(**payLoad).values()))
if not df.empty:
df = df.sort_values(by='createdtime', ascending=False)
columns = json_normalize(json.loads(df.to_json(orient='records')))
columns = columns.drop(['createdtime'], axis=1)
columns = columns.loc[:, ~columns.columns.duplicated()]
columns = columns[validColumns]
columns = columns.where(pd.notnull(columns), None)
cacheStore = columns.to_dict(orient='records')
else:
return ("404", "No records found in cache store")
if len(cacheStore) > 0:
logging.info("Successfully fetched the records from cache store for " + str(payLoad))
return ("200", cacheStore)
else:
return ("404", "No records found in cache store")
else:
return ("400", "Parameter passed should be part of partition columns")
else:
return ("400", "Invalid parameters passed to fetch values from cache store")
else:
return ("400", "Invalid parameters passed to fetch values from cache store")
except Exception as e:
logging.exception("Data service error -" + str(e))
return ("500", "Data service error -" + str(e))
"""
This function is used to store cache details in the cache_store table
"""
def postData(payLoad, configvalues):
from config.logger import configlogfile
logging = configlogfile()
logging.info("Started to store the records for " + str(payLoad))
try:
columnMapping = json.loads(configvalues.get('apiconfiguration', 'columnMapping', raw=True))
validKeys = json.loads(configvalues.get('apiconfiguration', 'validKeys', raw=True))
payLoad = payLoad.to_dict()
for k, v in columnMapping.items():
if k in list(payLoad.keys()):
payLoad[v] = payLoad.pop(k)
if set(list(payLoad.keys())).issubset(validKeys):
logging.debug('Arguments -' + str(payLoad))
payLoad['createdtime'] = datetime.now(timezone.utc).isoformat()
recs = cache_store(**payLoad).save()
logging.info("Successfully stored the records in cache store for " + str(payLoad))
return ("200", "Successfully stored the record in cache store")
else:
return ("400", "Invalid parameters passed to store values in cache store")
except Exception as e:
logging.exception("Data service error -" + str(e))
return ("500", "Data service error -" + str(e))