Untitled
plain_text
a month ago
918 B
1
Indexable
Never
_sql = f""" SELECT {cust_id}, date(end_observation) as end_observation, value, token FROM ( SELECT {cust_id}, end_observation, {', '.join([f"CAST({x} AS FLOAT64) AS {x}" for x in _cols2process])} FROM {table_id} ) UNPIVOT(value FOR token IN ({', '.join(_cols2process)})) WHERE end_observation='{end_observation}' """ _df = ( spark.read.format("bigquery") .option("maxParallelism", 666) .load(_sql) .withColumn( "token_sfx", F.concat(F.lit(table_api), F.lit("__"), F.col("token")), ) .withColumn("source", F.lit(table_api)) .select( "cust_id", "end_observation", "value", "token_sfx", "source" ) )