Untitled

# ...

batch_size = 10000

for _, row in df.iterrows():

  eprm_table_name = row['eprm_table_name']

  source_query = ...
  
  source_df = pd_read_sql(source_query, con=connection_source_ext)

  batches = [source_df[i:i+batch_size] for i in range(0, len(source_df), batch_size)]

  for batch in batches:
    
    if 'updated_by' in batch:
      batch['updated_by'] = replicationJobId

    batch.to_sql(eprm_table_name, con=connection_ext, if_exists='append', index=False, method='multi')

    if len(batch) % batch_size == 0:
      connection_ext.commit()

# Update status  

try:

  query_update = ...
  connection_source.execute(query_update)

  query_ppm_update = ... 
  connection_source.execute(query_ppm_update)

except:

  # handle error
  pass

finally:

  if connection_ext:
    connection_ext.close()

  if connection_source:
    connection_source.close()

  # etc