apex_scraper
unknown
python
a year ago
3.2 kB
65
Indexable
import asyncio import os import sys from datetime import datetime from pathlib import Path from loguru import logger from selenium import webdriver from selenium.webdriver.chrome.options import Options from selenium.webdriver.chrome.service import Service from selenium.webdriver.common.by import By from webdriver_manager.chrome import ChromeDriverManager class ApexPayoutScraper(object): def __init__(self): self.worker_name = str(os.path.basename(__file__)).split('.')[0] options = Options() options.headless = True self.driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options) self.driver.implicitly_wait(10) self.checkIntervalInSecs = 4 * 60 async def __check(self): try: scrapeTillDateString = "Jan 20, 2024" page = 1 csvFile = open("apex_payouts.csv", "w") while True: print("Scraping page {}".format(page)) self.driver.get("https://www.apextraderfunding.com/payouts.html?p={}".format(page)) await asyncio.sleep(1) els = self.driver.find_elements(By.CSS_SELECTOR, "#overview > div.divTable.PA > div > div") for el in els: dateString = el.find_element(By.CSS_SELECTOR, "div.divTableCell:nth-child(1)") if dateString.text == "Date": continue formatDateString = datetime.strptime(dateString.text, "%b %d, %Y").strftime("%Y-%m-%d") nameString = el.find_element(By.CSS_SELECTOR, "div.divTableCell:nth-child(2)") payoutString = el.find_element(By.CSS_SELECTOR, "div.divTableCell:nth-child(3)") if dateString.text == scrapeTillDateString: csvFile.close() raise KeyboardInterrupt("Done scraping!") csvFile.write("{}\t{}\t{}\n".format(formatDateString, nameString.text, payoutString.text.replace("$", ""))) page += 1 except Exception as e: raise e def main(self): logger.info("Scraping payouts from Apex.. {}", self.worker_name) print("Starting {}".format(self.worker_name)) loop = asyncio.get_event_loop() tasks = [ loop.create_task(self.__check()) ] try: loop.run_forever() except KeyboardInterrupt or Exception as ex: logger.warning("{}", ex) finally: try: self.driver.close() self.driver.quit() except Exception as _: self.driver.quit() list(map(lambda task: task.cancel(), tasks)) logger.info("Exiting. Bye!") sys.exit(0) if __name__ == '__main__': worker_name = str(os.path.basename(__file__)).split('.')[0] log_path = str(Path.home().joinpath("TradeBot/" + worker_name)) logger.add(log_path + "/file_{time}.log", format="{time} {level} {name} {message}", level="INFO", rotation="1 day") ApexPayoutScraper().main()
Editor is loading...
Leave a Comment