Untitled

const start = Date.now();
const fs = require("fs");
const fileContents = fs.readFileSync("cui.txt", "utf8");
const lines = fileContents.split("\n");
const nCUI = lines.length;

function sleep(milliseconds) {
  const date = Date.now();
  let currentDate = null;
  do {
    currentDate = Date.now();
  } while (currentDate - date < milliseconds);
}

async function fetchData() {
  let output = {};

  for (let i = 0; i < nCUI; i++) {
    if (!output[lines[i]]) {
      output[lines[i]] = {};
    }
    for (let j = 0; j < 5; j++) {
      console.log(
        `DONE: (${i}/${nCUI})\nCURRENT CUI: ${lines[i]}\nPROGRESS: (${
          j + 1
        }/5)\nELAPSED: ${(Date.now() - start) / 1000}s`
      );

      const year = 2017 + j;
      const url = 'probably dont wanna show this'

      try {
        const data = await fetch(url).then((response) => response.json());
        if (!output[lines[i]][year]) {
          output[lines[i]][year] = [];
        }
        output[lines[i]][year].push(data);
      } catch (error) {
        console.error("ERROR: " + error + "CONTINUING");
      }

      sleep(1000)
    }
  }

  fs.writeFileSync("output.json", JSON.stringify(output, null, 2), {
    flag: "w",
  });
}
fetchData();