Untitled

const start = Date.now();
const fs = require("fs");
const fileContents = fs.readFileSync("cui.txt", "utf8");
const lines = fileContents.split("\n");
const nCUI = lines.length;

function sleep(milliseconds) {
  const date = Date.now();
  let currentDate = null;
  do {
    currentDate = Date.now();
  } while (currentDate - date < milliseconds);
}

async function fetchData() {
  let output = {};

  for (let i = 0; i < nCUI; i++) {
    if (!output[lines[i]]) {
      output[lines[i]] = {};
    }
    for (let j = 0; j < 5; j++) {
      console.log(
        `DONE: (${i}/${nCUI})\nCURRENT CUI: ${lines[i]}\nPROGRESS: (${
          j + 1
        }/5)\nELAPSED: ${(Date.now() - start) / 1000}s`
      );

      const year = 2017 + j;
      const url = 'probably dont wanna show this'

      try {
        const data = await fetch(url).then((response) => response.json());
        if (!output[lines[i]][year]) {
          output[lines[i]][year] = [];
        }
        output[lines[i]][year].push(data);
      } catch (error) {
        console.error("ERROR: " + error + "CONTINUING");
      }

      sleep(1000)
    }
  }

  const reformattedData = reformatData(output);
  fs.writeFileSync("output.json", JSON.stringify(reformattedData, null, 2), {
    flag: "w",
  });
}

function reformatData(data) {
  const reformattedData = {};

  Object.keys(data).forEach((key) => {
    reformattedData[key] = {};
    Object.keys(data[key]).forEach((year) => {
      reformattedData[key][year] = data[key][year].map((item) => {
        const indicators = {};
        item.i.forEach((indicator) => {
          indicators[indicator.val_den_indicator] = indicator.val_indicator;
        });
        const { i, ...rest } = item;
        return {
          ...rest,
          ...indicators,
        };
      });
    });
  });

  return reformattedData;
}

fetchData();