Untitled

import { BaseCommand } from "@adonisjs/core/build/standalone";
import Document from 'App/Models/Document'
import Database from '@ioc:Adonis/Lucid/Database'
import JobListing from 'App/Models/JobListing'
import LegalEntity from 'App/Models/LegalEntity'

export default class Scraper extends BaseCommand {
    public static commandName = "scraper";
    public static description = "";
    public static settings = {
        loadApp: true,
        stayAlive: false
    };

    // 1- save in document
    // 2- joblistingIngest from googleJobDocument --> create the joblistings

    public async run() {
        const { default: parallel } = await import("@ioc:Library/parallel");
        const { default: Google } = await import("Config/google");

        console.log(`getting hrefs...`)
        const hrefs = await Google.getHrefs("web developer Rome jobs");

        console.log(`scraping hrefs...`)
        let document = await Google.jobsScraper(hrefs.response.hrefs) 
        console.log(document.response.listings)
        console.log(document.response.errors)

        for (const datum of document.response.listings) {
            if (!datum.companyProfileHref) continue
            const company = await LegalEntity.firstOrCreate(
                { linkedinUrl: datum.companyProfileHref },
                {
                    type: 'company',
                    name: datum.companyName,
                },
            )

            const jobList = await JobListing.ingestFromDocumentJobListing(datum, document, company)
            console.log(jobList)
            break
        }


    }
}