Untitled

<?php

class Crawler extends application
{

    function DSTruyenMoi($page, $site)
    {
        $run = new $site;
        $data = $run->DSTruyen($page, 'TruyenMoi');

        $inputData = [];
        foreach ($data as $key => $manga) {
            $CrawlerID = $this->isExistCrawlers($manga['name'], 'DSTruyenMoi');
            $MangaID = $this->isExistManga($manga['name']);

            if (!$CrawlerID && !$MangaID) {
                $inputData[] = ["name" => $manga['name'], "url" => $manga['url'], "type" => 'DSTruyenMoi', "site" => $site];
            } else {
                print "$manga[name] already exist \n";
            }
            flush();
            ob_flush();
        }

        if (empty($inputData)) {
            return print 'No new manga found';
        }

        $ids = $this->DB->insertMulti('crawlers', $inputData);
        if (!$ids) {
            echo "an error occurred while adding url: " . $chapter['url'] . $this->DB->getLastError();
            LOG::writeError('An error occurred while adding url', "Manga New List");
        } else {
            foreach ($inputData as $manga) {
                print '[OK]' . $manga['name'] . "\n";
            }
        }
    }

    function DSCapNhat($page, $site)
    {
        $run = new $site;
        $data = $run->DSTruyen($page, 'MoiCapNhat');

        $inputData = [];
        foreach ($data as $key => $manga) {
            $CrawlerID = $this->isExistCrawlers($manga['name'], 'DSCapNhat');
            $MangaID = $this->isExistManga($manga['name']);

            //  Chỉ lấy truyện chưa có trong crawlers và đã có trong mangas
            if (!$CrawlerID && $MangaID) {
                $inputData[] = ["name" => $manga['name'], "url" => $manga['url'], "type" => 'DSCapNhat', "site" => $site];

                print '[OK]' . $manga['name'] . "\n";
                flush();
                ob_flush();
            }
        }
        if (!empty($inputData[0])) {

            if (!$this->DB->insertMulti('crawlers', $inputData)) {
                echo "an error occurred while adding url: " . $chapter['url'] . $this->DB->getLastError();
                LOG::writeError('An error occurred while adding url', "Manga Update List");
            }
            echo 'All have been queued...';
        } else {
            echo "Can't Find Manga To Update.";
        }

        return true;

    }

    function ThongTinTruyen($site)
    {
        $DBCrawlers = $this->getCrawlerItem($site, 'DSTruyenMoi');

        if (empty($DBCrawlers['url'])) {
            print 'New Manga List is Empty!';
            return false;
        }

        $this->removeCrawlers($DBCrawlers['id']);


        $run = new $site;
        $manga = $run->ThongTinTruyen($DBCrawlers['url']);

        if ($DBCrawlers['name'] == trim($DBCrawlers['name']) && strpos($DBCrawlers['name'], ' ') !== false) {
            $manga['name'] = $DBCrawlers['name'];
        }

        $MangaID = $this->isExistManga($manga['name']);
        if (!empty($MangaID)) {
            print "$manga[name] already exist!";
        } elseif (!$this->stopCrawlerbyCategory($manga['list_categories'])) {
            print "$manga[name] Contains forbidden genre!";
        } else {

            $MangaID = $this->CreateManga($manga);
            $this->DB->getLastError();

            if ($MangaID) {
                if (!empty($manga['list_chapter'][0])) {
                    $oldChapters = $this->listChapters($MangaID);
                    $newChapters = $manga['list_chapter'];

                    $manga['list_chapter'] = $this->newChapters($oldChapters, $newChapters);

                    foreach ($manga['list_chapter'] as $chapter) {
                        $crawler_chapter_id = $this->DB->insert('crawler_chapter', ["is_crawling" => 0, "url" => $chapter['url'], "name" => $chapter['name'], "manga_id" => $MangaID, "chapter_id" => null, "site" => $site, "created_at" => $this->DB->now(), "updated_at" => $this->DB->now()]);

                        if (!$crawler_chapter_id) {
                            echo "an error occurred while adding url: " . $chapter['url'] . $this->DB->getLastError();
                            LOG::writeError('An error occurred while adding url', "Manga Info");
                        }
                    }

                }
            }

            echo "[OK] $manga[name]";

            LOG::write("$manga[name]", $site, 'manga-details');
        }

        return true;
    }

    function CapNhatChuongMoi($site)
    {
        $DBCrawlers = $this->getCrawlerItem($site, 'DSCapNhat');

        if (empty($DBCrawlers)) {
            print 'Empty Update List!';
            return false;
        }

        $this->removeCrawlers($DBCrawlers['id']);

        $run = new $site;
        $manga = $run->ThongTinTruyen($DBCrawlers['url']);

        $MangaID = $this->isExistManga($DBCrawlers['name']);
        if (empty($MangaID)) {
            print "$manga[name] does not exist!";
            return true;
        } else {

            $this->UpdateMangaStatus($MangaID, $manga['status']);


            if (!empty($manga['list_chapter'][0])) {
                $oldChapters = $this->listChapters($MangaID);
                $oldChapters2 = $this->listChaptersCrawler($MangaID);

                $all_old_chapter = array_merge($oldChapters, $oldChapters2);

                $crawlerChapters = $manga['list_chapter'];

                $newChapters = $this->newChapters($all_old_chapter, $crawlerChapters);

                if (empty($newChapters)) {
                    echo "no new chapters";
                    return true;
                }

                foreach ($newChapters as $chapter) {
                    $crawler_chapter_id = $this->DB->insert('crawler_chapter', ["is_crawling" => 0, "url" => $chapter['url'], "name" => $chapter['name'], "manga_id" => $MangaID, "chapter_id" => null, "site" => $site, "created_at" => $this->DB->now(), "updated_at" => $this->DB->now()]);

                    if (!$crawler_chapter_id) {
                        echo "an error occurred while adding url: " . $chapter['url'] . $this->DB->getLastError();
                        LOG::writeError('An error occurred while adding url', "Manga Info");
                    }
                }
            }

            echo "Updated $manga[name]";
            LOG::write("$manga[name]", $site, 'manga-update-chap');
        }

        return true;
    }

    function ThongTinChuong($site)
    {
        $this->DB->where("site", $site);
        $this->DB->where("is_crawling", 0);
        $this->DB->where("chapter_id", NULL, "IS");
        $this->DB->orderBy("id", "asc");

        $db_crawler_chapter = $this->DB->getOne('crawler_chapter');

        if (empty($db_crawler_chapter)) {
            echo "All chapters have been saved";
            return false;
        }

        $this->DB->where('id', $db_crawler_chapter['id']);
        $this->DB->update('crawler_chapter', ['is_crawling' => 1]);

        $this->DB->where('chapter_name', $db_crawler_chapter['name']);
        $this->DB->where('post_id', $db_crawler_chapter['manga_id']);
        if ($this->DB->getOne('manga_chapters', 'chapter_id')) { // CHECK CHAP IS EXIST
            $this->DB->where('id', $db_crawler_chapter['id']);
            $this->DB->update('crawler_chapter', ['is_crawling' => 1]);
        }


        $run = new $site;
        $chapter = $run->ThongTinChuong($db_crawler_chapter['url']);

        echo "ID--$db_crawler_chapter[id] || $chapter[info] \n";
        flush();
        ob_flush();
        if (!empty($chapter['server1'])) {


            $content = array_unique(array_filter($chapter['server1']));
            $respone_upload = $this->RawArrayConvert($content);


            $ChapterID = $this->CreateChapter(['manga_id' => $db_crawler_chapter['manga_id'], 'chapter_name' => $db_crawler_chapter['name'], 'storage_in_use' => 'proxy']);

            $this->CreateChapterData($ChapterID, $respone_upload, 'proxy');

            $this->UpdateMangaTime($db_crawler_chapter['manga_id']);
            // $this->UpdatePostStatus($db_crawler_chapter['manga_id'], 'publish');

            $this->UpdateCrawlerStatus($db_crawler_chapter['id'], $ChapterID);

            LOG::write("ChapterID: $ChapterID - $chapter[info]", $site, 'info-update');

        }
        return true;
    }

    function UpAnhTruyen($site = null)
    {
        if ($site) {
            $this->DB->where("site", $site);
        }

        $this->DB->where("is_crawling", 0);
        $this->DB->where("chapter_id", NULL, "IS NOT");
        $this->DB->orderBy("id", "asc");

        $db_crawler_chapter = $this->DB->getOne('crawler_chapter');

        if (empty($db_crawler_chapter)) {
            echo "All images saved";
            return false;
        }

        $this->removeChapterCrawlers($db_crawler_chapter['id']);

        echo "Start scanning chapters: $db_crawler_chapter[url] \n";
        flush();
        ob_flush();

        $site = $db_crawler_chapter['site'];
        $run = new $site;

        $chapter = $run->ThongTinChuong($db_crawler_chapter['url'], false);
        $content = array_unique(array_filter($chapter['server1']));
        $manga_id = $db_crawler_chapter['manga_id'];
        $chapter_id = $db_crawler_chapter['chapter_id'];
        $directory = trim(IMAGE_SAVE_PATH, '/') . "/$manga_id/$chapter_id/";
        $public_url = trim(IMAGE_CDN_URL, '/') . "/$manga_id/$chapter_id/";

        $UpAnh = new UpAnh();

        if (SAVE_TO_LOCAL) {
        }

        if (SAVE_TO_FTP) {
            (new ftp())->mkdir($directory);
            $respone_upload = $this->RawArrayConvert($UpAnh->MultiUploadFtp($content, $directory, $public_url, $run->options));

            $this->CreateChapterData($db_crawler_chapter['chapter_id'], $respone_upload, 'ftp_storage');
        }


        $this->DB->where('chapter_id', $db_crawler_chapter['chapter_id']);
        $this->DB->update('manga_chapters', ['storage_in_use' => DEFAULT_STORAGE]);

        LOG::write("ChapterID: $chapter_id - MangaID: $manga_id", $site, 'chapter-upload-storage');

        return true;
    }
}