scrape.js
Web scraping program to scrape tweets from an account then scroll down and repeat...unknown
javascript
3 years ago
1.1 kB
30
Indexable
const puppeteer = require('puppeteer-extra');
(async () => {
const browser = await puppeteer.launch({
executablePath: '/usr/bin/brave',
headless: false,
defaultViewport: null,
ignoreDefaultArgs: ["--disable-extensions"],
args: ["--start-maximized", "--no-sandbox", "--disable-setuid-sandbox"],
});
const page = await browser.newPage();
await page.goto('https://twitter.com/CNN', { waitUntil: 'networkidle2' });
await autoScroll(page);
const results = await page.$$eval('article div[lang]', (tweets) => tweets.map((tweet) => tweet.textContent));
console.log(results);
browser.close();
})();
async function autoScroll(page) {
await page.evaluate(async () => {
await new Promise((resolve) => {
var totalHeight = 0;
var distance = 100;
var timer = setInterval(() => {
var scrollHeight = document.body.scrollHeight;
window.scrollBy(0, distance);
totalHeight += distance;
if (totalHeight >= scrollHeight - window.innerHeight) {
clearInterval(timer);
resolve();
}
}, 400);
});
});
}
Editor is loading...