我正在使用下面的代码滚动到YouTube页面的底部,它很有效.我的问题是,在站点被向下滚动到底部后,我如何才能安慰.log已经到达底部?先谢谢你.
note:这个解决方案应该适用于youtube.com.我已经try 获取文档高度,并将其与滚动高度进行比较,但不起作用!
const puppeteer = require('puppeteer');
let thumbArr = []
const scrapeInfiniteScrollItems = async(page) => {
while (true) {
const previousHeight = await page.evaluate(
"document.querySelector('ytd-app').scrollHeight"
);
await page.evaluate(() => {
const youtubeScrollHeight =
document.querySelector("ytd-app").scrollHeight;
window.scrollTo(0, youtubeScrollHeight);
});
await page.waitForFunction(
`document.querySelector('ytd-app').scrollHeight > ${previousHeight}`, {
timeout: 0
}
);
const thumbnailLength = (await page.$$('ytd-grid-video-renderer')).length
//this logs the amount of thumbnails every loop but once bottom scroll has been reached it stops logging (obviously) but the question is how am I supposed to compare the last amount of thumbnail's found with total thumbnails once the loop has stopped running. Take a look below to better understand my question.
thumbArr.push(thumbnailLength)
if (thumbnailLength == thumbArr.at(-1)) {
console.log('bottom has been reached')
}
await page.waitForTimeout(1000)
}
};
(async() => {
const browser = await puppeteer.launch({
headless: false
});
const page = await browser.newPage();
await page.goto('https://www.youtube.com', {
waitUntil: 'networkidle2',
});
await scrapeInfiniteScrollItems(page)
})();
UPDATE:个
let clientHeightArr = []
let clientHeightArrTracker = []
const scrapeInfiniteScrollItems = async(browser, page) => {
var infiniteScrollTrackerInterval = setInterval(async() => {
clientHeightArrTracker.push(clientHeightArr.length)
if (clientHeightArrTracker.some((e, i, arr) => arr.indexOf(e) !== i) == true) {
clearInterval(infiniteScrollTrackerInterval)
console.log('Bottom is reached')
//causes error "ProtocolError: Protocol error (Runtime.callFunctionOn): Target closed."
await browser.close()
}
}, 2000)
while (true) {
const previousHeight = await page.evaluate(
"document.querySelector('ytd-app').scrollHeight"
);
await page.evaluate(() => {
const youtubeScrollHeight =
document.querySelector("ytd-app").scrollHeight;
window.scrollTo(0, youtubeScrollHeight);
});
await page.waitForFunction(
`document.querySelector('ytd-app').scrollHeight > ${previousHeight}`, {
timeout: 0
},
);
const clientHeight = await page.$$eval("ytd-app", el => el.map(x => x.clientHeight));
clientHeightArr.push(clientHeight[0])
await page.waitForTimeout(1000)
}
};
(async() => {
const browser = await puppeteer.launch({
headless: false
});
const page = await browser.newPage();
await page.goto('https://www.youtube.com/c/mkbhd/videos', {
waitUntil: 'networkidle2',
});
await scrapeInfiniteScrollItems(browser, page)
})();