const { chromium, firefox, webkit } = require('playwright') const axios = require('axios').default const fs = require('fs') ;(async () => { let response = await axios.post('https://www.sif.com/api/user/login?_t=' + new Date().getTime(), { phone: '18390881087', password: 'Gaole1401' }) let token = response.headers['authorization'] axios.defaults.headers.common['authorization'] = token console.log(token) const browser = await chromium.launch({ headless: true }) // Or 'firefox' or 'webkit'. const page = await browser.newPage() await page.goto('https://www.amazon.com/gp/bestsellers/pet-supplies/2975292011/ref=pd_zg_hrsr_pet-supplies') // await page.evaluate(async () => { // await new Promise((resolve, reject) => { // let i = setInterval(() => { // console.log( // (window.innerHeight, document.documentElement.scrollTop, document.documentElement.scrollHeight) // ) // if (window.innerHeight + document.documentElement.scrollTop > document.documentElement.scrollHeight) { // clearInterval(i) // resolve() // } else { // window.scrollBy(0, 200) // } // }, 100) // }) // }) let bottom = 1 while (bottom > 0) { await page.evaluate(() => { window.scrollBy(0, 200) }) await page.waitForTimeout(100) bottom = await page.evaluate(() => { return document.documentElement.scrollHeight - (window.innerHeight + document.documentElement.scrollTop) }) console.log(bottom) } await page.waitForTimeout(2000) //find a tag using page.locator const items = await page.locator('.p13n-desktop-grid .a-column').all() console.log(items.length) const res = [] for (let item of items) { let atags = await item.locator('a').all() if (atags && atags[1]) { let text = await atags[1].innerText() let href = await atags[1].getAttribute('href') let match = /\/dp\/(.*?)\//.exec(href) let asin if (match && match[1]) { asin = match[1] } let price = (await item.locator('.a-row').last().allInnerTexts()).join() let sif = await axios.get('https://www.sif.com/api/search/asinFlowOverview', { params: { country: 'US', asin, _t: new Date().getTime() } }) res.push({ text, href, price, asin, data: sif.data.data }) } } fs.writeFileSync('res.json', JSON.stringify(res, null, 4)) await browser.close() })()