Почему код не фильтрует элементы массива на puppeteer?
const puppeteer = require('puppeteer');
const {
promisify
} = require('util');
const credentials = {
login: '[email protected]', //! тут должен быть записан твой логин
password: 'alex07112005' //! а здесь пароль
};
(async () => {
try {
await spy();
process.exit(0);
} catch (err) {
console.error(err);
process.exit(1);
}
})();
async function spy() {
const browser = await puppeteer.launch({
headless: true
});
const page = await browser.newPage();
await page.goto(
'https://etfdb.com/members/login/', {
waitUntil: 'networkidle2',
timeout: 0
}
);
await page.click('#user_login');
await page.keyboard.type(credentials.login);
await page.click('#password');
await page.keyboard.type(credentials.password);
await page.click('#login-button');
await promisify(setTimeout)(2000);
const etfData = await page.evaluate(async () => {
const sleep = (timeout = 500) => new Promise((r) => setTimeout(r, timeout));
let resultData = [];
//! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!
const structure = 'ETF',
dividend = 'Monthly';
//! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!
const fetchBody = {
structure: [structure],
dividend_frequency: [dividend],
only: ['meta', 'data', 'count']
};
const fetchOptions = {
method: 'post',
headers: {
'Content-Type': 'application/json',
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Methods': 'GET, POST, PUT, DELETE, OPTIONS'
},
};
for (let i = 1, curr = 1; i <= curr; i++) {
try {
fetchBody.page = i;
const res = await fetch(
'https://etfdb.com/api/screener/', {
...fetchOptions,
body: JSON.stringify(fetchBody)
}
);
const json = await res.json();
if (json) {
if (json.data && Array.isArray(json.data)) {
resultData = resultData.concat(json.data);
}
if (json.meta && 'total_pages' in json.meta && +json.meta.total_pages > 0) {
curr = +json.meta.total_pages;
}
}
await sleep(); // на всякий случай, вдруг там ограничение по количеству запросов
} catch (err) {
console.error(err);
}
}
return resultData;
});
await browser.close();
//! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
const filteringOverallRating = ['A+', 'B+']; //! тут записываешь рейтинги в верхнем регистре
//! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
const etfABData = etfData.filter(({
overall_rating
}) => filteringOverallRating.includes(overall_rating));
Object.keys(etfABData).forEach(function (i) {
etfABData[i].symbol.text = "_" + etfABData[i].symbol.text + ",";
console.log(etfABData[i].symbol.text)
})
console.log(`total items = ${etfData.length}, filtered items = ${etfABData.length}`);
}
в терминале установил npm и puppeteer вот что выдает
MacBook-Pro-Aleksandr:ETF 3 2 aleksandr$ npm i
npm WARN [email protected] No description
npm WARN [email protected] No repository field.
audited 56 packages in 0.605s
8 packages are looking for funding
run `npm fund` for details
found 0 vulnerabilities
MacBook-Pro-Aleksandr:ETF 3 2 aleksandr$ npm start
> [email protected] start /Users/aleksandr/Downloads/ETF 3 2
> node script.js
total items = 415, filtered items = 0
MacBook-Pro-Aleksandr:ETF 3 2 aleksandr$