Почему код не фильтрует данные?
script.js
const puppeteer = require('puppeteer');
const {
promisify
} = require('util');
const credentials = {
login: '[email protected]', //! тут должен быть записан твой логин
password: 'alex07112005' //! а здесь пароль
};
(async () => {
try {
await spy();
process.exit(0);
} catch (err) {
console.error(err);
process.exit(1);
}
})();
async function spy() {
const browser = await puppeteer.launch({
headless: true
});
const page = await browser.newPage();
await page.goto(
'https://etfdb.com/members/login/', {
waitUntil: 'networkidle2',
timeout: 0
}
);
await page.click('#user_login');
await page.keyboard.type(credentials.login);
await page.click('#password');
await page.keyboard.type(credentials.password);
await page.click('#login-button');
await promisify(setTimeout)(2000);
const etfData = await page.evaluate(async () => {
const sleep = (timeout = 500) => new Promise((r) => setTimeout(r, timeout));
let resultData = [];
const fetchBody = {
structure: ['ETF'],
dividend_frequency: ['Monthly'],
only: ['meta', 'data', 'count']
};
const fetchOptions = {
method: 'post',
headers: {
'Content-Type': 'application/json',
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Methods': 'GET, POST, PUT, DELETE, OPTIONS'
},
};
for (let i = 1, curr = 1; i <= curr; i++) {
try {
fetchBody.page = i;
const res = await fetch(
'https://etfdb.com/api/screener/', {
...fetchOptions,
body: JSON.stringify(fetchBody)
}
);
const json = await res.json();
if (json) {
if (json.data && Array.isArray(json.data)) {
resultData = resultData.concat(json.data);
}
if (json.meta && 'total_pages' in json.meta && +json.meta.total_pages > 0) {
curr = +json.meta.total_pages;
}
}
await sleep(); // на всякий случай, вдруг там ограничение по количеству запросов
} catch (err) {
console.error(err);
}
}
return resultData;
});
await browser.close();
const filteringOverallRating = ['A+', 'A', 'A-', 'B+'];
const etfABData = etfData.filter(({
overall_rating
}) => filteringOverallRating.includes(overall_rating));
console.log(JSON.stringify(etfABData, null, 2));
console.log(`total items = ${etfData.length}, filtered items = ${etfABData.length}`);
return etfABData;
}
package.json
{
"name": "parser",
"version": "1.0.0",
"main": "script.js",
"scripts": {
"start": "node script.js",
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"license": "ISC",
"dependencies": {
"puppeteer": "^5.5.0",
"puppeteer-core": "^5.5.0"
},
"devDependencies": {},
"description": ""
}
терминал
MacBook-Pro-Aleksandr:ETF 4 aleksandr$ npm i
> [email protected] install /Users/aleksandr/Downloads/ETF 4/node_modules/puppeteer
> node install.js
Downloading Chromium r818858 - 132.4 Mb [====================] 100% 0.0s
Chromium (818858) downloaded to /Users/aleksandr/Downloads/ETF 4/node_modules/puppeteer/.local-chromium/mac-818858
npm notice created a lockfile as package-lock.json. You should commit this file.
npm WARN [email protected] No description
npm WARN [email protected] No repository field.
added 55 packages from 75 contributors and audited 55 packages in 25.656s
8 packages are looking for funding
run `npm fund` for details
found 0 vulnerabilities
MacBook-Pro-Aleksandr:ETF 4 aleksandr$ npm i puppeteer
> [email protected] install /Users/aleksandr/Downloads/ETF 4/node_modules/puppeteer
> node install.js
Downloading Chromium r818858 - 132.4 Mb [====================] 100% 0.0s
nChromium (818858) downloaded to /Users/aleksandr/Downloads/ETF 4/node_modules/puppeteer/.local-chromium/mac-818858
pnpm WARN [email protected] No description
npm WARN [email protected] No repository field.
+ [email protected]
updated 1 package and audited 55 packages in 23.674s
8 packages are looking for funding
run `npm fund` for details
found 0 vulnerabilities
MacBook-Pro-Aleksandr:ETF 4 aleksandr$ npm start
> [email protected] start /Users/aleksandr/Downloads/ETF 4
> node script.js
[]
total items = 415, filtered items = 0
MacBook-Pro-Aleksandr:ETF 4 aleksandr$