nodejs/exceljs. Из-за чего происходит утечка памяти?
После выгрузки 100 000 товаров скрипт останавливается и выдает ошибку, что памяти не хватает. Использование памяти растет на глазах. Почему так, если все сделано через потоки?
async getAll() {
var th = this;
var categories = require("./cat_test");
var numProducts = 0;
for (var id in categories) {
this.clientDb = await DB.connect();
this.db = this.clientDb.db("ozon");
this.collection = this.db.collection("stat");
var row = 2;
var mainCategoryName = categories[id].name; //имя главной категории
var c1 = categories[id];
var filename = "xlsx/all/"+mainCategoryName+"_" + new Date().format("dd-MM-Y") + ".xlsx";
const options = {
writer: 'stream',
filename: filename,
useStyles: true,
useSharedStrings: true
};
var wb = new WorkbookWriter(options);
var worksheet = wb.addWorksheet("List1");
worksheet.columns = require("./columns");
for (var subc in c1.items.categories) {
var promises = [], promisesDB = [];
var c2 = c1.items.categories[subc];
var subCategoryId = c2.id; //ID подкатегории
var subCategoryName = c2.title; //имя подкатегории
for (var subc2 in c2.categories) {
var c3 = c2.categories[subc2];
var subCategory2Id = c3.id; //ID 2 подкатегории
var subCategory2Name = c3.title; //название 2 подкатегории
var subCategory2Url = c3.url; //ссылка на вторую категорию
var numPages = await this.getNumPages(subCategory2Url);
this.log(mainCategoryName + ">" + subCategoryName + ">" + subCategory2Name);
for (var page = 1; page <= numPages[0]; page++) {
var params = {
"url": subCategory2Url + "?layout_container=categorySearchMegapagination&page=" + page
};
var res = await this.req("https://www.ozon.ru/api/composer-api.bx/page/json/v1", params);
try {
var keys = Object.keys(res.body.catalog.searchResultsV2);
} catch (e) {
this.tg(e + "\n\n" + JSON.stringify(res.body.catalog));
fs.writeFileSync("searchResults-error-" + Math.floor(Math.random() * 10000) + "-" + new Date().format("dd-MM-Y") + ".log", e + "\n\n" + JSON.stringify(res.body));
continue;
}
var products = res.body.catalog.searchResultsV2[keys[keys.length - 1]].items;
for (var product in products) {
var components = this.getComponents(products[product].templateState);
var maxItems = this.getMaxItems(components) || 0;
var rating = this.getRating(components) || 0;
var seller = this.getSeller(components);
var id = products[product].cellTrackingInfo.id;
var title = products[product].cellTrackingInfo.title;
var price = products[product].cellTrackingInfo.finalPrice;
promises.push(this.addRow({
'id': id,
'Category1': mainCategoryName,
'Category2': subCategoryName,
'Category3': subCategory2Name,
'Name': title,
'Price': price,
'Quantity': maxItems,
'Reviews': rating,
'Seller': seller
}, worksheet));
// promisesDB.push(this.setMaxItems(id, maxItems));
this.log("#" + numProducts + " (" + page + ") " + title + " (" + id + ")\n");
row++;
numProducts++;
}
}
}
}
await Promise.all(promises).then(async (promises_res) => {
th.clientDb.close();
await wb.commit();
});
/* await Promise.all(promisesDB).then(() => {
th.clientDb.close();
})*/
}
}
addRow(obj, worksheet) {
return new Promise(async (resolve, reject) => {
var periods = ["day", "week", "month"];
for (var period in periods) {
obj[periods[period]] = await this.getStat(obj.id, periods[period], obj.Quantity, null, null);
}
worksheet.addRow(obj).commit();
resolve(obj);
});
}