Ускорение парсинга python beautifulsoup
Подскажите, пожалуйста, как можно ускорить парсинг каталогов на сайте? П.с. 80 000 товаров в одном каталоге
all_catalog = ['https://www.wildberries.ru/catalog/zhenshchinam/odezhda/bluzki-i-rubashki']
for url_page in all_catalog:
response = session.get(url_page, headers=headers, timeout=10)
if response.status_code == 200:
soup = bs4.BeautifulSoup(response.text, 'lxml')
js_scripts = soup.find_all('script')
#js_script = str(js_scripts[28])
m = re.search('ssrModel: (.+),', str(js_scripts))
data = json.loads(m.group(1))
totalPage = data['model']['pagerModel']['pagingInfo']['totalPages']
#print(url_page + " " + str(totalPage))
for page in range(1, totalPage+1):
url_catalog_page = url_page + "?page=" + str(page)
response = session.get(url_catalog_page, headers=headers, timeout=10)
soup = bs4.BeautifulSoup(response.text, 'lxml')
for link in soup.find_all('a', href=True):
links = link.get('href')
if "detail.aspx" in links:
url_detail = 'https://www.wildberries.ru' + links
response = session.get(url_detail, headers=headers, timeout=10)
if response.status_code == 200:
soup = bs4.BeautifulSoup(response.text, 'lxml')
js_scripts = soup.find_all('script')
#js_script = str(js_scripts[28])
ssrModel = re.search('ssrModel: (.+),', str(js_scripts))
dataPage = json.loads(ssrModel.group(1))
print(str(dataPage['rqCod1S']) + " " + str(dataPage['productCard']['goodsName']))