Как ускорить процесс скрапинга в Селениуме?
Нужно собрать оклоло 200,000 позиций с сайта. На одну позицию , уходит примерно 25 секунд. Нужно ускорить процесс если это возможно. А то я поседею пока извлеку всю информацию). Спасибо
from selenium import webdriver
import time
import pandas as pd
import xlsxwriter
from openpyxl.workbook import Workbook
from selenium.webdriver.firefox.options import Options
links_list = []
for x in range(4, 10):
options = Options()
options.headless = True
driver = webdriver.Firefox(options=options, executable_path='./geckodriver')
driver.implicitly_wait(3)
driver.get(f'https://www.imobiliare.ro/vanzare-apartamente/bucuresti?pagina={x}')
time.sleep(4)
cookies = driver.find_element_by_xpath('/html/body/div[2]/div[2]/div[4]/div/div/div/div[2]/div[2]/div[2]/a').click()
ap_link = driver.find_elements_by_class_name('img-block')
for item in ap_link:
all_links = item.get_property('href')
links_list.append(all_links)
try:
ivory = driver.find_element_by_xpath('/html/body/div[5]/div/div[1]/button').click()
except:
pass
ap_list = []
try:
ivory = driver.find_element_by_xpath('/html/body/div[5]/div/div[1]/button').click()
except:
pass
for index, item in enumerate(links_list):
try:
ivory = driver.find_element_by_xpath('/html/body/div[5]/div/div[1]/button').click()
except:
pass
driver.get(item)
time.sleep(1)
name = driver.find_element_by_xpath('/html/body/div[2]/div[2]/div/main/div/div[2]/div[1]/div/div/h1').text
try:
phone_click = driver.find_element_by_xpath('/html/body/div[2]/div[2]/div/main/div/div[2]/div[2]/div[2]/div[2]/div[1]/div[1]/div[2]/div[1]/span/a').click()
except:
phone_click = driver.find_element_by_xpath('/html/body/div[2]/div[2]/div/main/div/div[2]/div[2]/div[2]/div[2]/div[1]/div[1]/div[3]/div[1]/span/a').click()
time.sleep(2)
try:
ivory_phone = driver.find_element_by_xpath('/html/body/div[5]/div/div[2]/div/div/div[1]/div[2]/div/div[2]/div[1]/span').text
except:
pass
try:
ivory_phone = driver.find_element_by_class_name('nrtel').text
except:
pass
try:
ivory_phone = driver.find_element_by_xpath('/html/body/div[5]/div/div[2]/div/div/div[1]/div[2]/div/div[2]/div[1]/span').text
except:
pass
time.sleep(1)
try:
whois = driver.find_element_by_xpath('/html/body/div[6]/div/div[2]/div/div/div[1]/div[2]/div/div[1]').text
except:
whois = driver.find_element_by_xpath('/html/body/div[2]/div[2]/div/main/div/div[2]/div[2]/div[2]/div[2]/div[1]/div[1]/div[1]/div[2]').text
try:
ivory = driver.find_element_by_xpath('/html/body/div[5]/div/div[1]/button').click()
except:
pass
time.sleep(1)
time.sleep(1)
location = driver.find_element_by_xpath('/html/body/div[2]/div[2]/div/main/div/div[2]/div[1]/div/div/div/div/div[1]').text.replace(' - Vezi hartă', '')
time.sleep(1)
price = driver.find_element_by_xpath('/html/body/div[2]/div[2]/div/main/div/div[2]/div[2]/div[1]/div/div/div[1]').text
try:
more = driver.find_element_by_xpath('/html/body/div[2]/div[2]/div/main/div/div[2]/div[3]/div[1]/div[2]/p/span[1]/a').click()
except:
pass
time.sleep(1)
try:
details = driver.find_element_by_id('b_detalii_text').text.replace('Detalii', '')
except:
details = 'NA'
time.sleep(1)
caracteristici = driver.find_element_by_id('b_detalii_caracteristici').text.replace('Caracteristici', '')
time.sleep(1)
try:
specificatii = driver.find_element_by_id('b_detalii_specificatii').text.replace('Specificaţii', '')
except:
specificatii = 'NA'
apartamente = {
'Titlu': name,
'Locatie': location,
'Vanzator': whois,
'Telefon': ivory_phone,
'Pret': price,
'Detalii': details,
'Caracteristici': caracteristici,
'Specificatii': specificatii
}
ap_list.append(apartamente)
# with Pool(10) as p:
# p.map(ap_list)
print(str(index + 1) + ' Salvat: ', apartamente['Titlu'])
df = pd.DataFrame(ap_list)
df.to_excel('Apartamente_Bucuresti.xlsx', encoding='UTF-8', engine='xlsxwriter')
driver.close()
driver.quit()