import os
import re
import sys
import requests
import xlsxwriter
from bs4 import BeautifulSoup
from random_user_agent.params import SoftwareName, OperatingSystem
from random_user_agent.user_agent import UserAgent
class SubitoParser():
def __init__(self, url, id):
self.url = url
self.id = id
self.session = requests.session()
software_names = [SoftwareName.CHROME.value]
operating_systems = [OperatingSystem.WINDOWS.value, OperatingSystem.LINUX.value]
user_agent_rotator = UserAgent(software_names=software_names, operating_systems=operating_systems, limit=100)
headers = {
'User-Agent': user_agent_rotator.get_random_user_agent(),
'Referer': 'https://www.subito.it/annunci-italia/vendita/auto/?from=top-bar'
}
self.session.headers = headers
def Get(self):
all_ads = []
self.session.get("https://www.subito.it/annunci-italia/vendita/auto/?from=top-bar")
soup = BeautifulSoup(self.session.get(self.url).content, "lxml")
print(soup)
for ads in soup.findAll('div', {'class': 'items__item'}):
try:
url = ads.findAll("a")[0]['href']
print(url)
ad_soup = BeautifulSoup(self.session.get(url).content, "lxml")
_all_ads = {}
_all_ads["title"] = ad_soup.findAll('h1')[0].text
_all_ads["price"] = ad_soup.findAll('p', {'class': re.compile('^classes_price.*')})[0].text
_all_ads["description"] = ad_soup.findAll('p', {'class': 'description'})[0].text
_all_ads["created"] = ad_soup.findAll('span', {'class': re.compile('^classes_insertion-date.*')})[0].text
_all_ads["location"] = ad_soup.findAll('div', {'class': re.compile('^AdInfo_ad-info__location.*')})[0].text
_all_ads["seller-name"] = ad_soup.findAll('h6', {'class': re.compile('^index-module_name.*')})[0].text
_all_ads["seller-active"] = ad_soup.findAll('button', {'class': re.compile('^index-module_socialization.*')})[0].text
_all_ads["url"] = url
all_ads.append(_all_ads)
except Exception as e:
print("[~] error: " + str(e) + ". line: " + str(sys.exc_info()[-1].tb_lineno))
re.compile('^classes_price.*')
if all_ads is not None:
table_name = 'ultimate-sorter-subito-' + str(self.id) +'.xlsx'
workbook = xlsxwriter.Workbook(table_name)
worksheet = workbook.add_worksheet()
worksheet.write(0, 0, "id")
worksheet.write(0, 1, "title")
worksheet.write(0, 2, "price")
worksheet.write(0, 3, "phone")
worksheet.write(0, 4, "posted")
worksheet.write(0, 5, "location")
worksheet.write(0, 6, "description")
worksheet.write(0, 7, "seller_name")
worksheet.write(0, 8, "seller_registered")
worksheet.write(0, 9, "seller_active_items")
worksheet.write(0, 8, "url")
for i in range(len(all_ads)):
try:
j = all_ads[i]
l = i + 1
worksheet.write(l, 0, i)
worksheet.write(l, 1, j["title"])
worksheet.write(l, 2, j["price"])
worksheet.write(l, 3, j["phone"])
worksheet.write(l, 4, j["posted"])
worksheet.write(l, 5, j["location"])
worksheet.write(l, 6, j["description"])
worksheet.write(l, 7, j["seller_name"])
worksheet.write(l, 8, j["seller-joined"])
worksheet.write(l, 9, j["seller_items"])
worksheet.write(l, 8, j["url"])
except:
print("xlsx row error")
workbook.close()
return os.path.abspath(table_name)
else:
return "error"
def Person(url, uid):
SubitoParser(url, uid).Get()
Person(url, uid)