Сохранения большого кол-ва данных в xml PYTHON
Имеется данный сайт с объявлениями(https://kvartiry-bolgarii.ru/) Со всех объявлений нужно забрать всю информацию (картинки, описание, цена и т.д.), к примеру с этого (https://kvartiry-bolgarii.ru/neveroyatnaya-kvartira-s-vidom-na-more-tip-pentkhaus-o26253)
Сбор всех данных работает, однако наполнение xml файла не удается в полной мере. Т.к. скрипт за место того чтобы информацию из каждого объявления записывать друг за другом, он заменяет данные из первого объявления, данными из второго объявления.
Xml файл должен иметь примерно такой вид
<items>
<item>
<id>1</id>
<price> 1700 </price>
</item>
<item>
<id>2</id>
<prive> 500 </price>
</item>
</items>
P.s. извиняюсь за то что закинул весь код, с частью выше комментария перед обработкой для xml никаких проблем нет.
Вопрос в том, как можно реализовать записать данных каждой страницы, поместив данные в конце xml в теге
<item> ... </item>
Код который использую на данный момент. Получение xml начинается после: ( # ↓ Putting information in xml ↓ ):
import time
from selenium import webdriver
import requests
from bs4 import BeautifulSoup
from selenium.common.exceptions import NoSuchElementException
from xml.etree import ElementTree
options = webdriver.ChromeOptions()
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option("useAutomationExtension", False)
options.add_argument("--disable-blink-features=AutomationControlled")
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36"
}
def get_link_info():
number = 0
all_links = []
try:
url = "https://kvartiry-bolgarii.ru"
driver = webdriver.Chrome(
executable_path=r'C:\Users\kk\Desktop\scrap_house\drivers\chromedriver.exe',
options=options
)
driver.get(url)
time.sleep(3)
old_links = set() # Empty Set
tr = True
while tr:
# Scroll to get more ads
driver.execute_script("window.scrollBy(0,3825)", "")
# Wait for new ads to load
time.sleep(5)
links_divs = driver.find_elements_by_xpath('//div[@class="content"]//a') # Find Elements
ans = set(links_divs) - set(old_links) # Remove old elements
for link in ans:
# Scroll to the link.
driver.execute_script("arguments[0].scrollIntoView();", link)
fir = link.get_attribute('href')
all_links.append(fir)
# Remove Duplicates
old_links = links_divs
#tr = False
for finale_link in all_links:
check = "https://kvartiry-bolgarii.ru/kvartira-v-elitnom-komplekse-s-unikalynym-sadom-o21751"
if finale_link == check:
print("Завершено: ", len(all_links))
tr = False
result_list = []
for finale_link in all_links: # Get data from all links
driver.get(finale_link)
time.sleep(3)
try:
id_info = driver.find_element_by_xpath('/html/body/main/div[2]/div/div[1]/div[3]/ul/li[1]/span')
id_data = id_info.text
except NoSuchElementException:
id_data = ""
try:
title = driver.find_element_by_xpath('/html/body/main/div[2]/div/div[1]/div[1]/div/h1') # Find Elements
title_data = title.text
except NoSuchElementException:
title_data = ""
try:
price = driver.find_element_by_xpath('/html/body/main/div[2]/div/div[1]/div[2]/div[1]/div/div[1]/p[1]')
price_data = price.text
except NoSuchElementException:
price_data = ""
try:
area = driver.find_element_by_xpath('/html/body/main/div[2]/div/div[1]/div[2]/div[1]/div/div[1]/p[2]/span')
area_data = area.text
except NoSuchElementException:
area_data = ""
try:
town = driver.find_element_by_xpath('/html/body/main/div[2]/div/div[1]/div[3]/ul/li[2]/span')
town_data = town.text
except NoSuchElementException:
town_data = ""
try:
complex = driver.find_element_by_xpath('/html/body/main/div[2]/div/div[1]/div[3]/ul/li[3]/span')
complex_data = complex.text
except NoSuchElementException:
complex_data = ""
try:
year = driver.find_element_by_xpath('/html/body/main/div[2]/div/div[1]/div[3]/ul/li[4]/span')
year_data = year.text
except NoSuchElementException:
year_data = ""
try:
furniture = driver.find_element_by_xpath('/html/body/main/div[2]/div/div[1]/div[3]/ul/li[5]/span')
furniture_data = furniture.text
except NoSuchElementException:
title_data = ""
try:
distance = driver.find_element_by_xpath('/html/body/main/div[2]/div/div[1]/div[3]/ul/li[6]/span')
distance_data = distance.text
except NoSuchElementException:
title_data = ""
try:
look = driver.find_element_by_xpath('/html/body/main/div[2]/div/div[1]/div[3]/ul/li[7]/span')
look_data = look.text
except NoSuchElementException:
look_data = ""
try:
season = driver.find_element_by_xpath('/html/body/main/div[2]/div/div[1]/div[3]/ul/li[8]/span')
season_data = season.text
except NoSuchElementException:
season_data = ""
try:
floors = driver.find_element_by_xpath('/html/body/main/div[2]/div/div[1]/div[3]/ul/li[9]/span')
floors_data = floors.text
except NoSuchElementException:
floors_data = ""
try:
material = driver.find_element_by_xpath('/html/body/main/div[2]/div/div[1]/div[3]/ul/li[10]/span')
material_data = material.text
except NoSuchElementException:
material_data = ""
try:
heating = driver.find_element_by_xpath('/html/body/main/div[2]/div/div[1]/div[3]/ul/li[11]/span')
heating_data = heating.text
except NoSuchElementException:
heating_data = ""
try:
parking = driver.find_element_by_xpath('/html/body/main/div[2]/div/div[1]/div[3]/ul/li[12]/span')
parking_data = parking.text
except NoSuchElementException:
parking_data = ""
try:
guard = driver.find_element_by_xpath('/html/body/main/div[2]/div/div[1]/div[3]/ul/li[13]/span')
guard_data = guard.text
except NoSuchElementException:
guard_data = ""
try:
elevator = driver.find_element_by_xpath('/html/body/main/div[2]/div/div[1]/div[3]/ul/li[14]/span')
elevator_data = elevator.text
except NoSuchElementException:
elevator_data = " "
try:
description = driver.find_element_by_xpath('/html/body/main/div[2]/div/div[1]/div[4]/p/span')
description_data = description.text
except NoSuchElementException:
description_data = ""
all_rooms = []
try:
rooms = driver.find_elements_by_xpath('/html/body/main/div[2]/div/div[1]/div[2]/div[2]/div[1]/span[2]/span')
for room in rooms:
room_data = room.text
all_rooms.append(room_data)
except NoSuchElementException:
all_rooms = ""
time.sleep(3)
try:
coords = driver.find_element_by_xpath('//*[@id="map"]')
coords_data = coords.get_attribute("data-coords")
#print(coords_data)
except NoSuchElementException:
coords_data = ""
try:
image_list = []
base = finale_link
rs = requests.get(base)
root = BeautifulSoup(rs.content, 'html.parser')
urls = root.select('#slider > li > img[src]')
for url in urls:
images_data = (base + url['src'])
image_list.append(images_data)
except NoSuchElementException:
image_list = []
# ↓ Putting information in xml ↓
root1 = ElementTree.Element("items")
item = ElementTree.SubElement(root1, "item")
ads_id = ElementTree.SubElement(item, "id")
ads_id.text = id_data
title_x = ElementTree.SubElement(item, "title")
title_x.text = title_data
area_x = ElementTree.SubElement(item, "total_area")
area_x.text = area_data
price_x = ElementTree.SubElement(item, "price")
price_x.text = price_data
town_x = ElementTree.SubElement(item, "town")
town_x.text = town_data
complex_x = ElementTree.SubElement(item, "complex")
complex_x.text = complex_data
year_x = ElementTree.SubElement(item, "year")
year_x.text = year_data
furniture_x = ElementTree.SubElement(item, "furniture")
furniture_x.text = furniture_data
distance_x = ElementTree.SubElement(item, "distance")
distance_x.text = distance_data
look_x = ElementTree.SubElement(item, "look_at_wind")
look_x.text = look_data
season_x = ElementTree.SubElement(item, "season")
season_x.text = season_data
floors_x = ElementTree.SubElement(item, "floors")
floors_x.text = floors_data
material_x = ElementTree.SubElement(item, "material")
material_x.text = material_data
heating_x = ElementTree.SubElement(item, "heating")
heating_x.text = heating_data
parking_x = ElementTree.SubElement(item, "parking")
parking_x.text = parking_data
guard_x = ElementTree.SubElement(item, "guard")
guard_x.text = guard_data
rooms_x = ElementTree.SubElement(item, "rooms")
rooms_x.text = room_data
for room_a in all_rooms:
room_xm = ElementTree.SubElement(rooms_x, "room")
room_xm.text = room_a
elevator_x = ElementTree.SubElement(item, "elevator")
elevator_x.text = elevator_data
description_x = ElementTree.SubElement(item, "description")
description_x.text = description_data
coords_x = ElementTree.SubElement(item, "coords")
coords_x.text = coords_data
images_x = ElementTree.SubElement(item, "images")
for image_a in image_list:
image_xm = ElementTree.SubElement(images_x, "image")
image_xm.text = image_a
tree = ElementTree.ElementTree(root1)
tree.write("result_list.xml", encoding='utf-8')
number += 1
print("[+]", number, "//", len(all_links))
except Exception as ex:
raise ex
finally:
print("♥ Завершение работы ♥")
driver.close()
driver.quit()
def main():
get_link_info()
if __name__ == "__main__":
main()