404/403 ошибка при парсинге Python
Час добрый. Подскажите пожалуйста из-за чего может вылетать ошибка 404/403 при переделке рабочего парсера на ассинхронный тип (только разбираю библиотеку). При том что обычная версия работает без единого сбоя
Подскажите пожалуйста каким образом можно решить проблему, код прилагаю
import time
import asyncio
import aiohttp
start_time = time.time()
headers = {
':authority': 'www.tiktok.com',
':method': 'GET',
':scheme': 'https',
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7',
'Connection': 'keep-alive',
'Content-Length': '4922',
'Content-Type': 'text/plain;charset=UTF-8',
'cache-control': 'max-age=0',
'cookie': '_ga=GA1.2.2053425744.1601738140; store-idc=alisg; store-country-code=ru; tt_webid_v2=6918849925640570374; tt_webid=6918849925640570374; d_ticket=317d201b797384294dc942d28c65082daf6dd; MONITOR_WEB_ID=6918849925640570374; cookie-consent={%22ga%22:true%2C%22af%22:true%2C%22fbp%22:true%2C%22lip%22:true%2C%22version%22:%22v2%22}; sid_guard=701e4c344b88a55babe4f213086f02e3%7C1619249438%7C5184000%7CWed%2C+23-Jun-2021+07%3A30%3A38+GMT; odin_tt=1271fabd4b20705347d43514e93c79d80ee4634a715c08f3670fb183ed97f859f2e49073602354f301c9c7988f98af81670de58ee671fae5ddf2cd1f7dcfe396f93cd9ce9439d90229775fc56d501f76; _abck=F8DEEA7D31D21B257CFEB8E1D127434A~-1~YAAQRiC81P7/B6R6AQAA0ei7qQawRobDufh5WX7xufAnI7/mSKhtULWkM7AtG2JxZHT5srvpkhlcNGsP3XW50C5wbo84fRfPCYIXEZYedpzb0cWUZnhJ1x/DEQh+Pp/KTP7TFdw2MnMgNuQez1PSTGZtiSHpaw4LM18IqEDXgtLGJDUodxUpQnYOhymPZ8vB7KssiaZOhPdEZdmh7jhpNOnE3AchuxRnkkCu3Ju3+RrKxdGxAvgkOo3qgIvkAEgorZYz2f0p3x0BPQw31KAtmCv6AxyDVF5n5TLxWi94qpGnvd2B63FAm2Otvq/Wq06ozI0LpTJbxYkgdpvLHmVtDfYThlK00ST6VrSqGqvj/twH6w3N97YQtgWGDweDjzhEgaC58dg5lzqeDA==~-1~-1~-1; _tea_utm_cache_1988={%22utm_source%22:%22vk%22%2C%22utm_medium%22:%22android%22%2C%22utm_campaign%22:%22client_share%22}; tt_csrf_token=0Qkr2A0oBlAYOW8Nl-kBoTYG; R6kq3TV7=AGJZAKB8AQAA2OPbusibnuORUAeFID5XqIljCr1GzdVDOl0zueQJaybgHH61|1|0|80b54e025b3bfa56f90eaa88fb43bad373d1b076; s_v_web_id=verify_kv050dt4_dcazG3SI_r1VV_4HJQ_Ayr6_Ey5twPlUScwx; msToken=tGzCnJRWpwGuM_E6M4p3JvelPeDP42BgE3vTMo22ST-J48FwgJeIXeIYba7tZx6cWfjj5DQTfzdD8pg7Jb0CMc58ITt054OFbt86LORJea9xiPAvV56SN9sjatZv8gxbXxQhWA==; __tea_cookie_tokens_1988=%257B%2522web_id%2522%253A%2522%2522%252C%2522timestamp%2522%253A1634772825405%257D; ttwid=1%7Cvt73kp5ht-ZHxXZEUQ3Y_H-7YPoh9f1n4o5V2brBmhs%7C1634772826%7C561289ad39dc560e27f561cb0f6fedad10ecf015832bab68e7540d7895976463',
'Host': 'mon-va.byteoversea.com',
'Origin': 'https://www.tiktok.com',
'Referer': 'https://www.tiktok.com/',
'sec-ch-ua': '"Chromium";v="94", "Google Chrome";v="94", ";Not A Brand";v="99"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Windows"',
'Sec-Fetch-Dest': 'empty',
'Sec-Fetch-Mode': 'cors',
'Sec-Fetch-Site': 'cross-site',
'upgrade-insecure-requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36'
}
async def get_page_date(session, all_links_number, url_tt, id_tt):
ready_number = 1
false_number = 1
async with session.get(url=url_tt, headers=headers) as r:
try:
#r = session.get(url_tt)
r = str(r.text)
print(r)
time.sleep(500)
start = r.find('"followingCount":')
end = r.find(',"heart"')
substring = r[start:end]
substring = substring.split(':')[1]
# print(substring)
# time.sleep(1)
sim_link = url_tt.split()
s = substring
l = s
l = [i.replace('K', '000') for i in l]
l = (''.join(l))
full_info = f"{id_tt};{sim_link[0]};{l};0"
with open('link_new.txt', 'a') as f:
f.write(f'{full_info}\n')
print(f'[INFO] {ready_number}/{all_links_number} | {l} | +')
ready_number += 1
except Exception as ex:
print(f'[INFO] {ready_number}/{all_links_number} | -')
sim_link = url_tt.split()
with open('errors_links.txt', 'a') as f:
f.write(f'{id_tt};{sim_link[0]};0;1\n')
ready_number += 1
print(false_number)
false_number += 1
pass
async def gather_data():
async with aiohttp.ClientSession() as session:
all_links = []
tasks = []
with open("links_old.txt", "r") as ins:
for line in ins:
all_links.append(line)
for link in all_links:
url = link.split(';')
url_tt = str(url[1])
id_tt = str(url[0])
id_tt = id_tt.rstrip()
all_links_number = len(all_links)
task = asyncio.create_task(get_page_date(session, all_links_number, url_tt, id_tt))
tasks.append(task)
await asyncio.gather(*tasks)
def main():
asyncio.run(gather_data())
if __name__ == "__main__":
main()