Ошибки при парсинге, многопоточности, packet sequence number wrong pymysql
Работаю с потоками в python, использую API VK для парсинга групп и страниц по тегам(названия городов) из таблицы SQL, то есть нахожу все посты с такими тегами и собираю инфу о группах, посте и тому подобном, но после использования 2 потоков (больше не пытался) выходит ошибка
pymysql.err.InternalError: Packet sequence number wrong - got 1 expected 2,
второй поток выдаёт ошибку NoneType, хотя я много раз проверял, вся инфа собирается верно, единственное, что я смог предположить - это то, что во время того, как один поток добавляет инфу о ресурсе в таблицу - второй сразу переходит к моменту, где нужно взять ID данного ресурса из таблицы, но т.к не находит информации о своём ресурсе (т.к сразу перепрыгнул) - выдаёт NoneType, а затем добавляет информацию о ресурсе (инфа о ресурсе в момент выдачи ошибки есть в таблице, но его ID он не берёт) и выдаёт окончательную ошибку, но это только моё предположение)
Вот основной код:
import requests
import pymysql.cursors
import time
import datetime
class Keko:
connection = pymysql.connect(host = 'localhost',
user = 'root',
password = '',
db = 'pars_global',
charset = 'utf8mb4',
autocommit = 'True')
#try:
#with connection:
cur = connection.cursor()
#except Exception as error:
#print(error)
def connect_resources(self, take_resources):
#Подключение ресурсов
insert_resources = "INSERT IGNORE INTO resources (owner_id, link, resource_name, resource_img, city_id, country_id) VALUES (%s, %s, %s, %s, %s, %s)"
self.cur.executemany(insert_resources, take_resources)
def connect_posts(self, take_posts): #Подключение постов в таблицу
insert_post_data = "INSERT IGNORE INTO posts(resource_id, `text`, `date`, post_id, from_id, comments, likes, reposts) VALUES (%s, %s, %s, %s, %s, %s, %s, %s)"
self.cur.executemany(insert_post_data, take_posts)
def connect_attachments(self, take_attachments): #Подключение картинки, видео, ссылок в таблицу
insert_attachments = "INSERT IGNORE INTO attachments(post_id, type, data) VALUES (%s, %s, %s)"
self.cur.executemany(insert_attachments, take_attachments)
def get_json(self, url, data = None):
#Функция запроса по ссылке метода
response = requests.get(url, params = data)
return(response.json())
def get_results(self, access_token, q, count, start_time, end_time):
# Отправляет запрос поиска и получает ВСЮ информацию о найденных постах по ключевому слову q и кол-ву count
all_results = []
time.sleep(0.40)
search = self.get_json('https://api.vk.com/method/newsfeed.search', {
'access_token' : access_token,
'q' : q,
'v' : '5.103',
'count' : count,
'start_time' : start_time,
'end_time' : end_time
})
#print(search)
posts = search['response']['items']
all_results.extend(posts)
return(all_results)
def pars(self, all_results, access_token, fields): #основная функция парсинга
att2 = []
attachments = []
posts = []
for result in all_results:
res_id = self.get_resource_id(access_token, result, fields)
posts.append(self.take_posts(result, res_id))
item_id = result['id']
attachment = self.take_attachments(result)
if attachment != None:
for attach in attachment:
attachments.append((res_id, attach, item_id))
#print(posts)
self.connect_posts(posts)
# sql = "SELECT LAST_INSERT_ID()"
# self.cur.execute(sql)
# last_id = self.cur.fetchone()[0]
for att in attachments:
res_id = att[0]
item_id = att[2]
post_id = self.get_post_id(res_id, item_id)
att_type = att[1][0]
att_data = att[1][1]
att2.append((
post_id,
att_type,
att_data
))
self.connect_attachments(att2)
# return(last_id)
def get_post_id(self, res_id, item_id):
sql = "SELECT posts.id FROM posts WHERE resource_id = {0} and post_id = {1}".format(res_id, item_id)
self.cur.execute(sql)
post = self.cur.fetchone()
post_id = post[0]
return(post_id)
def get_resource_id(self, access_token, result, fields): #получение id ресурса (функция с ошибкой)
owner_id = str(result['owner_id'])
print(owner_id)
print('first')
sql = "SELECT id FROM resources WHERE owner_id = '%s'" % owner_id
self.cur.execute(sql)
resource = self.cur.fetchone()
print(resource)
if resource == None:
if owner_id[0] == '-':
owner_id = owner_id[1:]
wall = self.get_json('https://api.vk.com/method/groups.getById', {
'access_token' : access_token,
'group_ids' : owner_id,
'fields' : fields,
'v' : '5.103'
})
resources = wall['response']
else:
wall = self.get_json('https://api.vk.com/method/users.get', {
'access_token' : access_token,
'user_ids' : owner_id,
'fields' : fields,
'v' : '5.103'
})
resources = wall['response']
print(resources)
get_resources = self.take_resources(resources)
owner_id = get_resources[0][0]
print(get_resources)
print(owner_id)
insert_resources = self.connect_resources(get_resources)
# time.sleep(4)
sql = "SELECT id FROM resources WHERE owner_id = '%s'" % owner_id
self.cur.execute(sql)
resource = self.cur.fetchone()
resource_id = resource[0]
print(resource_id)
return(resource_id)
def take_resources(self, filtered_info):
#print(filtered_info)
filtered_resources_info = []
city_id = ' '
country_id = ' '
resource_img = str(filtered_info[0]['photo_100'])
if filtered_info[0].get('country'):
country_id = str(filtered_info[0]['country']['id'])
if filtered_info[0].get('city'):
city_id = str(filtered_info[0]['city']['id'])
if filtered_info[0].get('type'):
resource_name = str(filtered_info[0]['name'])
owner_id = '-' + str(filtered_info[0]['id'])
link = 'vk.com/club' + owner_id[1:]
else:
page_first_name = str(filtered_info[0]['first_name'])
page_last_name = str(filtered_info[0]['last_name'])
resource_name = page_first_name + ' ' + page_last_name
owner_id = str(filtered_info[0]['id'])
link = 'vk.com/id' + owner_id
filtered_resources_info.append((
owner_id,
link,
resource_name,
resource_img,
city_id,
country_id
))
return(filtered_resources_info)
def take_posts(self, post, resource_id):
filtered_data = []
post_text = post['text']
post_data = datetime.datetime.fromtimestamp(int(post['date'])).strftime('%Y-%m-%d %H:%M:%S')
post_id = str(post['id'])
post_from_id = str(post['from_id'])
post_comments = str(post['comments']['count'])
post_likes = str(post['likes']['count'])
post_reposts = str(post['reposts']['count'])
filtered_data = (
resource_id,
post_text,
post_data,
post_id,
post_from_id,
post_comments,
post_likes,
post_reposts
)
return(filtered_data)
def take_attachments(self, result):
#print(result)
filtered_data = []
attachments = result.get('attachments')
if attachments:
for att in attachments:
if att['type'] == 'photo':
photo = str(att['photo']['sizes'][-1]['url'])
p_type = str(att['type'])
filtered_data.append((
p_type,
photo,
))
if att['type'] == 'link':
link = str(att['link']['url'])
l_type = str(att['type'])
filtered_data.append((
l_type,
link,
))
if att['type'] == 'video':
video = str(att['video']['title'])
v_type = str(att['type'])
filtered_data.append((
v_type,
video,
))
return(filtered_data)
def update_time(self, q):
insert_data = []
now = datetime.datetime.now()
last_time = now.strftime('%Y-%m-%d %H:%M:%S')
insert_data.append((last_time, str(q)))
sql = "UPDATE IGNORE tags SET last_time = %s WHERE tag = %s"
self.cur.executemany(sql, insert_data)
def take_tag(self):
sql = "SELECT tag, last_time, active FROM tags WHERE active = 'False' order by last_time"
self.cur.execute(sql)
tag = self.cur.fetchone()
return(tag)
def take_last_time(self, tag):
sql = "SELECT last_time FROM tags where tag = '%s'" % tag
self.cur.execute(sql)
result = self.cur.fetchone()
last_time = result[0]
return(last_time)
def end_time(self, start_pars):
sql = "SELECT date FROM posts WHERE id = %s" % start_pars
self.cur.execute(sql)
start = self.cur.fetchone()[0]
return(start)
def is_active(self, tag):
sql = "SELECT active FROM tags WHERE tag = '%s'" % tag
self.cur.execute(sql)
check = self.cur.fetchone()[0]
return(check)
def turn_on(self, tag):
# if check == 'False':
sql1 = "UPDATE IGNORE tags SET active = 'True' WHERE tag = '%s'" % tag
self.cur.execute(sql1)
# else:
# sql1 = "UPDATE IGNORE tags SET active = 'False' WHERE tag = '%s'" % tag
# self.cur.execute(sql1)
def turn_off(self, tag):
sql = "UPDATE IGNORE tags SET active = 'False' WHERE tag = '%s'" % tag
self.cur.execute(sql)
А это исполняемый файл:
from parss import Keko
import datetime
import warnings
from datetime import timedelta
import threading
import time
import queue
#print(start_time, end_time)
def prescript():
k1 = Keko()
now = datetime.datetime.now()
time_now = now.strftime('%Y-%m-%d %H:%M:%S')
tag = k1.take_tag()
print(tag)
q = tag[0]
print(q)
activate = k1.turn_on(q)
count = 200
access_token = '6196c6fb6196c6fb6196c6fb7461f9a7fc661966196c6fb3fac4d7661553f642d036cc5'
fields = ('city, country, photo_100')
start = tag[1]#k1.take_last_time(tag)
print(start)
start_time = int(start.timestamp())
end = now
end_time = int(end.timestamp())
#warnings.filterwarnings('ignore')
while end_time > start_time:
all_results = k1.get_results(access_token, q, count, start_time, end_time)
length = len(all_results)
if length == 0:
break
end_time = all_results[length - 1]['date']
start_pars = k1.pars(all_results, access_token, fields)
if len(all_results) < 200:
break
update_time = k1.update_time(q)
k1.turn_off(q)
while True:
thread1 = threading.Thread(target = prescript)
#time.sleep(1)
thread2 = threading.Thread(target = prescript)
thread1.start()
#time.sleep(1)
thread2.start()
thread1.join()
thread2.join()



