import string
import json
import re
from nltk import word_tokenize
import nltk
from nltk.probability import FreqDist
import pandas as pd
import openpyxl
import xlrd
import csv
import re
file = open('sw_templates.json', 'w')
pd.set_option("max_colwidth", 10000)
pd.set_option("max_rows", 10000)
fields = ['DialogId', 'Text']
excel_data = pd.read_excel('data.xlsx', nrows = 3, usecols=fields)
for i in excel_data.values:
text = i[1]
text = text.lower()
text_tokens = word_tokenize(text)
text_tokens[:10]
r = nltk.Text(text_tokens)
r[:10]
fdist = FreqDist(r)
a = fdist.most_common(1000000)
#json_data = json.dumps(a, separators=(':', ';'), indent=4)
b = str(i[0])
to_json = {'DialogId': b,'Text_tokens': a}
templates = json.dumps(to_json)
file.write(templates)
print('Готово')
Выводит:
{"DialogId": "10000020335-client", "Text_tokens": [["\u0431\u043b\u0430\u0433\u043e\u0434\u0430\u0440\u044e89896251552", 1], ["\u0430\u043b\u0435\u043d\u0430", 1]]}
{"DialogId": "10001311193-client", "Text_tokens": [[":", 3], [".", 2], ["https", 2], ["\u043c\u043e\u0436\u0435\u0442\u0435", 1], ["\u0445\u043e\u0442\u044c", 1], ["\u0441\u0435\u0439\u0447\u0430\u0441", 1], ["\u043f\u043e\u043a\u0430", 1], ["\u043d\u0435", 1], ["\u0441\u043f\u043b\u044e.\u0430\u043b\u044c\u044f\u043d\u0441", 1], ["select", 1], ["\u0447\u0435\u0431\u043e\u043a\u0441\u0430\u0440\u044b", 1], ["lada", 1], ["(", 1], ["\u0432\u0430\u0437", 1], [")", 1], ["kalina", 1], ["\u0445\u044d\u0442\u0447\u0431\u0435\u043a", 1], ["5", 1], ["\u0434\u0432", 1], [",", 1], ["285", 1], ["000", 1], ["\u0440\u0443\u0431", 1], ["//avatars.mds.yandex.net/get-autoru-vos/2167160/6cd512f94aa695911c23e96049043a5a/small", 1], ["?", 1], ["webp=false", 1], ["//auto.ru/cars/used/sale/1101793231-913638ef", 1]]}