Автоматическая адаптации учебных текстов для изучающих русский язык, её решение на лексическом уровне

import re

f = open('texts.txt', 'r', encoding='utf-8')

s = f.read()

f.close()

def get_clean_text(some_string):

s = re.findall('\w+', some_string)

for i in s:

if i.isalpha() != True:

s.pop(s.index(i))

clean_text = ' '.join(s).lower()

return clean_text

def ngrams(text, n):

l = []

a = 0

while a < (len(text)-1):

a1 = text[a:(a+n)]

if len(a1)==n:

l.append(a1)

a = a+1

return l

def get_ngrams_freq(trigrams_list):

trigram_freq = {}

for trigram in trigrams_list:

if trigram in trigram_freq.keys():

trigram_freq[trigram] +=1

else:

trigram_freq[trigram] = 1

return trigram_freq

unigram_freq = (get_ngrams_freq(ngrams(get_clean_text(s), 1)))

uni_average = float(sum(unigram_freq.values())/len(unigram_freq))

V = len(unigram_freq)

bigram_freq = (get_ngrams_freq(ngrams(get_clean_text(s), 2)))

bi_average = float(sum(bigram_freq.values())/len(bigram_freq))

bigram_prob = {}

for k, v in bigram_freq.items():

bigram_prob[k] = round((v+1.0)/(unigram_freq[k[0]]+V), 5)

trigram_freq = (get_ngrams_freq(ngrams(get_clean_text(s), 3)))

tri_average = float(sum(trigram_freq.values())/len(trigram_freq))

trigram_prob = {}

for k, v in trigram_freq.items():

trigram_prob[k] = round((v+1.0)/(bigram_freq[k[:-1]]+V), 5)

fourgram_freq = (get_ngrams_freq(ngrams(get_clean_text(s), 4)))

four_average = float(sum(fourgram_freq.values())/len(fourgram_freq))

fourgram_prob = {}

for k, v in fourgram_freq.items():

fourgram_prob[k] = round(float(v + 1)/float(trigram_freq[k[:-1]] + V), 5)

fivegram_freq = (get_ngrams_freq(ngrams(get_clean_text(s), 5)))

five_average = float(sum(fivegram_freq.values())/len(fivegram_freq))

fivegram_prob = {}

for k, v in fivegram_freq.items():

fivegram_prob[k] = round(float(v + 1)/float(fourgram_freq[k[:-1]] + V), 5)

sixgram_freq = (get_ngrams_freq(ngrams(get_clean_text(s), 6)))

six_average = float(sum(sixgram_freq.values())/len(sixgram_freq))

sixgram_prob = {}

for k, v in sixgram_freq.items():

sixgram_prob[k] = round(float(v + 1)/float(fivegram_freq[k[:-1]] + V), 5)

with open('unigramfreqs3.txt', 'w+', encoding='utf-8') as f1:

for k, v in unigram_freq.items():

f1.write('{}:{}{}'.format(k, str(v), '\n'))

with open('bigramfreqs3.txt', 'w+', encoding='utf-8') as f12:

for k, v in bigram_freq.items():

f12.write('{}:{}{}'.format(k, str(v), '\n'))

with open('trigramfreqs3.txt', 'w+', encoding='utf-8') as f13:

for k, v in trigram_freq.items():

f13.write('{}:{}{}'.format(k, str(v), '\n'))

with open('fourgramfreqs3.txt', 'w+', encoding='utf-8') as f14:

for k, v in fourgram_freq.items():

f14.write('{}:{}{}'.format(k, str(v), '\n'))

with open('fivegramfreqs3.txt', 'w+', encoding='utf-8') as f15:

for k, v in fivegram_freq.items():

f15.write('{}:{}{}'.format(k, str(v), '\n'))

with open('sixgramfreqs3.txt', 'w+', encoding='utf-8') as f16:

for k, v in sixgram_freq.items():

f16.write('{}:{}{}'.format(k, str(v), '\n'))

with open('bigramprobs3.txt', 'w+', encoding='utf-8') as f2:

for k, v in bigram_prob.items():

f2.write('{}:{}{}'.format(k, str(v), '\n'))

with open('trigramprobs3.txt', 'w+', encoding='utf-8') as f3:

for k, v in trigram_prob.items():

f3.write('{}:{}{}'.format(k, str(v), '\n'))

with open('fourgramprobs3.txt', 'w+', encoding='utf-8') as f4:

for k, v in fourgram_prob.items():

f4.write('{}:{}{}'.format(k, str(v), '\n'))

with open('fivegramprobs3.txt', 'w+', encoding='utf-8') as f5:

for k, v in fivegram_prob.items():

f5.write('{}:{}{}'.format(k, str(v), '\n'))

with open('sixgramprobs3.txt', 'w+', encoding='utf-8') as f6:

for k, v in sixgram_prob.items():

f6.write('{}:{}{}'.format(k, str(v), '\n'))



Приложение 6

Код, использовавшийся для вычисления перплексии

import re

import math

ff = open('sixgramprobs3.txt', 'r', encoding='utf-8')

grams = ff.readlines()

ff.close()

f = open('test.txt', 'r', encoding='utf-8')

s = f.read().lower()

f.close()

f1 = open('fivegramfreqs3.txt')

pre_grams = f1.readlines()

f1.close()

V = 64.0

A1 = 5599.71875

A2 = 379.6419491525424

A3 = 49.295873452544704

A4 = 11.661460367044123

A5 = 4.448266017923193

A6 = 2.4740806196626925

def get_clean_text(some_string):

s = re.findall('\w+', some_string)

for i in s:

if i.isalpha() != True:

s.pop(s.index(i))

clean_text = ' '.join(s).lower()

return clean_text

N = len(get_clean_text(s))

def ngrams(text, n):

l = []

a = 0

while a < (len(text)-1):

a1 = text[a:(a+n)]

if len(a1)==n:

l.append(a1)

a = a+1

return l

def count_prp(ngrams_list):

d = dict.fromkeys(ngrams_list, 0.0)

for i in ngrams_list:

for gram in grams:

if i == gram.split(':')[0]:

d[i] = float(gram.split(':')[1])

n = 1.0

for k, v in d.items():

if v != 0.0:

n = n * (1.0/(v**(1.0/N)))

else:

l = []

for gram in pre_grams:

if k[:-1] == gram.split(':')[0]:

n = n * (1.0/((1.0/(float(gram.split(':')[1])+V))**(1.0/N)))

l.append('d')

if not l:

n = n * (1.0/((1.0/A5+V)**(1.0/N)))

return n

sen_grams = ngrams(get_clean_text(s), 6)

prp = count_prp(sen_grams), print(prp)

Приложение 7

Код программы для упрощения текстов (используется языковая модель на четырёхграммах)

import re

import pymorphy2

import math

import timeit

f = open('substitutions2.txt', 'r', encoding='utf-8')

file_name = input('Введите имя файла с текстом для упрощения: ')

exc_words = input('Введите список слов, которые не надо упрощать

(опционально): ')

f1 = open(file_name, 'r', encoding='utf-8')

s = f1.read()

f1.close()

dictionary = f.readlines()

f.close()

ff = open('fourgramprobs3.txt', 'r', encoding='utf-8')

grams = ff.readlines()

ff.close()

exceptions = str(exc_words)

f1 = open('trigramfreqs.txt')

pre_grams = f1.readlines()

f1.close()

V = 64.0

#average frequencies for each n-gram set (unigrams included)

A1 = 5599.71875

A2 = 379.6419491525424

A3 = 49.295873452544704

A4 = 11.661460367044123

A5 = 4.448266017923193

A6 = 2.4740806196626925

def ngrams(text, n):

l = []

a = 0

while a < (len(text)-1):

a1 = text[a:(a+n)]

if len(a1)==n:

l.append(a1)

a = a+1

return l

def count_prob(ngrams_list):

d = dict.fromkeys(ngrams_list, 0.0)

for i in ngrams_list:

for gram in grams:

if i == gram.split(':')[0]:

d[i] = float(gram.split(':')[1])

n = 0.0

for k, v in d.items():

if v != 0.0:

n = n + math.log(v)

else:

l = []

for gram in pre_grams:

if k[:-1] == gram.split(':')[0]:

n = n + math.log(1.0/(float(gram.split(':')[1])+V))

l.append('d')

if not l:

n = n + math.log(1.0/A3+V)

return math.exp(n)

def simplify(s, exceptions):

possible = []

done = []

extra = []

word_tokens = re.findall('\w+', s)

all_tokens = re.findall('\w+|[.,:;\'\"---?!«»]|\n', s)

subs = []

d = []

morph = pymorphy2.MorphAnalyzer()

pre_dict = re.findall('\w+', exceptions.lower())

exeptions_dict = []

for i in pre_dict:

form = morph.parse(i)[0]

lex = form.lexeme

for ii in range(len(lex)):

exeptions_dict.append(lex[ii].word)

for token in word_tokens:

if token.lower() not in exeptions_dict:

for pair in dictionary:

if token.lower() == pair.split()[0]:

n = word_tokens.index(token)

if (n > 3) and (n < (len(word_tokens) - 3)):

s = '{} {} {} {} {} {} {}'.format(word_tokens[n-3], word_tokens[n-2],

word_tokens[n-1], word_tokens[n], word_tokens[n+1], word_tokens[n+2],

word_tokens[n+3])

s1 = '{} {} {} {} {} {} {}'.format(word_tokens[n-3], word_tokens[n-2],

word_tokens[n-1], pair.split()[1], word_tokens[n+1], word_tokens[n+2],

word_tokens[n+3])

elif n <= 3:

s = '{} {} {} {}'.format(word_tokens[n], word_tokens[n+1],

word_tokens[n+2], word_tokens[n+3])

s1 = '{} {} {} {}'.format(pair.split()[1], word_tokens[n+1],

word_tokens[n+2], word_tokens[n+3])

elif n >= (len(word_tokens) - 3):

s = '{} {} {} {}'.format(word_tokens[n-3], word_tokens[n-2],

word_tokens[n-1], word_tokens[n])

s1 = '{} {} {} {}'.format(word_tokens[n-3], word_tokens[n-2],

word_tokens[n-1], pair.split()[1])

if (count_prob(ngrams(s, 4))) <= (count_prob(ngrams(s1, 4))):

subs.append(pair)

for i in subs:

if i[-2] == '1':

d.append(i)

else:

possible.append('{} --> {}'.format(i.split()[0], i.split()[1]))

for token in all_tokens:

for i in d:

try:

if token == i.split()[0]:

all_tokens.insert(all_tokens.index(token), i.split()[1])

all_tokens.pop(all_tokens.index(token))

done.append('{} --> {}'.format(i.split()[0], i.split()[1]))

except ValueError:

ex = d.pop(d.index(i))

extra.append('{} --> {}'.format(ex.split()[0], ex.split()[1]))

text = ' '.join(i for i in all_tokens)

text = text.replace(' .', '.')

text = text.replace(' ,', ',')

text = text.replace(' ?', '?')

text = text.replace(' !', '!')

text = text.replace(' :', ':')

text = text.replace(' ;', ';')

text = text.replace(' )', ')')

text = text.replace('( ', '(')

text = text.replace('« ', '«')

text = text.replace(' »', '»')

done1 = ' \n'.join(i for i in set(done))

extra1 = ' \n'.join(i for i in set(extra))

possible1 = ' \n'.join(i for i in set(possible))

return text + '\n' + 'Выполненные замены: ' + done1 + '\n' + 'Варианты

выполненных замен: ' + extra1 + '\n' + 'Другие возможные замены: ' +

possible1 + '\n'

print(simplify(s, exceptions))

input('Нажмите любую клавишу для выхода')

Размещено на Allbest.ru