Thêm Dấu Tiếng Việt Cho Câu Không Dấu

 1from keras.models import load_model
 2model = load_model('a_best_weight.h5')
 3
 4from collections import Counter
 5
 6import numpy as np
 7
 8import utils
 9import string
10import re
11
12alphabet = set('\x00 _' + string.ascii_lowercase + string.digits + ''.join(utils.ACCENTED_TO_BASE_CHAR_MAP.keys()))
13
14print("alphabet",alphabet)
15codec = utils.CharacterCodec(alphabet, utils.MAXLEN)
16
17def guess(ngram):
18    text = ' '.join(ngram)
19    text += '\x00' * (utils.MAXLEN - len(text))
20    if utils.INVERT:
21        text = text[::-1]
22    preds = model.predict_classes(np.array([codec.encode(text)]), verbose=0)
23    rtext = codec.decode(preds[0], calc_argmax=False).strip('\x00')
24    if len(rtext)>0:
25        index = rtext.find('\x00')
26        if index>-1:
27            rtext = rtext[:index]
28    return rtext
29
30
31def add_accent(text):
32    # lowercase the input text as we train the model on lowercase text only
33    # but we keep the map of uppercase characters to restore cases in output
34    is_uppercase_map = [c.isupper() for c in text]
35    text = utils.remove_accent(text.lower())
36
37    outputs = []
38    words_or_symbols_list = re.findall('\w[\w ]*|\W+', text)
39
40    # print(words_or_symbols_list)
41
42    for words_or_symbols in words_or_symbols_list:
43        if utils.is_words(words_or_symbols):
44            outputs.append(_add_accent(words_or_symbols))
45        else:
46            outputs.append(words_or_symbols)
47        # print(outputs)
48    output_text = ''.join(outputs)
49
50    # restore uppercase characters
51    output_text = ''.join(c.upper() if is_upper else c
52                            for c, is_upper in zip(output_text, is_uppercase_map))
53    return output_text
54
55def _add_accent(phrase):
56    grams = list(utils.gen_ngram(phrase.lower(), n=utils.NGRAM, pad_words=utils.PAD_WORDS_INPUT))
57
58    guessed_grams = list(guess(gram) for gram in grams)
59    # print("phrase",phrase,'grams',grams,'guessed_grams',guessed_grams)
60    candidates = [Counter() for _ in range(len(guessed_grams) + utils.NGRAM - 1)]
61    for idx, gram in enumerate(guessed_grams):
62        for wid, word in enumerate(re.split(' +', gram)):
63            candidates[idx + wid].update([word])
64    output = ' '.join(c.most_common(1)[0][0] for c in candidates if c)
65    return output.strip('\x00 ')
66
67
68
69# print(add_accent('do,'))
70# print(add_accent('7.3 inch,'))
71# print(add_accent('Truoc do, tren san khau su kien SDC 2018, giam doc cao cap mang marketing san pham di dong cua Samsung, ong Justin Denison da cam tren tay nguyen mau cua thiet bi nay. Ve co ban, no chang khac gi mot chiec may tinh bang 7.3 inch, duoc cau thanh tu nhieu lop phu khac nhau nhu polyme, lop man chong soc, lop phan cuc voi do mong gan mot nua so voi the he truoc, lop kinh linh hoat va mot tam lung da nang co the bien thanh man hinh. Tat ca se duoc ket dinh bang mot loai keo cuc ben, cho phep chiec may nay co the gap lai hang tram ngan lan ma khong bi hu hong.'))
72# print(add_accent('man hinh. Tat ca se duoc ket dinh bang mot loai keo cuc ben, cho phep chiec may nay co the gap lai hang tram ngan lan ma khong bi hu hong.'))
73print(add_accent('Hom nay, bao cao cua Counterpoint Research cho thay, trong nam 2018 Apple da ban duoc khoang 35 trieu cap tai nghe khong day AirPods. Theo hang phan tich nay, AirPods hien la tai nghe khong day pho bien nhat.'))

  1import re
  2import string
  3import time
  4from contextlib import contextmanager
  5import numpy as np
  6
  7
  8
  9# maximum string length to train and predict
 10# this is set based on our ngram length break down below
 11MAXLEN = 32
 12
 13# minimum string length to consider
 14MINLEN = 3
 15
 16# how many words per ngram to consider in our model
 17NGRAM = 5
 18
 19# inverting the input generally help with accuracy
 20INVERT = True
 21
 22# mini batch size
 23BATCH_SIZE = 128
 24
 25# number of phrases set apart from training set to validate our model
 26VALIDATION_SIZE = 100000
 27
 28# using g2.2xl GPU is ~5x faster than a Macbook Pro Core i5 CPU
 29HAS_GPU = True
 30
 31PAD_WORDS_INPUT  = True
 32
 33### Ánh xạ từ không dấu sang có dấu
 34
 35ACCENTED_CHARS = {
 36	'a': u'a á à ả ã ạ â ấ ầ ẩ ẫ ậ ă ắ ằ ẳ ẵ ặ',
 37	'o': u'o ó ò ỏ õ ọ ô ố ồ ổ ỗ ộ ơ ớ ờ ở ỡ ợ',
 38	'e': u'e é è ẻ ẽ ẹ ê ế ề ể ễ ệ',
 39	'u': u'u ú ù ủ ũ ụ ư ứ ừ ử ữ ự',
 40	'i': u'i í ì ỉ ĩ ị',
 41	'y': u'y ý ỳ ỷ ỹ ỵ',
 42	'd': u'd đ',
 43}
 44
 45### Ánh xạ từ có dấu sang không dấu
 46ACCENTED_TO_BASE_CHAR_MAP = {}
 47for c, variants in ACCENTED_CHARS.items():
 48	for v in variants.split(' '):
 49		ACCENTED_TO_BASE_CHAR_MAP[v] = c
 50
 51# \x00 ký tự padding
 52
 53### Những ký tự cơ bản, bao gồm ký tự padding, các chữ cái và các chữ số
 54BASE_ALPHABET = set('\x00 _' + string.ascii_lowercase + string.digits)
 55
 56### Bộ ký tự bao gồm những ký tự cơ bản và những ký tự có dấu
 57ALPHABET = BASE_ALPHABET.union(set(''.join(ACCENTED_TO_BASE_CHAR_MAP.keys())))
 58
 59
 60def is_words(text):
 61	return re.fullmatch('\w[\w ]*', text)
 62
 63# Hàm bỏ dấu khỏi một câu
 64def remove_accent(text):
 65	""" remove accent from text """
 66	return u''.join(ACCENTED_TO_BASE_CHAR_MAP.get(char, char) for char in text)
 67
 68#hàm thêm padding vào một câu
 69def pad(phrase, maxlen):
 70	""" right pad given string with \x00 to exact "maxlen" length """
 71	return phrase + u'\x00' * (maxlen - len(phrase))
 72
 73
 74def gen_ngram(words, n=3, pad_words=True):
 75	""" gen n-grams from given phrase or list of words """
 76	if isinstance(words, str):
 77		words = re.split('\s+', words.strip())
 78
 79	if len(words) < n:
 80		if pad_words:
 81			words += ['\x00'] * (n - len(words))
 82		yield tuple(words)
 83	else:
 84		for i in range(len(words) - n + 1):
 85			yield tuple(words[i: i + n])
 86
 87def extract_phrases(text):
 88	""" extract phrases, i.e. group of continuous words, from text """
 89	return re.findall(r'\w[\w ]+', text, re.UNICODE)
 90
 91
 92@contextmanager
 93def timing(label):
 94	begin = time.monotonic()
 95	print(label, end='', flush=True)
 96	try:
 97		yield
 98	finally:
 99		duration = time.monotonic() - begin
100	print(': took {:.2f}s'.format(duration))
101
102class CharacterCodec(object):
103    def __init__(self, alphabet, maxlen):
104        self.alphabet = list(sorted(set(alphabet)))
105        self.index_alphabet = dict((c, i) for i, c in enumerate(self.alphabet))
106        self.maxlen = maxlen
107
108    def encode(self, C, maxlen=None):
109        maxlen = maxlen if maxlen else self.maxlen
110        X = np.zeros((maxlen, len(self.alphabet)))
111        for i, c in enumerate(C[:maxlen]):
112            X[i, self.index_alphabet[c]] = 1
113        return X
114
115    def try_encode(self, C, maxlen=None):
116        try:
117            return self.encode(C, maxlen)
118        except KeyError:
119            return None
120
121    def decode(self, X, calc_argmax=True):
122        if calc_argmax:
123            X = X.argmax(axis=-1)
124        return ''.join(self.alphabet[x] for x in X)