alihan/clean-text-nlp-preprocessing
1
0
Fork 0
mirror of https://github.com/jfilter/clean-text.git synced 2021-09-19 22:32:58 +03:00
Code Issues Projects Releases Wiki Activity

make to_ascii work when unidecode isn't installed

Browse Source
This commit is contained in:
Johannes Filter
2020-10-18 01:17:45 +02:00
parent 4a2de47a59
commit 355bc7d0bc
1 changed files with 3 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
cleantext/clean.py
View File

@@ -26,7 +26,7 @@ except:
using_unidecode = False
unidecode = lambda x: normalize("NFKD", x)
unidecode = lambda x: normalize("NFD", x).encode("ASCII", "ignore").decode("utf-8")
log.warning(
"Since the GPL-licensed package `unidecode` is not installed, using Python's `unicodedata` package which yields worse results."
)
@@ -78,8 +78,7 @@ def to_ascii_unicode(text, lang="en", no_emoji=False):
# normalize quotes before since this improves transliteration quality
text = fix_strange_quotes(text)
# for background: https://github.com/jfilter/clean-text/issues/11
if using_unidecode and not no_emoji:
if not no_emoji:
text = demojize(text, use_aliases=True)
lang = lang.lower()
@@ -93,7 +92,7 @@ def to_ascii_unicode(text, lang="en", no_emoji=False):
if lang == "de":
text = save_replace(text, lang=lang, back=True)
if using_unidecode and not no_emoji:
if not no_emoji:
text = emojize(text, use_aliases=True)
return text