alihan/clean-text-nlp-preprocessing
1
0
Fork 0
mirror of https://github.com/jfilter/clean-text.git synced 2021-09-19 22:32:58 +03:00
Code Issues Projects Releases Wiki Activity

remove obsolete variable

Browse Source
This commit is contained in:
Johannes Filter
2020-10-18 01:27:22 +02:00
parent 355bc7d0bc
commit 375e049f5b
1 changed files with 1 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
cleantext/clean.py
View File

@@ -20,12 +20,9 @@ log = logging.getLogger()
try:
from unidecode import unidecode
using_unidecode = True
except:
from unicodedata import normalize
using_unidecode = False
unidecode = lambda x: normalize("NFD", x).encode("ASCII", "ignore").decode("utf-8")
log.warning(
"Since the GPL-licensed package `unidecode` is not installed, using Python's `unicodedata` package which yields worse results."
@@ -277,7 +274,7 @@ def clean(
else:
text = replace_punct(text, replace_with_punct)
if no_emoji and (not to_ascii or not using_unidecode):
if no_emoji and not to_ascii:
text = remove_emoji(text)
if lower: