alihan/clean-text-nlp-preprocessing
1
0
Fork 0
mirror of https://github.com/jfilter/clean-text.git synced 2021-09-19 22:32:58 +03:00
Code Issues Projects Releases Wiki Activity

formatting

Browse Source
This commit is contained in:
Johannes Filter
2021-04-12 20:04:31 +02:00
parent 5fcf48f27b
commit 910df7bc6c
1 changed files with 4 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
cleantext/clean.py
View File

@@ -190,7 +190,7 @@ def remove_punct(text):
def remove_emoji(text):
for x in UNICODE_EMOJI['en']:
for x in UNICODE_EMOJI["en"]:
if x in text:
text = text.replace(x, "")
return text
@@ -293,6 +293,8 @@ def clean(
text = text.lower()
if normalize_whitespace:
text = _normalize_whitespace(text, no_line_breaks, strip_lines, keep_two_line_breaks)
text = _normalize_whitespace(
text, no_line_breaks, strip_lines, keep_two_line_breaks
)
return text