alihan/TurkishBERTweet
1
0
Fork 0
mirror of https://github.com/ViralLab/TurkishBERTweet.git synced 2023-12-19 18:19:59 +03:00
Code Issues Packages Projects Releases Wiki Activity

prep bug fix

Browse Source
This commit is contained in:
AliNajafi
2023-10-20 19:06:37 +03:00
parent 6424b0437a
commit a451d548fe
1 changed files with 3 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
Preprocessor/preprocessor.py
View File

@@ -28,9 +28,10 @@ url_extractor = URLExtract()
def url_handler(text: str):
urls = list(url_extractor.gen_urls(text))
updated_urls = [url if "http" in url else f"https://{url}" for url in urls]
updated_urls = list(
set([url if "http" in url else f"https://{url}" for url in urls])
)
domains = [urllib.parse.urlparse(url_text).netloc for url_text in updated_urls]
domains = list(set(domains))
for i in range(len(domains)):
text = text.replace(urls[i], f" <http> {domains[i]} </http> ")
return text