alihan/social-analyzer
1
0
Fork 0
mirror of https://github.com/qeeqbox/social-analyzer.git synced 2021-05-13 03:10:39 +03:00
Code Issues Projects Releases Wiki Activity

[auto-b] enable text (users requests) - ref 9bc0a3be & d6b8c4c3

Browse Source
This commit is contained in:
G
2021-04-28 21:39:39 -07:00
committed by GitHub
parent d6ed06f04c
commit efa2e722bd
2 changed files with 15 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
app.py
View File

@@ -50,7 +50,7 @@ SITES_PATH = path.join(path.dirname(__file__), "data", "sites.json")
LANGUAGES_PATH = path.join(path.dirname(__file__), "data", "languages.json")
STRINGS_PAGES = recompile('captcha-info|Please enable cookies|Completing the CAPTCHA', IGNORECASE)
STRINGS_TITLES = recompile('not found|blocked|attention required|cloudflare', IGNORECASE)
STRINGS_META = recompile(r'regionsAllowed|width|height|color|rgba\(|charset|viewport|refresh|equiv', IGNORECASE)
STRINGS_META = recompile('regionsAllowed|width|height|color|rgba\(|charset|viewport|refresh|equiv', IGNORECASE)
LANGUAGES_JSON = {}
WORKERS = 15
CUSTOM_MESSAGE = 51
@@ -80,7 +80,7 @@ def clean_up_item(in_object, keys_str):
del in_object["image"]
if keys_str == "" or keys_str is None:
with suppress(Exception):
del in_object["text"]
pass
else:
for key in in_object.copy():
if key not in keys_str:
@@ -372,8 +372,14 @@ def find_username_normal(req):
soup = BeautifulSoup(content, "html.parser")
with suppress(Exception):
[tag.extract() for tag in soup(["head", "title", "style", "script", "[document]"])]
temp_profile["text"] = soup.getText()
temp_text_arr = []
temp_text_list = []
soup = BeautifulSoup(content, "html.parser")
for item in soup.stripped_strings:
if item not in temp_text_list:
temp_text_list.append(item)
temp_text_arr.append(repr(item).replace("'", ""))
temp_profile["text"] = " ".join(temp_text_arr)
temp_profile["text"] = resub(r"\s\s+", " ", temp_profile["text"])
with suppress(Exception):
temp_profile["language"] = get_language_by_parsing(source)
@@ -558,7 +564,7 @@ def check_user_cli(argv):
item = clean_up_item(item, argv.options)
temp_detected["detected"].append(item)
else:
item = delete_keys(item, ["found", "rate", "status", "method", "good", "extracted", "metadata"])
item = delete_keys(item, ["found", "rate", "status", "method", "good", "text","extracted", "metadata"])
item = clean_up_item(item, argv.options)
temp_detected["unknown"].append(item)
elif item["method"] == "find":
@@ -567,7 +573,7 @@ def check_user_cli(argv):
item = clean_up_item(item, argv.options)
temp_detected["detected"].append(item)
elif item["method"] == "get":
item = delete_keys(item, ["found", "rate", "status", "method", "good", "extracted", "metadata"])
item = delete_keys(item, ["found", "rate", "status", "method", "good", "text","extracted", "metadata"])
item = clean_up_item(item, argv.options)
temp_detected["unknown"].append(item)
else:

2
setup.py
View File

@@ -12,7 +12,7 @@ setup(
author_email='gigaqeeq@gmail.com',
description="API, CLI & Web App for analyzing & finding a person's profile across 300+ social media websites (Detections are updated regularly)",
long_description=long_description,
version='0.18',
version='0.21',
license='AGPL-3.0',
url='https://github.com/qeeqbox/social-analyzer',
packages=['social-analyzer'],