alihan/social-analyzer
1
0
Fork 0
mirror of https://github.com/qeeqbox/social-analyzer.git synced 2021-05-13 03:10:39 +03:00
Code Issues Projects Releases Wiki Activity

[auto-b] enable text (users requests) - ref 9bc0a3be & d6b8c4c3

Browse Source
This commit is contained in:
G
2021-04-28 21:39:39 -07:00
committed by GitHub
parent d6ed06f04c
commit efa2e722bd
2 changed files with 15 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
app.py
View File

@@ -50,7 +50,7 @@ SITES_PATH = path.join(path.dirname(__file__), "data", "sites.json")
LANGUAGES_PATH = path.join(path.dirname(__file__), "data", "languages.json") LANGUAGES_PATH = path.join(path.dirname(__file__), "data", "languages.json")
STRINGS_PAGES = recompile('captcha-info|Please enable cookies|Completing the CAPTCHA', IGNORECASE) STRINGS_PAGES = recompile('captcha-info|Please enable cookies|Completing the CAPTCHA', IGNORECASE)
STRINGS_TITLES = recompile('not found|blocked|attention required|cloudflare', IGNORECASE) STRINGS_TITLES = recompile('not found|blocked|attention required|cloudflare', IGNORECASE)
STRINGS_META = recompile(r'regionsAllowed|width|height|color|rgba\(|charset|viewport|refresh|equiv', IGNORECASE) STRINGS_META = recompile('regionsAllowed|width|height|color|rgba\(|charset|viewport|refresh|equiv', IGNORECASE)
LANGUAGES_JSON = {} LANGUAGES_JSON = {}
WORKERS = 15 WORKERS = 15
CUSTOM_MESSAGE = 51 CUSTOM_MESSAGE = 51
@@ -80,7 +80,7 @@ def clean_up_item(in_object, keys_str):
del in_object["image"] del in_object["image"]
if keys_str == "" or keys_str is None: if keys_str == "" or keys_str is None:
with suppress(Exception): with suppress(Exception):
del in_object["text"] pass
else: else:
for key in in_object.copy(): for key in in_object.copy():
if key not in keys_str: if key not in keys_str:
@@ -273,7 +273,7 @@ def find_username_normal(req):
with suppress(Exception): with suppress(Exception):
session = Session() session = Session()
session.headers.update(headers) session.headers.update(headers)
response = session.get(site["url"].replace("{username}", username), timeout=5, verify=False) response = session.get(site["url"].replace("{username}", username),timeout=5, verify=False)
source = response.text source = response.text
content = response.content content = response.content
answer = dict((k.lower(), v.lower()) for k, v in response.headers.items()) answer = dict((k.lower(), v.lower()) for k, v in response.headers.items())
@@ -372,8 +372,14 @@ def find_username_normal(req):
soup = BeautifulSoup(content, "html.parser") soup = BeautifulSoup(content, "html.parser")
with suppress(Exception): with suppress(Exception):
[tag.extract() for tag in soup(["head", "title", "style", "script", "[document]"])] temp_text_arr = []
temp_profile["text"] = soup.getText() temp_text_list = []
soup = BeautifulSoup(content, "html.parser")
for item in soup.stripped_strings:
if item not in temp_text_list:
temp_text_list.append(item)
temp_text_arr.append(repr(item).replace("'", ""))
temp_profile["text"] = " ".join(temp_text_arr)
temp_profile["text"] = resub(r"\s\s+", " ", temp_profile["text"]) temp_profile["text"] = resub(r"\s\s+", " ", temp_profile["text"])
with suppress(Exception): with suppress(Exception):
temp_profile["language"] = get_language_by_parsing(source) temp_profile["language"] = get_language_by_parsing(source)
@@ -558,7 +564,7 @@ def check_user_cli(argv):
item = clean_up_item(item, argv.options) item = clean_up_item(item, argv.options)
temp_detected["detected"].append(item) temp_detected["detected"].append(item)
else: else:
item = delete_keys(item, ["found", "rate", "status", "method", "good", "extracted", "metadata"]) item = delete_keys(item, ["found", "rate", "status", "method", "good", "text","extracted", "metadata"])
item = clean_up_item(item, argv.options) item = clean_up_item(item, argv.options)
temp_detected["unknown"].append(item) temp_detected["unknown"].append(item)
elif item["method"] == "find": elif item["method"] == "find":
@@ -567,7 +573,7 @@ def check_user_cli(argv):
item = clean_up_item(item, argv.options) item = clean_up_item(item, argv.options)
temp_detected["detected"].append(item) temp_detected["detected"].append(item)
elif item["method"] == "get": elif item["method"] == "get":
item = delete_keys(item, ["found", "rate", "status", "method", "good", "extracted", "metadata"]) item = delete_keys(item, ["found", "rate", "status", "method", "good", "text","extracted", "metadata"])
item = clean_up_item(item, argv.options) item = clean_up_item(item, argv.options)
temp_detected["unknown"].append(item) temp_detected["unknown"].append(item)
else: else:
@@ -665,4 +671,4 @@ if __name__ == "__main__":
list_all_websites() list_all_websites()
elif ARGV.mode == "fast": elif ARGV.mode == "fast":
if ARGV.username != "" and ARGV.websites != "": if ARGV.username != "" and ARGV.websites != "":
check_user_cli(ARGV) check_user_cli(ARGV)

2
setup.py
View File

@@ -12,7 +12,7 @@ setup(
author_email='gigaqeeq@gmail.com', author_email='gigaqeeq@gmail.com',
description="API, CLI & Web App for analyzing & finding a person's profile across 300+ social media websites (Detections are updated regularly)", description="API, CLI & Web App for analyzing & finding a person's profile across 300+ social media websites (Detections are updated regularly)",
long_description=long_description, long_description=long_description,
version='0.18', version='0.21',
license='AGPL-3.0', license='AGPL-3.0',
url='https://github.com/qeeqbox/social-analyzer', url='https://github.com/qeeqbox/social-analyzer',
packages=['social-analyzer'], packages=['social-analyzer'],