diff --git a/Dockerfile b/Dockerfile old mode 100644 new mode 100755 diff --git a/LICENSE b/LICENSE old mode 100644 new mode 100755 diff --git a/README.md b/README.md old mode 100644 new mode 100755 index 5dfe852..7eb24ef --- a/README.md +++ b/README.md @@ -46,6 +46,8 @@ Profile images **will not** be blurred. If you want them to be blurred, turn tha - Find profile using multiple techniques (HTTPS library & Webdriver) - Multi layers detections (OCR, normal, advanced & special) - Metadata & Patterns extraction (Added from Qeeqbox osint project) +- Force-directed Graph for Metadata (Needs ExtractPatterns) +- Auto-flirtation to unnecessary output - Search engine lookup (Google API - optional) - Custom search queries (Google API & DuckDuckGo API - optional) - Profile screenshot, title, info and website description diff --git a/README.rst b/README.rst old mode 100644 new mode 100755 diff --git a/app.js b/app.js old mode 100644 new mode 100755 index c5afb69..ed05be5 --- a/app.js +++ b/app.js @@ -86,9 +86,10 @@ var specialScan = require("./modules/special-scan.js") var externalApis = require("./modules/external-apis.js") var stringAnalysis = require("./modules/string-analysis.js") var nameAnalysis = require("./modules/name-analysis.js") +var visualize = require("./modules/visualize.js") var app = express(); - +app.set('etag', false) app.use(express.urlencoded({ extended: true })); @@ -239,10 +240,17 @@ app.post("/analyze_string", async function(req, res, next) { var custom_search = [] var logs = "" var fast = false + var graph = { + "nodes": [], + "links": [] + } + if (req.body.string == null || req.body.string == "") { res.json("Error"); } else { req.body.uuid = req.body.uuid.replace(/[^a-zA-Z0-9\-]+/g, ''); + helper.log_to_file_queue(req.body.uuid, "[Setting] Log file name: " + req.body.uuid) + helper.log_to_file_queue(req.body.uuid, "[Setting] Username: " + req.body.string) if (req.body.option.includes("FindUserProfilesFast") || req.body.option.includes("GetUserProfilesFast")) { fast = true helper.log_to_file_queue(req.body.uuid, "[Starting] Checking user profiles normal") @@ -363,7 +371,27 @@ app.post("/analyze_string", async function(req, res, next) { Object.keys(all_words).forEach((key) => (all_words[key].length == 0) && delete all_words[key]); } - logs = fs.readFileSync(helper.get_log_file(req.body.uuid), 'utf8'); + if (req.body.option.includes("NetworkGraph")) { + if ('data' in user_info_normal) { + if (user_info_normal.data.length > 0) { + if (req.body.option.includes("ExtractMetadata")) + { + helper.log_to_file_queue(req.body.uuid, "[Starting] Network Graph") + graph = await visualize.visualize_force_graph(req.body.string, user_info_normal.data, "fast") + helper.log_to_file_queue(req.body.uuid, "[Done] Network Graph") + } + else{ + helper.log_to_file_queue(req.body.uuid, "[Warning] NetworkGraph needs ExtractMetadata") + } + } + } + } + + try { + logs = fs.readFileSync(helper.get_log_file(req.body.uuid), 'utf8'); + } catch { + + } res.json({ info, @@ -375,6 +403,7 @@ app.post("/analyze_string", async function(req, res, next) { user_info_special: user_info_special, names_origins: names_origins, custom_search: custom_search, + graph: graph, logs: logs }); } @@ -386,12 +415,16 @@ app.use((err, req, res, next) => { res.json("Error"); }); +app.use((req, res, next) => { + res.set('Cache-Control', 'no-store') + next() +}) + process.on('uncaughtException', function(err) { helper.verbose && console.log(" --- Uncaught Error ---") helper.verbose && console.log(pe.render(err)); }) - process.on('unhandledRejection', function(err) { helper.verbose && console.log(" --- Uncaught Rejection ---") helper.verbose && console.log(pe.render(err)); @@ -584,7 +617,7 @@ if (argv.cli) { if (argv.list) { list_all_websites(); } else if (argv.mode == "fast") { - if (argv.usernmae != "" && argv.websites != "") { + if (argv.username != "" && argv.websites != "") { check_user_cli(argv) } } diff --git a/app.py b/app.py old mode 100644 new mode 100755 diff --git a/auto_pip.sh b/auto_pip.sh old mode 100644 new mode 100755 diff --git a/clean-up.logs b/clean-up.logs old mode 100644 new mode 100755 diff --git a/data/dict.json b/data/dict.json old mode 100644 new mode 100755 diff --git a/data/languages.json b/data/languages.json old mode 100644 new mode 100755 diff --git a/data/names.json b/data/names.json old mode 100644 new mode 100755 diff --git a/data/site_updated.json b/data/site_updated.json old mode 100644 new mode 100755 diff --git a/data/sites.json b/data/sites.json old mode 100644 new mode 100755 diff --git a/data/sites.json_new b/data/sites.json_new old mode 100644 new mode 100755 diff --git a/docker-compose.yml b/docker-compose.yml old mode 100644 new mode 100755 diff --git a/info b/info old mode 100644 new mode 100755 index 925a842..1001f6d --- a/info +++ b/info @@ -1,4 +1,4 @@ -{"version":"2021.V.2.28", +{"version":"2021.V.2.29", "build":"pass", "test":"pass", "grid_test":"pass", @@ -8,7 +8,7 @@ "linux":"pass", "windows":"pass", "docker":"pass", - "full_scan":"15 workers < 23secs", + "full_scan":"15 workers < 24secs", "max_retries":"3", "awaiting_verification":"22", - "auto_testing":"9dafa826-140a-4880-b2e8-ceba9cf88a70"} + "auto_testing":"c6d2d8a6-74f9-4416-8ab5-bbe760ef0e47"} diff --git a/modules/engine.js b/modules/engine.js old mode 100644 new mode 100755 diff --git a/modules/external-apis.js b/modules/external-apis.js old mode 100644 new mode 100755 diff --git a/modules/extraction.js b/modules/extraction.js old mode 100644 new mode 100755 index 8bbae5a..a5cc556 --- a/modules/extraction.js +++ b/modules/extraction.js @@ -1,6 +1,6 @@ var helper = require('./helper.js'); var cheerio = require('cheerio'); - +var strings_meta = new RegExp('regionsAllowed|width|height|color|rgba\\(', 'i') async function extract_metadata(site, source) { try { var $ = cheerio.load(source); @@ -9,10 +9,11 @@ async function extract_metadata(site, source) { var temp_metadata_for_checking = [] Object.keys(meta).forEach(function(key) { if (meta[key].attribs) { - if (!temp_metadata_for_checking.includes(meta[key].attribs)) { + if (!temp_metadata_for_checking.includes(meta[key].attribs) && !strings_meta.test(JSON.stringify(meta[key].attribs))) { temp_metadata_for_checking.push(meta[key].attribs) var temp_dict = {} var add = true + if (meta[key].attribs.property) { temp_dict["property"] = meta[key].attribs.property } diff --git a/modules/fast-scan.js b/modules/fast-scan.js old mode 100644 new mode 100755 diff --git a/modules/helper.js b/modules/helper.js old mode 100644 new mode 100755 diff --git a/modules/name-analysis.js b/modules/name-analysis.js old mode 100644 new mode 100755 diff --git a/modules/slow-scan.js b/modules/slow-scan.js old mode 100644 new mode 100755 diff --git a/modules/special-scan.js b/modules/special-scan.js old mode 100644 new mode 100755 diff --git a/modules/string-analysis.js b/modules/string-analysis.js old mode 100644 new mode 100755 diff --git a/modules/visualize.js b/modules/visualize.js new file mode 100755 index 0000000..315198f --- /dev/null +++ b/modules/visualize.js @@ -0,0 +1,85 @@ +var helper = require('./helper.js'); + +async function visualize_force_graph(username, detected, type) { + graph = { + "nodes": [], + "links": [] + } + + try { + filter_items = "good" + temp_filtered = [] + temp_filtered = detected.filter(item => filter_items.includes(item.status)) + if (temp_filtered.length > 0) { + graph.nodes.push({ + "id": username + }) + temp_filtered.forEach(site => { + graph.nodes.push({ + "id": site.link + }) + graph.links.push({ + "source": username, + "target": site.link + }) + if ("metadata" in site){ + if (site.metadata.length > 0) { + site.metadata.forEach(meta => { + if ("content" in meta) { + temp_string = "" + if ("name" in meta) { + temp_string = meta.name + " -> " + meta.content + } else if ("itemprop" in meta) { + temp_string = meta.itemprop + " -> " + meta.content + } else if ("property" in meta) { + temp_string = meta.property + " -> " + meta.content + } + + if (temp_string.length > 50) { + temp_string = temp_string.substring(0, 50).replace(/\r?\n|\r/g, "") + ".." + } else { + temp_string = temp_string.replace(/\r?\n|\r/g, "") + } + + if (temp_string != "" && temp_string.length > 0) { + var index = undefined; + graph.nodes.some(function(item, i) { + if (item.id == temp_string) { + index = i; + return true; + } + }); + + if (index != undefined) { + + } else { + graph.nodes.push({ + "id": temp_string + }) + } + + if (graph.links.some(v => v.source == temp_string && v.target == site.link) || graph.links.some(v => v.source == site.link && v.target == temp_string)) { + + } else { + graph.links.push({ + "source": site.link, + "target": temp_string + }) + } + } + } + }) + } + } + }); + } + } catch { + + } + + return graph +} + +module.exports = { + visualize_force_graph +} diff --git a/package-lock.json b/package-lock.json old mode 100644 new mode 100755 diff --git a/package.json b/package.json old mode 100644 new mode 100755 diff --git a/public/app.html b/public/app.html old mode 100644 new mode 100755 index d940526..1d78537 --- a/public/app.html +++ b/public/app.html @@ -19,7 +19,7 @@ - +