commit ac0be37480dc0e90b5bd08f789acaf8b717a9a98 Author: Soxoj Date: Wed Jan 8 09:51:07 2020 +0300 first commit diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..9978ae9 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +.git/ +.vscode/ +screenshot/ +tests/ +*.txt +!/requirements.txt +venv/ + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3bf7f81 --- /dev/null +++ b/.gitignore @@ -0,0 +1,29 @@ +# Virtual Environment +venv/ + +# Editor Configurations +.vscode/ +.idea/ + +# Python +__pycache__/ + +# Pip +src/ + +# Jupyter Notebook +.ipynb_checkpoints +*.ipynb + +# Output files, except requirements.txt +*.txt +!requirements.txt + +# Comma-Separated Values (CSV) Reports +*.csv + +# Excluded sites list +tests/.excluded_sites + +# MacOS Folder Metadata File +.DS_Store diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..5af2c73 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +FROM python:3.7-alpine as build +WORKDIR /wheels +RUN apk add --no-cache \ + g++ \ + gcc \ + git \ + libxml2 \ + libxml2-dev \ + libxslt-dev \ + linux-headers +COPY requirements.txt /opt/maigret/ +RUN pip3 wheel -r /opt/maigret/requirements.txt + + +FROM python:3.7-alpine +WORKDIR /opt/maigret +ARG VCS_REF +ARG VCS_URL="https://gitlab.com/soxoj/maigret" +LABEL org.label-schema.vcs-ref=$VCS_REF \ + org.label-schema.vcs-url=$VCS_URL +COPY --from=build /wheels /wheels +COPY . /opt/maigret/ +RUN pip3 install -r requirements.txt -f /wheels \ + && rm -rf /wheels \ + && rm -rf /root/.cache/pip/* + +ENTRYPOINT ["python", "maigret.py"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..efc6caa --- /dev/null +++ b/LICENSE @@ -0,0 +1,45 @@ +MIT License + +Copyright (c) 2019 Soxoj + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +------------------------------------------------------------------------------- + +MIT License + +Copyright (c) 2019 Sherlock Project + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..2b0d27a --- /dev/null +++ b/README.md @@ -0,0 +1,54 @@ +# Maigret + +

+ +

+ +The Commissioner Jules Maigret is a fictional French police detective, created by Georges Simenon. His investigation method is based on understanding the personality of different people and their interactions. + +## About + +Purpose of Maigret - **collect a dossier on a person by username only**, checking for accounts on a huge number of sites. + +This is a [sherlock](https://github.com/sherlock-project/) fork with cool features under heavy development. +*Don't forget to regularly update source code from repo*. + +Currently supported >1300 sites ([full list](/sites.md)). + +## Main features + +* Profile pages parsing, [extracting](https://github.com/soxoj/socid_extractor) personal info, links to other profiles, etc. +* Recursive search by new usernames found +* Search by tags (site categories, countries) +* Censorship and captcha detection +* Very few false positives + +## Installation + +**NOTE**: Python 3.7 or higher and pip is required. + +**Python 3.8 is recommended.** + +```bash +# clone the repo and change directory +$ git clone https://git.rip/soxoj/maigret && cd maigret + +# install the requirements +$ python3 -m pip install -r requirements.txt +``` + +## Demo with page parsing and recursive username search + +```bash +python3 maigret alexaimephotographycars +``` + +![animation of recursive search](./static/recursive_search.svg) + +[Full output](./static/recursive_search.md) + +## License + +MIT © [Maigret](https://git.rip/soxoj/maigret)
+MIT © [Sherlock Project](https://github.com/sherlock-project/)
+Original Creator of Sherlock Project - [Siddharth Dushantha](https://github.com/sdushantha) diff --git a/maigret/__init__.py b/maigret/__init__.py new file mode 100644 index 0000000..b0894c8 --- /dev/null +++ b/maigret/__init__.py @@ -0,0 +1,5 @@ +"""Sherlock Module + +This module contains the main logic to search for usernames at social +networks. +""" diff --git a/maigret/__main__.py b/maigret/__main__.py new file mode 100644 index 0000000..749a4e1 --- /dev/null +++ b/maigret/__main__.py @@ -0,0 +1,15 @@ +#! /usr/bin/env python3 + +""" +Maigret (Sherlock fork): Find Usernames Across Social Networks Module + +This module contains the main logic to search for usernames at social +networks. +""" + +import asyncio +import maigret + + +if __name__ == "__main__": + asyncio.run(maigret.main()) diff --git a/maigret/maigret.py b/maigret/maigret.py new file mode 100755 index 0000000..3591afd --- /dev/null +++ b/maigret/maigret.py @@ -0,0 +1,867 @@ +#! /usr/bin/env python3 + +""" +Maigret main module +""" + +import asyncio +import csv +import http.cookiejar as cookielib +import json +import logging +import os +import platform +import re +import ssl +import sys +from argparse import ArgumentParser, RawDescriptionHelpFormatter +from http.cookies import SimpleCookie + +import aiohttp +import requests +from mock import Mock +from notify import QueryNotifyPrint +from result import QueryResult, QueryStatus +from sites import SitesInformation +from socid_extractor import parse, extract + +module_name = "Maigret OSINT tool" +__version__ = "0.1.0" + +supported_recursive_search_ids = ( + 'yandex_public_id', + 'gaia_id', + 'vk_id', + 'ok_id', + 'wikimapia_uid', +) + +common_errors = { + 'Attention Required! | Cloudflare': 'Cloudflare captcha', + 'Доступ ограничен': 'Rostelecom censorship', + 'document.getElementById(\'validate_form_submit\').disabled=true': 'Mail.ru captcha', + 'Verifying your browser, please wait...
DDoS Protection by Blazingfast.io': 'Blazingfast protection', + '404

Мы не нашли страницу': 'MegaFon 404 page', +} + +unsupported_characters = '#' + +cookies_file = 'cookies.txt' + + +async def get_response(request_future, error_type, social_network, logger): + html_text = None + status_code = 0 + + error_text = "General Unknown Error" + expection_text = None + + try: + response = await request_future + + status_code = response.status + response_content = await response.content.read() + charset = response.charset or 'utf-8' + decoded_content = response_content.decode(charset, 'ignore') + html_text = decoded_content + + if status_code > 0: + error_text = None + + logger.debug(html_text) + + except asyncio.TimeoutError as errt: + error_text = "Timeout Error" + expection_text = str(errt) + except (ssl.SSLCertVerificationError, ssl.SSLError) as err: + error_text = "SSL Error" + expection_text = str(err) + except aiohttp.client_exceptions.ClientConnectorError as err: + error_text = "Error Connecting" + expection_text = str(err) + except aiohttp.http_exceptions.BadHttpMessage as err: + error_text = "HTTP Error" + expection_text = str(err) + except Exception as err: + logger.warning(f'Unhandled error while requesting {social_network}: {err}') + logger.debug(err, exc_info=True) + error_text = "Some Error" + expection_text = str(err) + + # TODO: return only needed information + return html_text, status_code, error_text, expection_text + + +async def update_site_data_from_response(site, site_data, site_info, semaphore, logger): + async with semaphore: + future = site_info.get('request_future') + if not future: + # ignore: search by incompatible id type + return + + error_type = site_info['errorType'] + site_data[site]['resp'] = await get_response(request_future=future, + error_type=error_type, + social_network=site, + logger=logger) + + +# TODO: move info separate module +def detect_error_page(html_text, status_code, fail_flags, ignore_403): + # Detect service restrictions such as a country restriction + for flag, msg in fail_flags.items(): + if flag in html_text: + return 'Some site error', msg + + # Detect common restrictions such as provider censorship and bot protection + for flag, msg in common_errors.items(): + if flag in html_text: + return 'Error', msg + + # Detect common site errors + if status_code == 403 and not ignore_403: + return 'Access denied', 'Access denied, use proxy/vpn' + elif status_code >= 500: + return f'Error {status_code}', f'Site error {status_code}' + + return None, None + + +async def maigret(username, site_data, query_notify, logger, + proxy=None, timeout=None, recursive_search=False, + id_type='username', tags=None, debug=False, forced=False, + max_connections=100): + """Main search func + + Checks for existence of username on various social media sites. + + Keyword Arguments: + username -- String indicating username that report + should be created against. + site_data -- Dictionary containing all of the site data. + query_notify -- Object with base type of QueryNotify(). + This will be used to notify the caller about + query results. + proxy -- String indicating the proxy URL + timeout -- Time in seconds to wait before timing out request. + Default is no timeout. + recursive_search -- Search for other usernames in website pages & recursive search by them. + + Return Value: + Dictionary containing results from report. Key of dictionary is the name + of the social network site, and the value is another dictionary with + the following keys: + url_main: URL of main site. + url_user: URL of user on site (if account exists). + status: QueryResult() object indicating results of test for + account existence. + http_status: HTTP status code of query which checked for existence on + site. + response_text: Text that came back from request. May be None if + there was an HTTP error when checking for existence. + """ + + # Notify caller that we are starting the query. + if tags is None: + tags = set() + query_notify.start(username, id_type) + + # TODO: connector + connector = aiohttp.TCPConnector(ssl=False) + session = aiohttp.ClientSession(connector=connector) + + # Results from analysis of all sites + results_total = {} + + # First create futures for all requests. This allows for the requests to run in parallel + for social_network, net_info in site_data.items(): + if net_info.get('type', 'username') != id_type: + continue + + site_tags = set(net_info.get('tags', [])) + if tags: + if not set(tags).intersection(site_tags): + continue + + if 'disabled' in net_info and net_info['disabled'] and not forced: + continue + + # Results from analysis of this specific site + results_site = {} + + # Record URL of main site + results_site['url_main'] = net_info.get("urlMain") + + headers = { + 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11.1; rv:55.0) Gecko/20100101 Firefox/55.0', + } + + if "headers" in net_info: + # Override/append any extra headers required by a given site. + headers.update(net_info["headers"]) + + # URL of user on site (if it exists) + url = net_info.get('url').format(username) + + # Don't make request if username is invalid for the site + regex_check = net_info.get("regexCheck") + if regex_check and re.search(regex_check, username) is None: + # No need to do the check at the site: this user name is not allowed. + results_site['status'] = QueryResult(username, + social_network, + url, + QueryStatus.ILLEGAL) + results_site["url_user"] = "" + results_site['http_status'] = "" + results_site['response_text'] = "" + query_notify.update(results_site['status']) + else: + # URL of user on site (if it exists) + results_site["url_user"] = url + url_probe = net_info.get("urlProbe") + if url_probe is None: + # Probe URL is normal one seen by people out on the web. + url_probe = url + else: + # There is a special URL for probing existence separate + # from where the user profile normally can be found. + url_probe = url_probe.format(username) + + if net_info["errorType"] == 'status_code' and net_info.get("request_head_only", True): + # In most cases when we are detecting by status code, + # it is not necessary to get the entire body: we can + # detect fine with just the HEAD response. + request_method = session.head + else: + # Either this detect method needs the content associated + # with the GET response, or this specific website will + # not respond properly unless we request the whole page. + request_method = session.get + + if net_info["errorType"] == "response_url": + # Site forwards request to a different URL if username not + # found. Disallow the redirect so we can capture the + # http status from the original URL request. + allow_redirects = False + else: + # Allow whatever redirect that the site wants to do. + # The final result of the request will be what is available. + allow_redirects = True + + # TODO: cookies using + def parse_cookies(cookies_str): + cookies = SimpleCookie() + cookies.load(cookies_str) + return {key: morsel.value for key, morsel in cookies.items()} + + if os.path.exists(cookies_file): + cookies_obj = cookielib.MozillaCookieJar(cookies_file) + cookies_obj.load(ignore_discard=True, ignore_expires=True) + else: + cookies_obj = [] + + # This future starts running the request in a new thread, doesn't block the main thread + if proxy is not None: + proxies = {"http": proxy, "https": proxy} + future = request_method(url=url_probe, headers=headers, + proxies=proxies, + allow_redirects=allow_redirects, + timeout=timeout, + ) + else: + future = request_method(url=url_probe, headers=headers, + allow_redirects=allow_redirects, + timeout=timeout, + ) + + # Store future in data for access later + net_info["request_future"] = future + + # Add this site's results into final dictionary with all of the other results. + results_total[social_network] = results_site + + # TODO: move into top-level function + + sem = asyncio.Semaphore(max_connections) + + tasks = [] + for social_network, net_info in site_data.items(): + future = asyncio.ensure_future(update_site_data_from_response(social_network, site_data, net_info, sem, logger)) + tasks.append(future) + await asyncio.gather(*tasks) + await session.close() + + # TODO: split to separate functions + for social_network, net_info in site_data.items(): + + # Retrieve results again + results_site = results_total.get(social_network) + if not results_site: + continue + + # Retrieve other site information again + url = results_site.get("url_user") + logger.debug(url) + + status = results_site.get("status") + if status is not None: + # We have already determined the user doesn't exist here + continue + + # Get the expected error type + error_type = net_info["errorType"] + + # Get the failure messages and comments + failure_errors = net_info.get("errors", {}) + + # TODO: refactor + resp = net_info.get('resp') + if not resp: + logger.error(f'No response for {social_network}') + continue + + html_text, status_code, error_text, expection_text = resp + + # TODO: add elapsed request time counting + response_time = None + + if debug: + with open('debug.txt', 'a') as f: + status = status_code or 'No response' + f.write(f'url: {url}\nerror: {str(error_text)}\nr: {status}\n') + if html_text: + f.write(f'code: {status}\nresponse: {str(html_text)}\n') + + if status_code and not error_text: + error_text, site_error_text = detect_error_page(html_text, status_code, failure_errors, + 'ignore_403' in net_info) + + # presense flags + # True by default + presense_flags = net_info.get("presenseStrs", []) + is_presense_detected = html_text and all( + [(presense_flag in html_text) for presense_flag in presense_flags]) or not presense_flags + + if error_text is not None: + logger.debug(error_text) + result = QueryResult(username, + social_network, + url, + QueryStatus.UNKNOWN, + query_time=response_time, + context=error_text) + elif error_type == "message": + absence_flags = net_info.get("errorMsg") + is_absence_flags_list = isinstance(absence_flags, list) + absence_flags_set = set(absence_flags) if is_absence_flags_list else {absence_flags} + # Checks if the error message is in the HTML + is_absence_detected = any([(absence_flag in html_text) for absence_flag in absence_flags_set]) + if not is_absence_detected and is_presense_detected: + result = QueryResult(username, + social_network, + url, + QueryStatus.CLAIMED, + query_time=response_time) + else: + result = QueryResult(username, + social_network, + url, + QueryStatus.AVAILABLE, + query_time=response_time) + elif error_type == "status_code": + # Checks if the status code of the response is 2XX + if (not status_code >= 300 or status_code < 200) and is_presense_detected: + result = QueryResult(username, + social_network, + url, + QueryStatus.CLAIMED, + query_time=response_time) + else: + result = QueryResult(username, + social_network, + url, + QueryStatus.AVAILABLE, + query_time=response_time) + elif error_type == "response_url": + # For this detection method, we have turned off the redirect. + # So, there is no need to check the response URL: it will always + # match the request. Instead, we will ensure that the response + # code indicates that the request was successful (i.e. no 404, or + # forward to some odd redirect). + if 200 <= status_code < 300 and is_presense_detected: + result = QueryResult(username, + social_network, + url, + QueryStatus.CLAIMED, + query_time=response_time) + else: + result = QueryResult(username, + social_network, + url, + QueryStatus.AVAILABLE, + query_time=response_time) + else: + # It should be impossible to ever get here... + raise ValueError(f"Unknown Error Type '{error_type}' for " + f"site '{social_network}'") + + extracted_ids_data = {} + + if recursive_search and result.status == QueryStatus.CLAIMED: + try: + extracted_ids_data = extract(html_text) + except Exception as e: + logger.warning(f'Error while parsing {social_network}: {e}', exc_info=True) + + if extracted_ids_data: + new_usernames = {} + for k, v in extracted_ids_data.items(): + if 'username' in k: + new_usernames[v] = 'username' + if k in supported_recursive_search_ids: + new_usernames[v] = k + + results_site['ids_usernames'] = new_usernames + result.ids_data = extracted_ids_data + + is_similar = net_info.get('similarSearch', False) + # Notify caller about results of query. + query_notify.update(result, is_similar) + + # Save status of request + results_site['status'] = result + + # Save results from request + results_site['http_status'] = status_code + results_site['is_similar'] = is_similar + # results_site['response_text'] = html_text + results_site['rank'] = net_info.get('rank', 0) + + # Add this site's results into final dictionary with all of the other results. + results_total[social_network] = results_site + + # Notify caller that all queries are finished. + query_notify.finish() + + return results_total + + +def timeout_check(value): + """Check Timeout Argument. + + Checks timeout for validity. + + Keyword Arguments: + value -- Time in seconds to wait before timing out request. + + Return Value: + Floating point number representing the time (in seconds) that should be + used for the timeout. + + NOTE: Will raise an exception if the timeout in invalid. + """ + from argparse import ArgumentTypeError + + try: + timeout = float(value) + except ValueError: + raise ArgumentTypeError(f"Timeout '{value}' must be a number.") + if timeout <= 0: + raise ArgumentTypeError(f"Timeout '{value}' must be greater than 0.0s.") + return timeout + + +async def site_self_check(site_name, site_data, logger): + query_notify = Mock() + changes = { + 'disabled': False, + } + + check_data = [ + (site_data['username_claimed'], QueryStatus.CLAIMED), + (site_data['username_unclaimed'], QueryStatus.AVAILABLE), + ] + + logger.info(f'Checking {site_name}...') + + for username, status in check_data: + results = await maigret( + username, + {site_name: site_data}, + query_notify, + logger, + timeout=30, + forced=True, + ) + # don't disable entries with other ids types + if site_name not in results: + logger.info(results) + changes['disabled'] = True + continue + site_status = results[site_name]['status'].status + if site_status != status: + if site_status == QueryStatus.UNKNOWN: + msg = site_data.get('errorMsg') + etype = site_data.get('errorType') + logger.info(f'Error while searching {username} in {site_name}: {msg}, type {etype}') + # don't disable in case of available username + if status == QueryStatus.CLAIMED: + changes['disabled'] = True + elif status == QueryStatus.CLAIMED: + logger.info(f'Not found `{username}` in {site_name}, must be claimed') + changes['disabled'] = True + else: + logger.info(f'Found `{username}` in {site_name}, must be available') + changes['disabled'] = True + + logger.info(f'Site {site_name} is okay') + return changes + + +async def self_check(json_file, logger): + sites = SitesInformation(json_file) + all_sites = {} + + def disabled_count(data): + return len(list(filter(lambda x: x.get('disabled', False), data))) + + async def update_site_data(site_name, site_data, all_sites, logger): + updates = await site_self_check(site_name, dict(site_data), logger) + all_sites[site_name].update(updates) + + for site in sites: + all_sites[site.name] = site.information + + disabled_old_count = disabled_count(all_sites.values()) + + tasks = [] + for site_name, site_data in all_sites.items(): + future = asyncio.ensure_future(update_site_data(site_name, site_data, all_sites, logger)) + tasks.append(future) + + await asyncio.gather(*tasks) + + disabled_new_count = disabled_count(all_sites.values()) + total_disabled = disabled_new_count - disabled_old_count + if total_disabled > 0: + message = 'Disabled' + else: + message = 'Enabled' + total_disabled *= -1 + print(f'{message} {total_disabled} checked sites. Run with `--info` flag to get more information') + + with open(json_file, 'w') as f: + json.dump(all_sites, f, indent=4) + + +async def main(): + version_string = f"%(prog)s {__version__}\n" + \ + f"{requests.__description__}: {requests.__version__}\n" + \ + f"Python: {platform.python_version()}" + + parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter, + description=f"{module_name} (Version {__version__})" + ) + parser.add_argument("--version", + action="version", version=version_string, + help="Display version information and dependencies." + ) + parser.add_argument("--info", + action="store_true", dest="info", default=False, + help="Display service information." + ) + parser.add_argument("--verbose", "-v", + action="store_true", dest="verbose", default=False, + help="Display extra information and metrics." + ) + parser.add_argument("-d", "--debug", + action="store_true", dest="debug", default=False, + help="Saving debugging information and sites responses in debug.txt." + ) + parser.add_argument("--rank", "-r", + action="store_true", dest="rank", default=False, + help="Present websites ordered by their Alexa.com global rank in popularity.") + parser.add_argument("--folderoutput", "-fo", dest="folderoutput", + help="If using multiple usernames, the output of the results will be saved to this folder." + ) + parser.add_argument("--output", "-o", dest="output", + help="If using single username, the output of the result will be saved to this file." + ) + parser.add_argument("--csv", + action="store_true", dest="csv", default=False, + help="Create Comma-Separated Values (CSV) File." + ) + parser.add_argument("--site", + action="append", metavar='SITE_NAME', + dest="site_list", default=None, + help="Limit analysis to just the listed sites (use several times to specify more than one)" + ) + parser.add_argument("--proxy", "-p", metavar='PROXY_URL', + action="store", dest="proxy", default=None, + help="Make requests over a proxy. e.g. socks5://127.0.0.1:1080" + ) + parser.add_argument("--json", "-j", metavar="JSON_FILE", + dest="json_file", default=None, + help="Load data from a JSON file or an online, valid, JSON file.") + parser.add_argument("--timeout", + action="store", metavar='TIMEOUT', + dest="timeout", type=timeout_check, default=10, + help="Time (in seconds) to wait for response to requests." + "Default timeout of 10.0s." + "A longer timeout will be more likely to get results from slow sites." + "On the other hand, this may cause a long delay to gather all results." + ) + parser.add_argument("--print-not-found", + action="store_true", dest="print_not_found", default=False, + help="Print sites where the username was not found." + ) + parser.add_argument("--print-errors", + action="store_true", dest="print_check_errors", default=False, + help="Print errors messages: connection, captcha, site country ban, etc." + ) + parser.add_argument("--no-color", + action="store_true", dest="no_color", default=False, + help="Don't color terminal output" + ) + parser.add_argument("--browse", "-b", + action="store_true", dest="browse", default=False, + help="Browse to all results on default bowser." + ) + parser.add_argument("--no-recursion", + action="store_true", dest="disable_recursive_search", default=False, + help="Disable parsing pages for other usernames and recursive search by them." + ) + parser.add_argument("--self-check", + action="store_true", default=False, + help="Do self check for sites and database and disable non-working ones." + ) + parser.add_argument("--use-disabled-sites", + action="store_true", default=False, + help="Use disabled sites to search (may cause many false positives)." + ) + parser.add_argument("--parse", + dest="parse_url", default='', + help="Parse page by URL and extract username and IDs to use for search." + ) + parser.add_argument("username", + nargs='+', metavar='USERNAMES', + action="store", + help="One or more usernames to check with social networks." + ) + parser.add_argument("--tags", + dest="tags", default='', + help="Specify tags of sites." + ) + args = parser.parse_args() + + # Logging + log_level = logging.ERROR + logging.basicConfig( + format='[%(filename)s:%(lineno)d] %(levelname)-3s %(asctime)s %(message)s', + datefmt='%H:%M:%S', + level=logging.ERROR + ) + + if args.debug: + log_level = logging.DEBUG + elif args.info: + log_level = logging.INFO + elif args.verbose: + log_level = logging.WARNING + + logger = logging.getLogger('maigret') + logger.setLevel(log_level) + + # Usernames initial list + usernames = { + u: 'username' + for u in args.username + if u not in ['-'] + } + + recursive_search_enabled = not args.disable_recursive_search + + # Make prompts + if args.proxy is not None: + print("Using the proxy: " + args.proxy) + + # Check if both output methods are entered as input. + if args.output is not None and args.folderoutput is not None: + print("You can only use one of the output methods.") + sys.exit(1) + + # Check validity for single username output. + if args.output is not None and len(args.username) != 1: + print("You can only use --output with a single username") + sys.exit(1) + + if args.parse_url: + page, _ = parse(args.parse_url, cookies_str='') + info = extract(page) + text = 'Extracted ID data from webpage: ' + ', '.join([f'{a}: {b}' for a, b in info.items()]) + print(text) + for k, v in info.items(): + if 'username' in k: + usernames[v] = 'username' + if k in supported_recursive_search_ids: + usernames[v] = k + + if args.tags: + args.tags = set(str(args.tags).split(',')) + + if args.json_file is None: + args.json_file = \ + os.path.join(os.path.dirname(os.path.realpath(__file__)), + "resources/data.json" + ) + + # Database self-checking + if args.self_check: + print('Maigret sites database self-checking...') + await self_check(args.json_file, logger) + + # Create object with all information about sites we are aware of. + try: + sites = SitesInformation(args.json_file) + except Exception as error: + print(f"ERROR: {error}") + sys.exit(1) + + # Create original dictionary from SitesInformation() object. + # Eventually, the rest of the code will be updated to use the new object + # directly, but this will glue the two pieces together. + site_data_all = {} + for site in sites: + site_data_all[site.name] = site.information + + if args.site_list is None: + # Not desired to look at a sub-set of sites + site_data = site_data_all + else: + # User desires to selectively run queries on a sub-set of the site list. + + # Make sure that the sites are supported & build up pruned site database. + site_data = {} + site_missing = [] + for site in args.site_list: + for existing_site in site_data_all: + if site.lower() == existing_site.lower(): + site_data[existing_site] = site_data_all[existing_site] + if not site_data: + # Build up list of sites not supported for future error message. + site_missing.append(f"'{site}'") + + if site_missing: + print( + f"Error: Desired sites not found: {', '.join(site_missing)}.") + sys.exit(1) + + if args.rank: + # Sort data by rank + site_dataCpy = dict(site_data) + ranked_sites = sorted(site_data, key=lambda k: ("rank" not in k, site_data[k].get("rank", sys.maxsize))) + site_data = {} + for site in ranked_sites: + site_data[site] = site_dataCpy.get(site) + + # Database consistency + enabled_count = len(list(filter(lambda x: not x.get('disabled', False), site_data.values()))) + print(f'Sites in database, enabled/total: {enabled_count}/{len(site_data)}') + + # Create notify object for query results. + query_notify = QueryNotifyPrint(result=None, + verbose=args.verbose, + print_found_only=not args.print_not_found, + skip_check_errors=not args.print_check_errors, + color=not args.no_color) + + already_checked = set() + + while usernames: + username, id_type = list(usernames.items())[0] + del usernames[username] + + if username.lower() in already_checked: + continue + else: + already_checked.add(username.lower()) + + # check for characters do not supported by sites generally + found_unsupported_chars = set(unsupported_characters).intersection(set(username)) + + if found_unsupported_chars: + pretty_chars_str = ','.join(map(lambda s: f'"{s}"', found_unsupported_chars)) + print(f'Found unsupported URL characters: {pretty_chars_str}, skip search by username "{username}"') + continue + + results = await maigret(username, + site_data, + query_notify, + proxy=args.proxy, + timeout=args.timeout, + recursive_search=recursive_search_enabled, + id_type=id_type, + tags=args.tags, + debug=args.verbose, + logger=logger, + forced=args.use_disabled_sites, + ) + + if args.output: + result_file = args.output + elif args.folderoutput: + # The usernames results should be stored in a targeted folder. + # If the folder doesn't exist, create it first + os.makedirs(args.folderoutput, exist_ok=True) + result_file = os.path.join(args.folderoutput, f"{username}.txt") + else: + result_file = f"{username}.txt" + + with open(result_file, "w", encoding="utf-8") as file: + exists_counter = 0 + for website_name in results: + dictionary = results[website_name] + + new_usernames = dictionary.get('ids_usernames') + if new_usernames: + for u, utype in new_usernames.items(): + usernames[u] = utype + + if dictionary.get("status").status == QueryStatus.CLAIMED: + exists_counter += 1 + file.write(dictionary["url_user"] + "\n") + file.write(f"Total Websites Username Detected On : {exists_counter}") + + if args.csv: + with open(username + ".csv", "w", newline='', encoding="utf-8") as csv_report: + writer = csv.writer(csv_report) + writer.writerow(['username', + 'name', + 'url_main', + 'url_user', + 'exists', + 'http_status', + 'response_time_s' + ] + ) + for site in results: + response_time_s = results[site]['status'].query_time + if response_time_s is None: + response_time_s = "" + writer.writerow([username, + site, + results[site]['url_main'], + results[site]['url_user'], + str(results[site]['status'].status), + results[site]['http_status'], + response_time_s + ] + ) + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except KeyboardInterrupt: + print('Maigret is interrupted.') + sys.exit(1) diff --git a/maigret/notify.py b/maigret/notify.py new file mode 100644 index 0000000..2ac0301 --- /dev/null +++ b/maigret/notify.py @@ -0,0 +1,283 @@ +"""Sherlock Notify Module + +This module defines the objects for notifying the caller about the +results of queries. +""" +from colorama import Fore, Style, init +from result import QueryStatus + + +class QueryNotify(): + """Query Notify Object. + + Base class that describes methods available to notify the results of + a query. + It is intended that other classes inherit from this base class and + override the methods to implement specific functionality. + """ + + def __init__(self, result=None): + """Create Query Notify Object. + + Contains information about a specific method of notifying the results + of a query. + + Keyword Arguments: + self -- This object. + result -- Object of type QueryResult() containing + results for this query. + + Return Value: + Nothing. + """ + + self.result = result + + return + + def start(self, message=None, id_type='username'): + """Notify Start. + + Notify method for start of query. This method will be called before + any queries are performed. This method will typically be + overridden by higher level classes that will inherit from it. + + Keyword Arguments: + self -- This object. + message -- Object that is used to give context to start + of query. + Default is None. + + Return Value: + Nothing. + """ + + return + + def update(self, result): + """Notify Update. + + Notify method for query result. This method will typically be + overridden by higher level classes that will inherit from it. + + Keyword Arguments: + self -- This object. + result -- Object of type QueryResult() containing + results for this query. + + Return Value: + Nothing. + """ + + self.result = result + + return + + def finish(self, message=None): + """Notify Finish. + + Notify method for finish of query. This method will be called after + all queries have been performed. This method will typically be + overridden by higher level classes that will inherit from it. + + Keyword Arguments: + self -- This object. + message -- Object that is used to give context to start + of query. + Default is None. + + Return Value: + Nothing. + """ + + return + + def __str__(self): + """Convert Object To String. + + Keyword Arguments: + self -- This object. + + Return Value: + Nicely formatted string to get information about this object. + """ + result = str(self.result) + + return result + + +class QueryNotifyPrint(QueryNotify): + """Query Notify Print Object. + + Query notify class that prints results. + """ + + def __init__(self, result=None, verbose=False, print_found_only=False, + skip_check_errors=False, color=True): + """Create Query Notify Print Object. + + Contains information about a specific method of notifying the results + of a query. + + Keyword Arguments: + self -- This object. + result -- Object of type QueryResult() containing + results for this query. + verbose -- Boolean indicating whether to give verbose output. + print_found_only -- Boolean indicating whether to only print found sites. + color -- Boolean indicating whether to color terminal output + + Return Value: + Nothing. + """ + + # Colorama module's initialization. + init(autoreset=True) + + super().__init__(result) + self.verbose = verbose + self.print_found_only = print_found_only + self.skip_check_errors = skip_check_errors + self.color = color + + return + + def start(self, message, id_type): + """Notify Start. + + Will print the title to the standard output. + + Keyword Arguments: + self -- This object. + message -- String containing username that the series + of queries are about. + + Return Value: + Nothing. + """ + + title = f"Checking {id_type}" + if self.color: + print(Style.BRIGHT + Fore.GREEN + "[" + + Fore.YELLOW + "*" + + Fore.GREEN + f"] {title}" + + Fore.WHITE + f" {message}" + + Fore.GREEN + " on:") + else: + print(f"[*] {title} {message} on:") + + return + + def get_additional_data_text(self, items, prepend=''): + text = '' + for num, item in enumerate(items): + box_symbol = '┣╸' if num != len(items) - 1 else '┗╸' + + if type(item) == tuple: + field_name, field_value = item + if field_value.startswith('[\''): + is_last_item = num == len(items) - 1 + prepend_symbols = ' ' * 3 if is_last_item else ' ┃ ' + field_value = self.get_additional_data_text(eval(field_value), prepend_symbols) + text += f'\n{prepend}{box_symbol}{field_name}: {field_value}' + else: + text += f'\n{prepend}{box_symbol} {item}' + + return text + + def update(self, result, is_similar=False): + """Notify Update. + + Will print the query result to the standard output. + + Keyword Arguments: + self -- This object. + result -- Object of type QueryResult() containing + results for this query. + + Return Value: + Nothing. + """ + self.result = result + + if not self.result.ids_data: + ids_data_text = "" + else: + ids_data_text = self.get_additional_data_text(self.result.ids_data.items(), ' ') + + def make_colored_terminal_notify(status, text, status_color, text_color, appendix): + text = [ + f'{Style.BRIGHT}{Fore.WHITE}[{status_color}{status}{Fore.WHITE}]' + + f'{text_color} {text}: {Style.RESET_ALL}' + + f'{appendix}' + ] + return ''.join(text) + + def make_simple_terminal_notify(status, text, appendix): + return f'[{status}] {text}: {appendix}' + + def make_terminal_notify(is_colored=True, *args): + if is_colored: + return make_colored_terminal_notify(*args) + else: + return make_simple_terminal_notify(*args) + + notify = None + + # Output to the terminal is desired. + if result.status == QueryStatus.CLAIMED: + color = Fore.BLUE if is_similar else Fore.GREEN + status = '?' if is_similar else '+' + notify = make_terminal_notify( + self.color, + status, result.site_name, + color, color, + result.site_url_user + ids_data_text + ) + elif result.status == QueryStatus.AVAILABLE: + if not self.print_found_only: + notify = make_terminal_notify( + self.color, + '-', result.site_name, + Fore.RED, Fore.YELLOW, + 'Not found!' + ids_data_text + ) + elif result.status == QueryStatus.UNKNOWN: + if not self.skip_check_errors: + notify = make_terminal_notify( + self.color, + '?', result.site_name, + Fore.RED, Fore.RED, + self.result.context + ids_data_text + ) + elif result.status == QueryStatus.ILLEGAL: + if not self.print_found_only: + text = 'Illegal Username Format For This Site!' + notify = make_terminal_notify( + self.color, + '-', result.site_name, + Fore.RED, Fore.YELLOW, + text + ids_data_text + ) + else: + # It should be impossible to ever get here... + raise ValueError(f"Unknown Query Status '{str(result.status)}' for " + f"site '{self.result.site_name}'") + + if notify: + print(notify) + + return + + def __str__(self): + """Convert Object To String. + + Keyword Arguments: + self -- This object. + + Return Value: + Nicely formatted string to get information about this object. + """ + result = str(self.result) + + return result diff --git a/maigret/resources/data.json b/maigret/resources/data.json new file mode 100644 index 0000000..8867bcc --- /dev/null +++ b/maigret/resources/data.json @@ -0,0 +1,18449 @@ +{ + "engines": { + "XenForo": { + "site": { + "errorMsg": [ + "\u0423\u043a\u0430\u0437\u0430\u043d\u043d\u044b\u0439 \u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d. \u041f\u043e\u0436\u0430\u043b\u0443\u0439\u0441\u0442\u0430, \u0432\u0432\u0435\u0434\u0438\u0442\u0435 \u0434\u0440\u0443\u0433\u043e\u0435 \u0438\u043c\u044f." + ], + "errorType": "message" + } + }, + "phpBB": { + "site": { + "errorMsg": [ + "\u041d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d\u043e \u043d\u0438 \u043e\u0434\u043d\u043e\u0433\u043e \u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044f \u043f\u043e \u0437\u0430\u0434\u0430\u043d\u043d\u044b\u043c \u043a\u0440\u0438\u0442\u0435\u0440\u0438\u044f\u043c" + ], + "errorType": "message" + } + }, + "vBulletin": { + "site": { + "errorMsg": [ + "\u041f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u0437\u0430\u0440\u0435\u0433\u0438\u0441\u0442\u0440\u0438\u0440\u043e\u0432\u0430\u043d \u0438 \u043d\u0435 \u0438\u043c\u0435\u0435\u0442 \u043f\u0440\u043e\u0444\u0438\u043b\u044f \u0434\u043b\u044f \u043f\u0440\u043e\u0441\u043c\u043e\u0442\u0440\u0430.", + "Bu \u00dcye kay\u0131tl\u0131 \u00dcyemiz de\u011fildir. Bu sebebten dolay\u0131 \u00dcyeye ait Profil g\u00f6sterilemiyor.", + "This user has not registered and therefore does not have a profile to view.", + "\u041a\u043e\u0440\u0438\u0441\u0442\u0443\u0432\u0430\u0447 \u043d\u0435 \u0437\u0430\u0440\u0435\u0454\u0441\u0442\u0440\u043e\u0432\u0430\u043d\u0438\u0439 \u0456 \u043d\u0435 \u043c\u0430\u0454 \u043f\u0440\u043e\u0444\u0456\u043b\u044e, \u044f\u043a\u0438\u0439 \u043c\u043e\u0436\u043d\u0430 \u043f\u0435\u0440\u0435\u0433\u043b\u044f\u043d\u0443\u0442\u0438." + ], + "errorType": "message" + } + } + }, + "sites": { + "0-3.RU": { + "disabled": false, + "errorMsg": "\u0423\u043a\u0430\u0437\u0430\u043d\u043d\u044b\u0439 \u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d", + "errorType": "message", + "rank": 4814775, + "tags": [ + "ru" + ], + "url": "http://0-3.ru/members/?username={}", + "urlMain": "http://0-3.ru", + "username_claimed": "donna", + "username_unclaimed": "noonewouldeverusethis7" + }, + "0k.clan.su": { + "disabled": false, + "errorMsg": "\u041f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d", + "errorType": "message", + "rank": 0, + "regexCheck": "^[^\\.]+$", + "tags": [ + "ru" + ], + "url": "http://0k.clan.su/index/8-0-{}", + "urlMain": "http://0k.clan.su", + "username_claimed": "eruzz", + "username_unclaimed": "noonewouldeverusethis7" + }, + "1001mem.ru": { + "errorMsg": "\u042d\u0442\u043e\u0442 \u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u0441\u0443\u0449\u0435\u0441\u0442\u0432\u0443\u0435\u0442, \u0438\u043b\u0438 \u0437\u0430\u0431\u043b\u043e\u043a\u0438\u0440\u043e\u0432\u0430\u043d.", + "errorType": "message", + "rank": 1714134, + "tags": [ + "ru" + ], + "url": "http://1001mem.ru/{}", + "urlMain": "http://1001mem.ru", + "username_claimed": "adam", + "username_unclaimed": "noonewouldeverusethis7" + }, + "101xp.com": { + "disabled": false, + "errorMsg": "\u0423\u043a\u0430\u0437\u0430\u043d\u043d\u044b\u0439 \u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d", + "errorType": "message", + "rank": 158497, + "tags": [ + "gaming", + "ru" + ], + "url": "https://forum-ru.101xp.com/members/?username={}", + "urlMain": "https://forum-ru.101xp.com", + "username_claimed": "aida", + "username_unclaimed": "noonewouldeverusethis7" + }, + "11x2": { + "disabled": false, + "errorType": "status_code", + "rank": 2324955, + "tags": [ + "global" + ], + "url": "https://11x2.com/user/home/{}", + "urlMain": "https://11x2.com", + "username_claimed": "hazelamy", + "username_unclaimed": "noonewouldeverusethis7" + }, + "123rf": { + "disabled": false, + "errorType": "response_url", + "rank": 925, + "tags": [ + "images", + "in", + "ru", + "us" + ], + "url": "https://ru.123rf.com/profile_{}", + "urlMain": "https://ru.123rf.com", + "username_claimed": "rawpixel", + "username_unclaimed": "noonewouldeverusethis7" + }, + "1337x": { + "disabled": true, + "errorMsg": "Error something went wrong", + "errorType": "message", + "rank": 487, + "tags": [ + "in", + "torrent" + ], + "url": "https://1337x.to/user/{}/", + "urlMain": "https://1337x.to", + "username_claimed": "adam", + "username_unclaimed": "noonewouldeverusethis7" + }, + "1x": { + "disabled": false, + "errorMsg": "This user does not exist or is not approved yet. Come back later.", + "errorType": "message", + "rank": 181101, + "tags": [ + "ba", + "in", + "se" + ], + "url": "https://1x.com/member/{}", + "urlMain": "https://1x.com", + "username_claimed": "blue", + "username_unclaimed": "noonewouldeverusethis7" + }, + "1xforum": { + "disabled": false, + "engine": "vBulletin", + "rank": 2579705, + "tags": [ + "ru" + ], + "url": "https://1xforum.com/member.php?username={}", + "urlMain": "https://1xforum.com", + "username_claimed": "adam", + "username_unclaimed": "noonewouldeverusethis7" + }, + "247sports": { + "disabled": false, + "errorType": "status_code", + "rank": 2624, + "request_head_only": false, + "tags": [ + "sport", + "us" + ], + "url": "https://247sports.com/user/{}/", + "urlMain": "https://247sports.com", + "username_claimed": "adam", + "username_unclaimed": "noonewouldeverusethis7" + }, + "24open": { + "disabled": false, + "errorType": "status_code", + "rank": 44526, + "request_head_only": false, + "tags": [ + "dating", + "ru" + ], + "url": "https://24open.ru/user/{}/", + "urlMain": "https://24open.ru", + "username_claimed": "niko3193", + "username_unclaimed": "noonewouldeverusethis7" + }, + "2Dimensions": { + "disabled": false, + "errorType": "status_code", + "rank": 0, + "url": "https://2Dimensions.com/a/{}", + "urlMain": "https://2Dimensions.com/", + "username_claimed": "blue", + "username_unclaimed": "noonewouldeverusethis7" + }, + "2berega.spb.ru": { + "disabled": false, + "errorMsg": "\u041f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d", + "errorType": "message", + "rank": 673622, + "tags": [ + "ru" + ], + "url": "https://2berega.spb.ru/user/{}", + "urlMain": "https://2berega.spb.ru", + "username_claimed": "adam", + "username_unclaimed": "noonewouldeverusethis7" + }, + "2d-3d": { + "disabled": false, + "errorType": "status_code", + "rank": 316524, + "tags": [ + "ru" + ], + "url": "https://www.2d-3d.ru/user/{}/", + "urlMain": "https://www.2d-3d.ru", + "username_claimed": "adam", + "username_unclaimed": "noonewouldeverusethis7" + }, + "2fast4u": { + "disabled": false, + "errorMsg": "Deze gebruiker is niet geregistreerd, zodat je zijn of haar profiel niet kunt bekijken.", + "errorType": "message", + "rank": 0, + "tags": [ + "nl" + ], + "url": "https://www.2fast4u.be/members/?username={}", + "urlMain": "https://www.2fast4u.be", + "username_claimed": "Schussboelie", + "username_unclaimed": "noonewouldeverusethis7" + }, + "33bru": { + "disabled": false, + "errorMsg": "\u0418\u0437\u0432\u0438\u043d\u0438\u0442\u0435, \u0442\u0430\u043a\u043e\u0433\u043e \u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044f \u043d\u0435 \u0441\u0443\u0449\u0435\u0441\u0442\u0432\u0443\u0435\u0442", + "errorType": "message", + "presenseStrs": [ + "\u041f\u0440\u043e\u0444\u0438\u043b\u044c \u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044f" + ], + "rank": 1412592, + "regexCheck": "^[a-zA-Z0-9-]{3,}$", + "tags": [ + "ru" + ], + "url": "http://{}.33bru.com/", + "urlMain": "http://33bru.com/", + "username_claimed": "adam", + "username_unclaimed": "noonewouldeverusethis7" + }, + "3dcadforums": { + "disabled": false, + "errorMsg": "The specified member cannot be found", + "errorType": "message", + "rank": 1348705, + "tags": [ + "global" + ], + "url": "https://www.3dcadforums.com/members/?username={}", + "urlMain": "https://www.3dcadforums.com/", + "username_claimed": "adam", + "username_unclaimed": "noonewouldeverusethis7" + }, + "3ddd": { + "disabled": false, + "errorType": "status_code", + "rank": 12564, + "tags": [ + "ru" + ], + "url": "https://3ddd.ru/users/{}", + "urlMain": "https://3ddd.ru", + "username_claimed": "adam", + "username_unclaimed": "noonewouldeverusethis7" + }, + "3dnews": { + "disabled": false, + "engine": "vBulletin", + "rank": 8731, + "tags": [ + "ru" + ], + "url": "http://forum.3dnews.ru/member.php?username={}", + "urlMain": "http://forum.3dnews.ru/", + "username_claimed": "red", + "username_unclaimed": "noonewouldeverusethis7" + }, + "3dtoday": { + "disabled": false, + "errorType": "response_url", + "rank": 78640, + "tags": [ + "ru" + ], + "url": "https://3dtoday.ru/blogs/{}", + "urlMain": "https://3dtoday.ru/", + "username_claimed": "adam", + "username_unclaimed": "noonewouldeverusethis7" + }, + "4cheat": { + "disabled": false, + "engine": "vBulletin", + "rank": 244139, + "tags": [ + "ru" + ], + "url": "https://4cheat.ru/member.php?username={}", + "urlMain": "https://4cheat.ru", + "username_claimed": "adam", + "username_unclaimed": "noonewouldeverusethis7" + }, + "4gameforum": { + "disabled": false, + "errorMsg": "\u0423\u043a\u0430\u0437\u0430\u043d\u043d\u044b\u0439 \u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d. \u041f\u043e\u0436\u0430\u043b\u0443\u0439\u0441\u0442\u0430, \u0432\u0432\u0435\u0434\u0438\u0442\u0435 \u0434\u0440\u0443\u0433\u043e\u0435 \u0438\u043c\u044f.", + "errorType": "message", + "rank": 89558, + "tags": [ + "ru" + ], + "url": "https://4gameforum.com/members/?username={}", + "urlMain": "https://4gameforum.com", + "username_claimed": "persty", + "username_unclaimed": "noonewouldeverusethis7" + }, + "4pda": { + "disabled": false, + "errorMsg": "\u041a \u0441\u043e\u0436\u0430\u043b\u0435\u043d\u0438\u044e, \u0412\u0430\u0448 \u043f\u043e\u0438\u0441\u043a \u043d\u0435 \u0434\u0430\u043b \u043d\u0438\u043a\u0430\u043a\u0438\u0445 \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u043e\u0432.", + "errorType": "message", + "rank": 2894, + "tags": [ + "ru" + ], + "url": "https://4pda.ru/forum/index.php?act=search&source=pst&noform=1&username={}", + "urlMain": "https://4pda.ru/", + "username_claimed": "green", + "username_unclaimed": "noonewouldeverusethis7" + }, + "4stor": { + "disabled": false, + "errorType": "status_code", + "rank": 241930, + "tags": [ + "ru" + ], + "url": "https://4stor.ru/user/{}", + "urlMain": "https://4stor.ru", + "username_claimed": "adam", + "username_unclaimed": "noonewouldeverusethis7" + }, + "500px": { + "disabled": false, + "errorMsg": "No message available", + "errorType": "message", + "errors": { + "INTERNAL_SERVER_ERROR": "Site error", + "Something just went wrong": "Site error" + }, + "rank": 2880, + "tags": [ + "images", + "in" + ], + "url": "https://500px.com/p/{}", + "urlMain": "https://500px.com/", + "urlProbe": "https://api.500px.com/graphql?operationName=ProfileRendererQuery&variables=%7B%22username%22%3A%22{}%22%7D&extensions=%7B%22persistedQuery%22%3A%7B%22version%22%3A1%2C%22sha256Hash%22%3A%225a17a9af1830b58b94a912995b7947b24f27f1301c6ea8ab71a9eb1a6a86585b%22%7D%7D", + "username_claimed": "blue", + "username_unclaimed": "noonewouldeverusethis7" + }, + "7Cups": { + "disabled": false, + "errorType": "status_code", + "rank": 32009, + "tags": [ + "in", + "pl" + ], + "url": "https://www.7cups.com/@{}", + "urlMain": "https://www.7cups.com/", + "username_claimed": "blue", + "username_unclaimed": "noonewouldeverusethis7" + }, + "7dach": { + "disabled": false, + "errorType": "status_code", + "rank": 11531, + "tags": [ + "ru" + ], + "url": "https://7dach.ru/profile/{}", + "urlMain": "https://7dach.ru/", + "username_claimed": "adam", + "username_unclaimed": "noonewouldeverusethis7" + }, + "7ya": { + "disabled": false, + "errorType": "status_code", + "rank": 44790, + "tags": [ + "ru" + ], + "url": "https://blog.7ya.ru/{}/", + "urlMain": "https://blog.7ya.ru", + "username_claimed": "trotter", + "username_unclaimed": "noonewouldeverusethis7" + }, + "9GAG": { + "disabled": false, + "errorType": "status_code", + "rank": 406, + "tags": [ + "de" + ], + "url": "https://www.9gag.com/u/{}", + "urlMain": "https://www.9gag.com/", + "username_claimed": "blue", + "username_unclaimed": "noonewouldeverusethis7" + }, + "Aback": { + "disabled": false, + "errorMsg": "\u041f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u0441 \u0442\u0430\u043a\u0438\u043c \u0438\u043c\u0435\u043d\u0435\u043c \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d.", + "errorType": "message", + "rank": 0, + "tags": [ + "ua" + ], + "url": "https://aback.com.ua/user/{}", + "urlMain": "https://aback.com.ua", + "username_claimed": "adam", + "username_unclaimed": "noonewouldeverusethis7" + }, + "About.me": { + "disabled": false, + "errorType": "status_code", + "rank": 11776, + "tags": [ + "in", + "social" + ], + "url": "https://about.me/{}", + "urlMain": "https://about.me/", + "username_claimed": "blue", + "username_unclaimed": "noonewouldeverusethis7" + }, + "Aboutcar": { + "disabled": false, + "errorMsg": "\u041f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u0437\u0430\u0440\u0435\u0433\u0438\u0441\u0442\u0440\u0438\u0440\u043e\u0432\u0430\u043d \u0438 \u043d\u0435 \u0438\u043c\u0435\u0435\u0442 \u043f\u0440\u043e\u0444\u0438\u043b\u044f \u0434\u043b\u044f \u043f\u0440\u043e\u0441\u043c\u043e\u0442\u0440\u0430.", + "errorType": "message", + "rank": 4948404, + "tags": [ + "ru" + ], + "url": "http://aboutcar.ru/members/{}.html", + "urlMain": "http://aboutcar.ru", + "username_claimed": "krolenya", + "username_unclaimed": "noonewouldeverusethis7" + }, + "Academia.edu": { + "disabled": false, + "errorType": "status_code", + "rank": 254, + "regexCheck": "^[^\\.]+$", + "tags": [ + "id" + ], + "url": "https://independent.academia.edu/{}", + "urlMain": "https://www.academia.edu/", + "username_claimed": "blue", + "username_unclaimed": "noonewouldeverusethis7" + }, + "Acomics": { + "disabled": false, + "errorType": "status_code", + "rank": 124831, + "tags": [ + "ru" + ], + "url": "https://acomics.ru/-{}", + "urlMain": "https://acomics.ru", + "username_claimed": "Garage", + "username_unclaimed": "noonewouldeverusethis7" + }, + "AdultFriendFinder": { + "disabled": false, + "errorMsg": "