Merge pull request #166 from soxoj/0.3.0

Bump to 0.3.0

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## [Unreleased]
 
+## [0.3.0] - 2021-06-02
+* added support of Tor and I2P sites
+* added experimental DNS checking feature
+* implemented sorting by data points for reports
+* reports fixes
+
 ## [0.2.4] - 2021-05-18
 * cli output report
 * various improvements

README.md

@@ -8,12 +8,6 @@
     <a href="https://pypi.org/project/maigret/">
       <img alt="PyPI - Downloads" src="https://img.shields.io/pypi/dw/maigret?style=flat-square">
     </a>
-    <a href="https://gitter.im/maigret-osint/community">
-      <img alt="Chat - Gitter" src="./static/chat_gitter.svg" />
-    </a>
-    <a href="https://twitter.com/intent/follow?screen_name=sox0j">
-      <img src="https://img.shields.io/twitter/follow/sox0j?label=Follow%20sox0j&style=social&color=blue" alt="Follow @sox0j" />
-    </a>
   </p>
   <p align="center">
     <img src="./static/maigret.png" height="200"/>
@@ -24,9 +18,9 @@
 
 ## About
 
-**Maigret** collect a dossier on a person **by username only**, checking for accounts on a huge number of sites and gathering all the available information from web pages. Maigret is an easy-to-use and powerful fork of [Sherlock](https://github.com/sherlock-project/sherlock).
+**Maigret** collect a dossier on a person **by username only**, checking for accounts on a huge number of sites and gathering all the available information from web pages. No API keys required. Maigret is an easy-to-use and powerful fork of [Sherlock](https://github.com/sherlock-project/sherlock).
 
-Currently supported more than 2000 sites ([full list](./sites.md)), search is launched against 500 popular sites in descending order of popularity by default.
+Currently supported more than 2000 sites ([full list](./sites.md)), search is launched against 500 popular sites in descending order of popularity by default. Also supported checking of Tor sites, I2P sites, and domains (via DNS resolving).  
 
 ## Main features
 

maigret/__version__.py

@@ -1,3 +1,3 @@
 """Maigret version file"""
 
-__version__ = '0.2.4'
+__version__ = '0.3.0'

maigret/activation.py

@@ -35,7 +35,7 @@ class ParsingActivator:
         site.headers["authorization"] = f"Bearer {bearer_token}"
 
 
-async def import_aiohttp_cookies(cookiestxt_filename):
+def import_aiohttp_cookies(cookiestxt_filename):
     cookies_obj = MozillaCookieJar(cookiestxt_filename)
     cookies_obj.load(ignore_discard=True, ignore_expires=True)
 

maigret/checking.py

@@ -9,6 +9,7 @@ from typing import Tuple, Optional, Dict, List
 from urllib.parse import quote
 
 import aiohttp
+import aiodns
 import tqdm.asyncio
 from aiohttp_socks import ProxyConnector
 from python_socks import _errors as proxy_errors
@@ -43,49 +44,142 @@ SUPPORTED_IDS = (
 BAD_CHARS = "#"
 
 
-async def get_response(request_future, logger) -> Tuple[str, int, Optional[CheckError]]:
-    html_text = None
-    status_code = 0
-    error: Optional[CheckError] = CheckError("Unknown")
+class CheckerBase:
+    pass
 
-    try:
-        response = await request_future
 
-        status_code = response.status
-        response_content = await response.content.read()
-        charset = response.charset or "utf-8"
-        decoded_content = response_content.decode(charset, "ignore")
-        html_text = decoded_content
+class SimpleAiohttpChecker(CheckerBase):
+    def __init__(self, *args, **kwargs):
+        proxy = kwargs.get('proxy')
+        cookie_jar = kwargs.get('cookie_jar')
+        self.logger = kwargs.get('logger', Mock())
 
-        error = None
-        if status_code == 0:
-            error = CheckError("Connection lost")
+        # make http client session
+        connector = (
+            ProxyConnector.from_url(proxy) if proxy else aiohttp.TCPConnector(ssl=False)
+        )
+        connector.verify_ssl = False
+        self.session = aiohttp.ClientSession(
+            connector=connector, trust_env=True, cookie_jar=cookie_jar
+        )
 
-        logger.debug(html_text)
-
-    except asyncio.TimeoutError as e:
-        error = CheckError("Request timeout", str(e))
-    except ClientConnectorError as e:
-        error = CheckError("Connecting failure", str(e))
-    except ServerDisconnectedError as e:
-        error = CheckError("Server disconnected", str(e))
-    except aiohttp.http_exceptions.BadHttpMessage as e:
-        error = CheckError("HTTP", str(e))
-    except proxy_errors.ProxyError as e:
-        error = CheckError("Proxy", str(e))
-    except KeyboardInterrupt:
-        error = CheckError("Interrupted")
-    except Exception as e:
-        # python-specific exceptions
-        if sys.version_info.minor > 6 and (
-            isinstance(e, ssl.SSLCertVerificationError) or isinstance(e, ssl.SSLError)
-        ):
-            error = CheckError("SSL", str(e))
+    def prepare(self, url, headers=None, allow_redirects=True, timeout=0, method='get'):
+        if method == 'get':
+            request_method = self.session.get
         else:
-            logger.debug(e, exc_info=True)
-            error = CheckError("Unexpected", str(e))
+            request_method = self.session.head
 
-    return str(html_text), status_code, error
+        future = request_method(
+            url=url,
+            headers=headers,
+            allow_redirects=allow_redirects,
+            timeout=timeout,
+        )
+
+        return future
+
+    async def close(self):
+        await self.session.close()
+
+    async def check(self, future) -> Tuple[str, int, Optional[CheckError]]:
+        html_text = None
+        status_code = 0
+        error: Optional[CheckError] = CheckError("Unknown")
+
+        try:
+            response = await future
+
+            status_code = response.status
+            response_content = await response.content.read()
+            charset = response.charset or "utf-8"
+            decoded_content = response_content.decode(charset, "ignore")
+            html_text = decoded_content
+
+            error = None
+            if status_code == 0:
+                error = CheckError("Connection lost")
+
+            self.logger.debug(html_text)
+
+        except asyncio.TimeoutError as e:
+            error = CheckError("Request timeout", str(e))
+        except ClientConnectorError as e:
+            error = CheckError("Connecting failure", str(e))
+        except ServerDisconnectedError as e:
+            error = CheckError("Server disconnected", str(e))
+        except aiohttp.http_exceptions.BadHttpMessage as e:
+            error = CheckError("HTTP", str(e))
+        except proxy_errors.ProxyError as e:
+            error = CheckError("Proxy", str(e))
+        except KeyboardInterrupt:
+            error = CheckError("Interrupted")
+        except Exception as e:
+            # python-specific exceptions
+            if sys.version_info.minor > 6 and (
+                isinstance(e, ssl.SSLCertVerificationError)
+                or isinstance(e, ssl.SSLError)
+            ):
+                error = CheckError("SSL", str(e))
+            else:
+                self.logger.debug(e, exc_info=True)
+                error = CheckError("Unexpected", str(e))
+
+        return str(html_text), status_code, error
+
+
+class ProxiedAiohttpChecker(SimpleAiohttpChecker):
+    def __init__(self, *args, **kwargs):
+        proxy = kwargs.get('proxy')
+        cookie_jar = kwargs.get('cookie_jar')
+        self.logger = kwargs.get('logger', Mock())
+
+        connector = ProxyConnector.from_url(proxy)
+        connector.verify_ssl = False
+        self.session = aiohttp.ClientSession(
+            connector=connector, trust_env=True, cookie_jar=cookie_jar
+        )
+
+
+class AiodnsDomainResolver(CheckerBase):
+    def __init__(self, *args, **kwargs):
+        loop = asyncio.get_event_loop()
+        self.logger = kwargs.get('logger', Mock())
+        self.resolver = aiodns.DNSResolver(loop=loop)
+
+    def prepare(self, url, headers=None, allow_redirects=True, timeout=0, method='get'):
+        return self.resolver.query(url, 'A')
+
+    async def check(self, future) -> Tuple[str, int, Optional[CheckError]]:
+        status = 404
+        error = None
+        text = ''
+
+        try:
+            res = await future
+            text = str(res[0].host)
+            status = 200
+        except aiodns.error.DNSError:
+            pass
+        except Exception as e:
+            self.logger.error(e, exc_info=True)
+            error = CheckError('DNS resolve error', str(e))
+
+        return text, status, error
+
+
+class CheckerMock:
+    def __init__(self, *args, **kwargs):
+        pass
+
+    def prepare(self, url, headers=None, allow_redirects=True, timeout=0, method='get'):
+        return None
+
+    async def check(self, future) -> Tuple[str, int, Optional[CheckError]]:
+        await asyncio.sleep(0)
+        return '', 0, None
+
+    async def close(self):
+        return
 
 
 # TODO: move to separate class
@@ -322,7 +416,8 @@ def make_site_result(
     # workaround to prevent slash errors
     url = re.sub("(?<!:)/+", "/", url)
 
-    session = options['session']
+    # always clearweb_checker for now
+    checker = options["checkers"][site.protocol]
 
     # site check is disabled
     if site.disabled and not options['forced']:
@@ -381,12 +476,12 @@ def make_site_result(
             # In most cases when we are detecting by status code,
             # it is not necessary to get the entire body:  we can
             # detect fine with just the HEAD response.
-            request_method = session.head
+            request_method = 'head'
         else:
             # Either this detect method needs the content associated
             # with the GET response, or this specific website will
             # not respond properly unless we request the whole page.
-            request_method = session.get
+            request_method = 'get'
 
         if site.check_type == "response_url":
             # Site forwards request to a different URL if username not
@@ -398,7 +493,8 @@ def make_site_result(
             # The final result of the request will be what is available.
             allow_redirects = True
 
-        future = request_method(
+        future = checker.prepare(
+            method=request_method,
             url=url_probe,
             headers=headers,
             allow_redirects=allow_redirects,
@@ -407,6 +503,7 @@ def make_site_result(
 
         # Store future request object in the results object
         results_site["future"] = future
+        results_site["checker"] = checker
 
     return results_site
 
@@ -419,7 +516,9 @@ async def check_site_for_username(
     if not future:
         return site.name, default_result
 
-    response = await get_response(request_future=future, logger=logger)
+    checker = default_result["checker"]
+
+    response = await checker.check(future=future)
 
     response_result = process_site_result(
         response, query_notify, logger, default_result, site
@@ -430,9 +529,9 @@ async def check_site_for_username(
     return site.name, response_result
 
 
-async def debug_ip_request(session, logger):
-    future = session.get(url="https://icanhazip.com")
-    ip, status, check_error = await get_response(future, logger)
+async def debug_ip_request(checker, logger):
+    future = checker.prepare(url="https://icanhazip.com")
+    ip, status, check_error = await checker.check(future)
     if ip:
         logger.debug(f"My IP is: {ip.strip()}")
     else:
@@ -456,6 +555,8 @@ async def maigret(
     logger,
     query_notify=None,
     proxy=None,
+    tor_proxy=None,
+    i2p_proxy=None,
     timeout=3,
     is_parsing_enabled=False,
     id_type="username",
@@ -465,6 +566,7 @@ async def maigret(
     no_progressbar=False,
     cookies=None,
     retries=0,
+    check_domains=False,
 ) -> QueryResultWrapper:
     """Main search func
 
@@ -508,23 +610,36 @@ async def maigret(
 
     query_notify.start(username, id_type)
 
-    # make http client session
-    connector = (
-        ProxyConnector.from_url(proxy) if proxy else aiohttp.TCPConnector(ssl=False)
-    )
-    connector.verify_ssl = False
-
     cookie_jar = None
     if cookies:
         logger.debug(f"Using cookies jar file {cookies}")
-        cookie_jar = await import_aiohttp_cookies(cookies)
+        cookie_jar = import_aiohttp_cookies(cookies)
 
-    session = aiohttp.ClientSession(
-        connector=connector, trust_env=True, cookie_jar=cookie_jar
+    clearweb_checker = SimpleAiohttpChecker(
+        proxy=proxy, cookie_jar=cookie_jar, logger=logger
     )
 
+    # TODO
+    tor_checker = CheckerMock()
+    if tor_proxy:
+        tor_checker = ProxiedAiohttpChecker(  # type: ignore
+            proxy=tor_proxy, cookie_jar=cookie_jar, logger=logger
+        )
+
+    # TODO
+    i2p_checker = CheckerMock()
+    if i2p_proxy:
+        i2p_checker = ProxiedAiohttpChecker(  # type: ignore
+            proxy=i2p_proxy, cookie_jar=cookie_jar, logger=logger
+        )
+
+    # TODO
+    dns_checker = CheckerMock()
+    if check_domains:
+        dns_checker = AiodnsDomainResolver(logger=logger)  # type: ignore
+
     if logger.level == logging.DEBUG:
-        await debug_ip_request(session, logger)
+        await debug_ip_request(clearweb_checker, logger)
 
     # setup parallel executor
     executor: Optional[AsyncExecutor] = None
@@ -538,7 +653,12 @@ async def maigret(
     # make options objects for all the requests
     options: QueryOptions = {}
     options["cookies"] = cookie_jar
-    options["session"] = session
+    options["checkers"] = {
+        '': clearweb_checker,
+        'tor': tor_checker,
+        'dns': dns_checker,
+        'i2p': i2p_checker,
+    }
     options["parsing"] = is_parsing_enabled
     options["timeout"] = timeout
     options["id_type"] = id_type
@@ -591,7 +711,11 @@ async def maigret(
             )
 
     # closing http client session
-    await session.close()
+    await clearweb_checker.close()
+    if tor_proxy:
+        await tor_checker.close()
+    if i2p_proxy:
+        await i2p_checker.close()
 
     # notify caller that all queries are finished
     query_notify.finish()
@@ -625,7 +749,13 @@ def timeout_check(value):
 
 
 async def site_self_check(
-    site: MaigretSite, logger, semaphore, db: MaigretDatabase, silent=False
+    site: MaigretSite,
+    logger,
+    semaphore,
+    db: MaigretDatabase,
+    silent=False,
+    tor_proxy=None,
+    i2p_proxy=None,
 ):
     changes = {
         "disabled": False,
@@ -649,6 +779,8 @@ async def site_self_check(
                 forced=True,
                 no_progressbar=True,
                 retries=1,
+                tor_proxy=tor_proxy,
+                i2p_proxy=i2p_proxy,
             )
 
             # don't disable entries with other ids types
@@ -658,6 +790,8 @@ async def site_self_check(
                 changes["disabled"] = True
                 continue
 
+            logger.debug(results_dict)
+
             result = results_dict[site.name]["status"]
 
         site_status = result.status
@@ -696,7 +830,13 @@ async def site_self_check(
 
 
 async def self_check(
-    db: MaigretDatabase, site_data: dict, logger, silent=False, max_connections=10
+    db: MaigretDatabase,
+    site_data: dict,
+    logger,
+    silent=False,
+    max_connections=10,
+    tor_proxy=None,
+    i2p_proxy=None,
 ) -> bool:
     sem = asyncio.Semaphore(max_connections)
     tasks = []
@@ -708,7 +848,9 @@ async def self_check(
     disabled_old_count = disabled_count(all_sites.values())
 
     for _, site in all_sites.items():
-        check_coro = site_self_check(site, logger, sem, db, silent)
+        check_coro = site_self_check(
+            site, logger, sem, db, silent, tor_proxy, i2p_proxy
+        )
         future = asyncio.ensure_future(check_coro)
         tasks.append(future)
 

maigret/maigret.py

@@ -33,6 +33,7 @@ from .report import (
     SUPPORTED_JSON_REPORT_FORMATS,
     save_json_report,
     get_plaintext_report,
+    sort_report_by_data_points,
 )
 from .sites import MaigretDatabase
 from .submit import submit_dialog
@@ -238,6 +239,26 @@ def setup_arguments_parser():
         default=None,
         help="Make requests over a proxy. e.g. socks5://127.0.0.1:1080",
     )
+    parser.add_argument(
+        "--tor-proxy",
+        metavar='TOR_PROXY_URL',
+        action="store",
+        default='socks5://127.0.0.1:9050',
+        help="Specify URL of your Tor gateway. Default is socks5://127.0.0.1:9050",
+    )
+    parser.add_argument(
+        "--i2p-proxy",
+        metavar='I2P_PROXY_URL',
+        action="store",
+        default='http://127.0.0.1:4444',
+        help="Specify URL of your I2P gateway. Default is http://127.0.0.1:4444",
+    )
+    parser.add_argument(
+        "--with-domains",
+        action="store_true",
+        default=False,
+        help="Enable (experimental) feature of checking domains on usernames.",
+    )
 
     filter_group = parser.add_argument_group(
         'Site filtering', 'Options to set site search scope'
@@ -420,6 +441,13 @@ def setup_arguments_parser():
         help=f"Generate a JSON report of specific type: {', '.join(SUPPORTED_JSON_REPORT_FORMATS)}"
         " (one report per username).",
     )
+
+    parser.add_argument(
+        "--reports-sorting",
+        default='default',
+        choices=('default', 'data'),
+        help="Method of results sorting in reports (default: in order of getting the result)",
+    )
     return parser
 
 
@@ -508,7 +536,12 @@ async def main():
     if args.self_check:
         print('Maigret sites database self-checking...')
         is_need_update = await self_check(
-            db, site_data, logger, max_connections=args.connections
+            db,
+            site_data,
+            logger,
+            max_connections=args.connections,
+            tor_proxy=args.tor_proxy,
+            i2p_proxy=args.i2p_proxy,
         )
         if is_need_update:
             if input('Do you want to save changes permanently? [Yn]\n').lower() in (
@@ -584,6 +617,8 @@ async def main():
             site_dict=dict(sites_to_check),
             query_notify=query_notify,
             proxy=args.proxy,
+            tor_proxy=args.tor_proxy,
+            i2p_proxy=args.i2p_proxy,
             timeout=args.timeout,
             is_parsing_enabled=parsing_enabled,
             id_type=id_type,
@@ -594,10 +629,14 @@ async def main():
             max_connections=args.connections,
             no_progressbar=args.no_progressbar,
             retries=args.retries,
+            check_domains=args.with_domains,
         )
 
         notify_about_errors(results, query_notify)
 
+        if args.reports_sorting == "data":
+            results = sort_report_by_data_points(results)
+
         general_results.append((username, id_type, results))
 
         # TODO: tests

maigret/report.py

@@ -36,6 +36,18 @@ def filter_supposed_data(data):
     return filtered_supposed_data
 
 
+def sort_report_by_data_points(results):
+    return dict(
+        sorted(
+            results.items(),
+            key=lambda x: len(
+                (x[1].get('status') and x[1]['status'].ids_data or {}).keys()
+            ),
+            reverse=True,
+        )
+    )
+
+
 """
 REPORTS SAVING
 """
@@ -243,14 +255,18 @@ def generate_csv_report(username: str, results: dict, csvfile):
         ["username", "name", "url_main", "url_user", "exists", "http_status"]
     )
     for site in results:
+        # TODO: fix the reason
+        status = 'Unknown'
+        if "status" in results[site]:
+            status = str(results[site]["status"].status)
         writer.writerow(
             [
                 username,
                 site,
-                results[site]["url_main"],
-                results[site]["url_user"],
-                str(results[site]["status"].status),
-                results[site]["http_status"],
+                results[site].get("url_main", ""),
+                results[site].get("url_user", ""),
+                status,
+                results[site].get("http_status", 0),
             ]
         )
 
@@ -262,7 +278,10 @@ def generate_txt_report(username: str, results: dict, file):
         # TODO: fix no site data issue
         if not dictionary:
             continue
-        if dictionary.get("status").status == QueryStatus.CLAIMED:
+        if (
+            dictionary.get("status")
+            and dictionary["status"].status == QueryStatus.CLAIMED
+        ):
             exists_counter += 1
             file.write(dictionary["url_user"] + "\n")
     file.write(f"Total Websites Username Detected On : {exists_counter}")
@@ -275,14 +294,18 @@ def generate_json_report(username: str, results: dict, file, report_type):
     for sitename in results:
         site_result = results[sitename]
         # TODO: fix no site data issue
-        if not site_result or site_result.get("status").status != QueryStatus.CLAIMED:
+        if not site_result or not site_result.get("status"):
+            continue
+
+        if site_result["status"].status != QueryStatus.CLAIMED:
             continue
 
         data = dict(site_result)
         data["status"] = data["status"].json()
         data["site"] = data["site"].json
-        if "future" in data:
-            del data["future"]
+        for field in ["future", "checker"]:
+            if field in data:
+                del data[field]
 
         if is_report_per_line:
             data["sitename"] = sitename
@@ -331,8 +354,11 @@ def design_xmind_sheet(sheet, username, results):
 
     for website_name in results:
         dictionary = results[website_name]
+        if not dictionary:
+            continue
         result_status = dictionary.get("status")
-        if result_status.status != QueryStatus.CLAIMED:
+        # TODO: fix the reason
+        if not result_status or result_status.status != QueryStatus.CLAIMED:
             continue
 
         stripped_tags = list(map(lambda x: x.strip(), result_status.tags))

maigret/resources/data.json

@@ -3438,6 +3438,7 @@
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "Destructoid": {
+            "disabled": true,
             "tags": [
                 "us"
             ],
@@ -5795,19 +5796,6 @@
             "usernameClaimed": "adam",
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
-        "Giphy": {
-            "tags": [
-                "photo",
-                "us",
-                "video"
-            ],
-            "checkType": "status_code",
-            "alexaRank": 653,
-            "urlMain": "https://giphy.com/",
-            "url": "https://giphy.com/{username}",
-            "usernameClaimed": "blue",
-            "usernameUnclaimed": "noonewouldeverusethis7"
-        },
         "GipsysTeam": {
             "tags": [
                 "ru"
@@ -8230,6 +8218,7 @@
             ],
             "checkType": "message",
             "absenceStrs": [
+                "\u0417\u0430\u043f\u0440\u043e\u0448\u0435\u043d\u043d\u0430\u044f \u0432\u0430\u043c\u0438 \u0441\u0442\u0440\u0430\u043d\u0438\u0446\u0430 \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d\u0430.",
                 "\u0414\u0430\u043d\u043d\u044b\u0435 \u043e \u0432\u044b\u0431\u0440\u0430\u043d\u043d\u043e\u043c \u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u0435 \u043d\u0435 \u0441\u0443\u0449\u0435\u0441\u0442\u0432\u0443\u044e\u0442",
                 "Information on selected user does not exist"
             ],
@@ -13035,7 +13024,7 @@
                 "us"
             ],
             "headers": {
-                "authorization": "Bearer BQBFMMVu1dPwJPlnzUteNyF8xlZy7545QnhHizEHWEUQGQrRLznY5k9B9v7JdAsL-wU-Tcep51JTqBesKKY"
+                "authorization": "Bearer BQCypIuUtz7zDFov8xN86mj1BelLf7Apf9WBaC5yYfNkmGe4r7Hz4Awp6dqPuCAP9K9F5yYtjbyZX_vlr4I"
             },
             "errors": {
                 "Spotify is currently not available in your country.": "Access denied in your country, use proxy/vpn"
@@ -13329,6 +13318,7 @@
                 "ru"
             ],
             "checkType": "response_url",
+            "regexCheck": "^[^-]+$",
             "alexaRank": 1499,
             "urlMain": "https://studfile.net",
             "url": "https://studfile.net/users/{username}/",
@@ -13811,16 +13801,13 @@
         },
         "TheSimsResource": {
             "tags": [
-                "de",
-                "gaming",
-                "it",
-                "us"
+                "gaming"
             ],
-            "checkType": "status_code",
+            "checkType": "response_url",
             "alexaRank": 12278,
             "urlMain": "https://www.thesimsresource.com/",
             "url": "https://www.thesimsresource.com/members/{username}/",
-            "usernameClaimed": "adam",
+            "usernameClaimed": "DanSimsFantasy",
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "TheStudentRoom": {
@@ -14463,7 +14450,7 @@
                 "sec-ch-ua": "Google Chrome\";v=\"87\", \" Not;A Brand\";v=\"99\", \"Chromium\";v=\"87\"",
                 "authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
                 "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
-                "x-guest-token": "1394397954526560260"
+                "x-guest-token": "1400174453577900043"
             },
             "errors": {
                 "Bad guest token": "x-guest-token update required"
@@ -14870,7 +14857,7 @@
                 "video"
             ],
             "headers": {
-                "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MjEyODYyODAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.mxLdaOuP260WcxBvhadTTUQyn8t75pWNhTmtZLFS-W4"
+                "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MjI2NjcxMjAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.V4VVbLzNwPU21rNP5moSxrPcPw--C7_Qz9VHgcJc1CA"
             },
             "activation": {
                 "url": "https://vimeo.com/_rv/viewer",
@@ -15834,6 +15821,32 @@
             "usernameClaimed": "yandex",
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
+        "YandexCollections API (by yandex_public_id)": {
+            "tags": [
+                "ru",
+                "sharing"
+            ],
+            "type": "yandex_public_id",
+            "headers": {
+                "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36"
+            },
+            "errors": {
+                "action=\"/checkcaptcha\" onsubmit": "Captcha detected, use proxy/vpn"
+            },
+            "checkType": "message",
+            "presenseStrs": [
+                "public_id"
+            ],
+            "absenceStrs": [
+                "cl-not-found-content__title"
+            ],
+            "alexaRank": 50,
+            "urlMain": "https://yandex.ru/collections/",
+            "url": "https://yandex.ru/collections/api/users/{username}/",
+            "source": "Yandex",
+            "usernameClaimed": "hx0aur0arkyebkxztq8pr8b4dg",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
         "YandexMarket": {
             "tags": [
                 "ru"
@@ -16275,8 +16288,8 @@
         },
         "author.today": {
             "tags": [
-                "ru",
-                "reading"
+                "reading",
+                "ru"
             ],
             "checkType": "status_code",
             "alexaRank": 12218,
@@ -27744,6 +27757,488 @@
             "tags": [
                 "business"
             ]
+        },
+        "HiddenAnswers": {
+            "tags": [
+                "tor"
+            ],
+            "protocol": "tor",
+            "url": "http://answerszuvs3gg2l64e6hmnryudl5zgrmwm3vh65hzszdghblddvfiqd.onion/user/{username}",
+            "urlMain": "http://answerszuvs3gg2l64e6hmnryudl5zgrmwm3vh65hzszdghblddvfiqd.onion",
+            "usernameClaimed": "theredqueen",
+            "usernameUnclaimed": "noonewouldeverusethis7",
+            "checkType": "message",
+            "absenceStrs": [
+                "Page not found"
+            ],
+            "presenseStrs": [
+                "qa-part-form-profile"
+            ]
+        },
+        ".com": {
+            "protocol": "dns",
+            "url": "{username}.com",
+            "urlMain": "{username}.com",
+            "usernameClaimed": "soxoj",
+            "usernameUnclaimed": "noonewouldeverusethis7",
+            "checkType": "status_code"
+        },
+        ".pro": {
+            "protocol": "dns",
+            "url": "{username}.pro",
+            "urlMain": "{username}.pro",
+            "usernameClaimed": "alex",
+            "usernameUnclaimed": "noonewouldeverusethis7",
+            "checkType": "status_code"
+        },
+        ".me": {
+            "protocol": "dns",
+            "url": "{username}.me",
+            "urlMain": "{username}.me",
+            "usernameClaimed": "alex",
+            "usernameUnclaimed": "noonewouldeverusethis7",
+            "checkType": "status_code"
+        },
+        ".biz": {
+            "protocol": "dns",
+            "url": "{username}.biz",
+            "urlMain": "{username}.biz",
+            "usernameClaimed": "alex",
+            "usernameUnclaimed": "noonewouldeverusethis7",
+            "checkType": "status_code"
+        },
+        ".email": {
+            "protocol": "dns",
+            "url": "{username}.email",
+            "urlMain": "{username}.email",
+            "usernameClaimed": "alex",
+            "usernameUnclaimed": "noonewouldeverusethis7",
+            "checkType": "status_code"
+        },
+        ".guru": {
+            "protocol": "dns",
+            "url": "{username}.guru",
+            "urlMain": "{username}.guru",
+            "usernameClaimed": "alex",
+            "usernameUnclaimed": "noonewouldeverusethis7",
+            "checkType": "status_code"
+        },
+        ".ddns.net": {
+            "protocol": "dns",
+            "url": "{username}.ddns.net",
+            "urlMain": "{username}.ddns.net",
+            "usernameClaimed": "repack",
+            "usernameUnclaimed": "noonewouldeverusethis7",
+            "checkType": "status_code"
+        },
+        "Ameblo": {
+            "absenceStrs": [
+                "THROW_NOT_FOUND_EXCEPTION"
+            ],
+            "presenseStrs": [
+                "profile"
+            ],
+            "url": "https://ameblo.jp/{username}",
+            "urlMain": "https://ameblo.jp",
+            "usernameClaimed": "senpai",
+            "usernameUnclaimed": "noonewouldeverusethis7",
+            "checkType": "message",
+            "alexaRank": 374,
+            "tags": [
+                "blog",
+                "jp"
+            ]
+        },
+        "Observable": {
+            "absenceStrs": [
+                "<title>Observable</title>"
+            ],
+            "presenseStrs": [
+                "profile_email"
+            ],
+            "url": "https://observablehq.com/@{username}",
+            "urlMain": "https://observablehq.com",
+            "usernameClaimed": "theabbie",
+            "usernameUnclaimed": "noonewouldeverusethis7",
+            "checkType": "message",
+            "alexaRank": 25120,
+            "tags": [
+                "sharing"
+            ]
+        },
+        "galactictalk.org": {
+            "urlMain": "https://galactictalk.org",
+            "engine": "Flarum",
+            "usernameClaimed": "theabbie",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "discuss.bootstrapped.fm": {
+            "urlMain": "https://discuss.bootstrapped.fm",
+            "engine": "Discourse",
+            "usernameClaimed": "theabbie",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "discourse.mozilla.org": {
+            "urlMain": "https://discourse.mozilla.org",
+            "engine": "Discourse",
+            "usernameClaimed": "theabbie",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "ipinit.in": {
+            "urlMain": "http://ipinit.in",
+            "engine": "Wordpress/Author",
+            "usernameClaimed": "god",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "donorbox": {
+            "absenceStrs": [
+                "/orgs/new"
+            ],
+            "presenseStrs": [
+                "donation_first_name"
+            ],
+            "url": "https://donorbox.org/{username}",
+            "urlMain": "https://donorbox.org",
+            "usernameClaimed": "theabbie",
+            "usernameUnclaimed": "noonewouldeverusethis7",
+            "checkType": "message",
+            "alexaRank": 19812,
+            "tags": [
+                "finance"
+            ]
+        },
+        "telescope.ac": {
+            "absenceStrs": [
+                ">Not found</h1>"
+            ],
+            "presenseStrs": [
+                "og:site_name",
+                "alternate",
+                "article",
+                "project",
+                "og:title"
+            ],
+            "url": "https://telescope.ac/{username}",
+            "urlMain": "https://telescope.ac",
+            "usernameClaimed": "theabbie",
+            "usernameUnclaimed": "noonewouldeverusethis7",
+            "checkType": "message",
+            "alexaRank": 167480,
+            "tags": [
+                "blog"
+            ]
+        },
+        "sessionize.com": {
+            "absenceStrs": [
+                "Page Not Found</h3>"
+            ],
+            "presenseStrs": [
+                "role=",
+                "filter"
+            ],
+            "url": "https://sessionize.com/{username}/",
+            "urlMain": "https://sessionize.com",
+            "usernameClaimed": "theabbie",
+            "usernameUnclaimed": "noonewouldeverusethis7",
+            "checkType": "message",
+            "alexaRank": 132025,
+            "tags": [
+                "business"
+            ]
+        },
+        "getmakerlog.com": {
+            "absenceStrs": [
+                "<title>Home | Makerlog</title>"
+            ],
+            "presenseStrs": [
+                "profile",
+                "first_name",
+                "username\\"
+            ],
+            "url": "https://getmakerlog.com/@{username}",
+            "urlMain": "https://getmakerlog.com",
+            "usernameClaimed": "theabbie",
+            "usernameUnclaimed": "noonewouldeverusethis7",
+            "checkType": "message",
+            "alexaRank": 224990,
+            "tags": [
+                "business"
+            ]
+        },
+        "giphy.com": {
+            "absenceStrs": [
+                "404 Not Found"
+            ],
+            "presenseStrs": [
+                "Giphy",
+                "al:ios:app_name"
+            ],
+            "url": "https://giphy.com/channel/{username}",
+            "urlMain": "https://giphy.com",
+            "usernameClaimed": "theabbie",
+            "usernameUnclaimed": "noonewouldeverusethis7",
+            "checkType": "message",
+            "alexaRank": 695,
+            "tags": [
+                "video"
+            ]
+        },
+        "clarity.fm": {
+            "absenceStrs": [
+                "On Demand Business Advice</title"
+            ],
+            "presenseStrs": [
+                "user-profile-image"
+            ],
+            "url": "https://clarity.fm/{username}",
+            "urlMain": "https://clarity.fm",
+            "usernameClaimed": "theabbie",
+            "usernameUnclaimed": "noonewouldeverusethis7",
+            "checkType": "message",
+            "alexaRank": 31250,
+            "tags": [
+                "business"
+            ]
+        },
+        "videohive.net": {
+            "absenceStrs": [
+                "Page Not Found | VideoHive"
+            ],
+            "presenseStrs": [
+                "user-info",
+                "user-info__badges"
+            ],
+            "url": "https://videohive.net/user/{username}",
+            "urlMain": "https://videohive.net",
+            "usernameClaimed": "theabbie",
+            "usernameUnclaimed": "noonewouldeverusethis7",
+            "checkType": "message",
+            "alexaRank": 4270,
+            "tags": [
+                "video"
+            ]
+        },
+        "boards.theforce.net": {
+            "urlMain": "https://boards.theforce.net",
+            "engine": "XenForo",
+            "usernameClaimed": "alex",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "aussiehomebrewer.com": {
+            "urlMain": "https://aussiehomebrewer.com",
+            "engine": "XenForo",
+            "usernameClaimed": "alex",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "forum-ukraina.net": {
+            "urlMain": "https://forum-ukraina.net",
+            "engine": "XenForo",
+            "usernameClaimed": "alex",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "forum-history.ru": {
+            "urlMain": "http://forum-history.ru",
+            "engine": "vBulletin",
+            "usernameClaimed": "red",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "forum.vn.ua": {
+            "urlMain": "http://forum.vn.ua",
+            "engine": "vBulletin",
+            "usernameClaimed": "alex",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "forum.bestflowers.ru": {
+            "urlMain": "https://forum.bestflowers.ru",
+            "engine": "XenForo",
+            "usernameClaimed": "alex",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "forum.alconar.ru": {
+            "urlMain": "https://forum.alconar.ru",
+            "engine": "phpBB/Search",
+            "usernameClaimed": "admin",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "forum.lancerx.ru": {
+            "urlMain": "https://forum.lancerx.ru",
+            "engine": "phpBB/Search",
+            "usernameClaimed": "alex",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "mfarmer.ru": {
+            "urlMain": "http://www.mfarmer.ru",
+            "engine": "vBulletin",
+            "usernameClaimed": "alex",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "forum.league17.ru": {
+            "urlMain": "https://forum.league17.ru",
+            "engine": "phpBB/Search",
+            "usernameClaimed": "alex",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "krskforum.com": {
+            "urlMain": "https://krskforum.com",
+            "engine": "phpBB/Search",
+            "usernameClaimed": "admin",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "forum.rarib.ag": {
+            "urlMain": "https://forum.rarib.ag",
+            "engine": "phpBB",
+            "usernameClaimed": "alex",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "forum.oneclickchicks.com": {
+            "urlMain": "https://forum.oneclickchicks.com",
+            "engine": "vBulletin",
+            "usernameClaimed": "alex",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "forum.trade-print.ru": {
+            "urlMain": "http://forum.trade-print.ru",
+            "engine": "vBulletin",
+            "usernameClaimed": "alex",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "forum.setcombg.com": {
+            "urlMain": "https://forum.setcombg.com",
+            "engine": "vBulletin",
+            "usernameClaimed": "alex",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "vw-bus.ru": {
+            "urlMain": "https://vw-bus.ru",
+            "engine": "phpBB",
+            "usernameClaimed": "alex",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "forum.ya1.ru": {
+            "urlMain": "https://forum.ya1.ru",
+            "engine": "vBulletin",
+            "usernameClaimed": "alex",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "forum.wordreference.com": {
+            "urlMain": "https://forum.wordreference.com",
+            "engine": "XenForo",
+            "usernameClaimed": "alex",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "forums.zooclub.ru": {
+            "urlMain": "https://forums.zooclub.ru",
+            "engine": "vBulletin",
+            "usernameClaimed": "alex",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "homebrewtalk.com": {
+            "urlMain": "https://www.homebrewtalk.com",
+            "engine": "XenForo",
+            "usernameClaimed": "alex",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "navimba.com": {
+            "urlMain": "https://navimba.com",
+            "engine": "phpBB/Search",
+            "usernameClaimed": "admin",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "niva-club.net": {
+            "urlMain": "https://www.niva-club.net",
+            "engine": "phpBB/Search",
+            "usernameClaimed": "red",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "red-forum.com": {
+            "urlMain": "https://red-forum.com",
+            "engine": "XenForo",
+            "usernameClaimed": "alex",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "scaleforum.ru": {
+            "urlMain": "http://www.scaleforum.ru",
+            "engine": "vBulletin",
+            "usernameClaimed": "alex",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "sign-forum.ru": {
+            "urlMain": "https://sign-forum.ru",
+            "engine": "phpBB/Search",
+            "usernameClaimed": "alex",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "rec.poker": {
+            "urlMain": "https://rec.poker",
+            "engine": "Wordpress/Author",
+            "usernameClaimed": "alex",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "uforum.uz": {
+            "urlMain": "https://uforum.uz",
+            "engine": "vBulletin",
+            "usernameClaimed": "alex",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "DarkNet Trust": {
+            "absenceStrs": [
+                "notify notify-red mx-auto text-center"
+            ],
+            "presenseStrs": [
+                "profiles"
+            ],
+            "protocol": "tor",
+            "url": "http://dntrustmucd4mwec.onion/u/{username}",
+            "urlMain": "http://dntrustmucd4mwec.onion",
+            "usernameClaimed": "cheshirecat82",
+            "usernameUnclaimed": "noonewouldeverusethis7",
+            "checkType": "message",
+            "headers": {
+                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; rv:78.0) Gecko/20100101 Firefox/78.0",
+                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+                "Accept-Language": "en-US,en;q=0.5"
+            },
+            "tags": [
+                "tor"
+            ]
+        },
+        "i2pforum": {
+            "protocol": "i2p",
+            "urlMain": "http://i2pforum.i2p",
+            "usernameClaimed": "zzz",
+            "usernameUnclaimed": "noonewouldeverusethis7",
+            "engine": "phpBB/Search",
+            "tags": [
+                "i2p"
+            ]
+        },
+        "forum.freeton.org": {
+            "urlMain": "https://forum.freeton.org",
+            "engine": "Discourse",
+            "usernameClaimed": "maximmuzychenka",
+            "usernameUnclaimed": "noonewouldeverusethis7",
+            "alexaRank": 192123,
+            "tags": [
+                "finance",
+                "forum"
+            ]
+        },
+        "lyricstranslate.com": {
+            "absenceStrs": [
+                "Page not found | Lyrics Translate"
+            ],
+            "presenseStrs": [
+                "profileid"
+            ],
+            "url": "https://lyricstranslate.com/sco/translator/{username}",
+            "urlMain": "https://lyricstranslate.com",
+            "usernameClaimed": "charming43",
+            "usernameUnclaimed": "noonewouldeverusethis7",
+            "checkType": "message",
+            "alexaRank": 3744,
+            "tags": [
+                "music"
+            ]
         }
     },
     "engines": {

maigret/resources/simple_report.tpl

@@ -68,7 +68,7 @@
         <div class="row-mb">
             <div class="col-md">
                 <div class="card flex-md-row mb-4 box-shadow h-md-250">
-                    <img class="card-img-right flex-auto d-md-block" alt="Photo" style="width: 200px; height: 200px; object-fit: scale-down;" src="{{ v.status.ids_data.image or 'https://i.imgur.com/040fmbw.png' }}" data-holder-rendered="true">
+                    <img class="card-img-right flex-auto d-md-block" alt="Photo" style="width: 200px; height: 200px; object-fit: scale-down;" src="{{ v.status and v.status.ids_data and v.status.ids_data.image or 'https://i.imgur.com/040fmbw.png' }}" data-holder-rendered="true">
                     <div class="card-body d-flex flex-column align-items-start" style="padding-top: 0;">
                     <h3 class="mb-0" style="padding-top: 1rem;">
                         <a class="text-dark" href="{{ v.url_main }}" target="_blank">{{ k }}</a>

maigret/sites.py

@@ -65,6 +65,8 @@ SUPPORTED_TAGS = [
     "review",
     "bookmarks",
     "design",
+    "tor",
+    "i2p",
 ]
 
 
@@ -122,6 +124,8 @@ class MaigretSite:
     alexa_rank = None
     source = None
 
+    protocol = ''
+
     def __init__(self, name, information):
         self.name = name
         self.url_subpath = ""
@@ -301,12 +305,18 @@ class MaigretDatabase:
             lambda x: isinstance(x.engine, str) and x.engine.lower() in normalized_tags
         )
         is_tags_ok = lambda x: set(x.tags).intersection(set(normalized_tags))
+        is_protocol_in_tags = lambda x: x.protocol and x.protocol in normalized_tags
         is_disabled_needed = lambda x: not x.disabled or (
             "disabled" in tags or disabled
         )
         is_id_type_ok = lambda x: x.type == id_type
 
-        filter_tags_engines_fun = lambda x: not tags or is_engine_ok(x) or is_tags_ok(x)
+        filter_tags_engines_fun = (
+            lambda x: not tags
+            or is_engine_ok(x)
+            or is_tags_ok(x)
+            or is_protocol_in_tags(x)
+        )
         filter_names_fun = lambda x: not names or is_name_ok(x) or is_source_ok(x)
 
         filter_fun = (

maigret/submit.py

@@ -32,6 +32,8 @@ HEADERS = {
     "User-Agent": get_random_user_agent(),
 }
 
+SEPARATORS = "\"'"
+
 RATIO = 0.6
 TOP_FEATURES = 5
 URL_RE = re.compile(r"https?://(www\.)?")
@@ -195,7 +197,7 @@ async def detect_known_engine(
 
 def extract_username_dialog(url):
     url_parts = url.rstrip("/").split("/")
-    supposed_username = url_parts[-1]
+    supposed_username = url_parts[-1].strip('@')
     entered_username = input(
         f'Is "{supposed_username}" a valid username? If not, write it manually: '
     )
@@ -203,38 +205,53 @@ def extract_username_dialog(url):
 
 
 async def check_features_manually(
-    db, url_exists, url_mainpage, cookie_file, logger, redirects=True
+    db, url_exists, url_mainpage, cookie_file, logger, redirects=False
 ):
+    custom_headers = {}
+    while True:
+        header_key = input(
+            'Specify custom header if you need or just press Enter to skip. Header name: '
+        )
+        if not header_key:
+            break
+        header_value = input('Header value: ')
+        custom_headers[header_key.strip()] = header_value.strip()
+
     supposed_username = extract_username_dialog(url_exists)
     non_exist_username = "noonewouldeverusethis7"
 
     url_user = url_exists.replace(supposed_username, "{username}")
     url_not_exists = url_exists.replace(supposed_username, non_exist_username)
 
+    headers = dict(HEADERS)
+    headers.update(custom_headers)
+
     # cookies
     cookie_dict = None
     if cookie_file:
         logger.info(f'Use {cookie_file} for cookies')
-        cookie_jar = await import_aiohttp_cookies(cookie_file)
+        cookie_jar = import_aiohttp_cookies(cookie_file)
         cookie_dict = {c.key: c.value for c in cookie_jar}
 
     exists_resp = requests.get(
-        url_exists, cookies=cookie_dict, headers=HEADERS, allow_redirects=redirects
+        url_exists, cookies=cookie_dict, headers=headers, allow_redirects=redirects
     )
+    logger.debug(url_exists)
     logger.debug(exists_resp.status_code)
     logger.debug(exists_resp.text)
 
     non_exists_resp = requests.get(
-        url_not_exists, cookies=cookie_dict, headers=HEADERS, allow_redirects=redirects
+        url_not_exists, cookies=cookie_dict, headers=headers, allow_redirects=redirects
     )
+    logger.debug(url_not_exists)
     logger.debug(non_exists_resp.status_code)
     logger.debug(non_exists_resp.text)
 
     a = exists_resp.text
     b = non_exists_resp.text
 
-    tokens_a = set(a.split('"'))
-    tokens_b = set(b.split('"'))
+    tokens_a = set(re.split(f'[{SEPARATORS}]', a))
+    tokens_b = set(re.split(f'[{SEPARATORS}]', b))
 
     a_minus_b = tokens_a.difference(tokens_b)
     b_minus_a = tokens_b.difference(tokens_a)
@@ -276,6 +293,9 @@ async def check_features_manually(
         "checkType": "message",
     }
 
+    if headers != HEADERS:
+        site_data['headers'] = headers
+
     site = MaigretSite(url_mainpage.split("/")[-1], site_data)
     return site
 
@@ -283,6 +303,7 @@ async def check_features_manually(
 async def submit_dialog(db, url_exists, cookie_file, logger):
     domain_raw = URL_RE.sub("", url_exists).strip().strip("/")
     domain_raw = domain_raw.split("/")[0]
+    logger.info('Domain is %s', domain_raw)
 
     # check for existence
     matched_sites = list(filter(lambda x: domain_raw in x.url_main + x.url, db.sites))

requirements.txt

@@ -1,3 +1,4 @@
+aiodns==3.0.0
 aiohttp==3.7.4
 aiohttp-socks==0.5.5
 arabic-reshaper==2.1.1
@@ -26,7 +27,7 @@ python-socks==1.1.2
 requests>=2.24.0
 requests-futures==1.0.0
 six==1.15.0
-socid-extractor>=0.0.20
+socid-extractor>=0.0.21
 soupsieve==2.1
 stem==1.8.0
 torrequest==0.1.0

setup.py

@@ -12,7 +12,7 @@ with open('requirements.txt') as rf:
     requires = rf.read().splitlines()
 
 setup(name='maigret',
-      version='0.2.4',
+      version='0.3.0',
       description='Collect a dossier on a person by username from a huge number of sites',
       long_description=long_description,
       long_description_content_type="text/markdown",

sites.md

@@ -1,5 +1,5 @@
 
-## List of supported sites (search methods): total 2515
+## List of supported sites (search methods): total 2560
 
 Rank data fetched from Alexa by domains.
 
@@ -39,8 +39,9 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://tanks.mail.ru) [Tanks (https://tanks.mail.ru)](https://tanks.mail.ru)*: top 50, forum, gaming, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://wf.mail.ru) [Warface (https://wf.mail.ru)](https://wf.mail.ru)*: top 50, forum, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://yandex.ru/) [YandexReviews (https://yandex.ru/)](https://yandex.ru/)*: top 50, ru*
-1. ![](https://www.google.com/s2/favicons?domain=https://yandex.ru/bugbounty/) [YandexBugbounty (https://yandex.ru/bugbounty/)](https://yandex.ru/bugbounty/)*: top 50, hacking, ru*
+1. ![](https://www.google.com/s2/favicons?domain=https://yandex.ru/bugbounty/) [YandexBugbounty (https://yandex.ru/bugbounty/)](https://yandex.ru/bugbounty/)*: top 50, hacking, ru*, search is disabled
 1. ![](https://www.google.com/s2/favicons?domain=https://yandex.ru/collections/) [YandexCollections API (https://yandex.ru/collections/)](https://yandex.ru/collections/)*: top 50, ru, sharing*
+1. ![](https://www.google.com/s2/favicons?domain=https://yandex.ru/collections/) [YandexCollections API (by yandex_public_id) (https://yandex.ru/collections/)](https://yandex.ru/collections/)*: top 50, ru, sharing*
 1. ![](https://www.google.com/s2/favicons?domain=https://market.yandex.ru/) [YandexMarket (https://market.yandex.ru/)](https://market.yandex.ru/)*: top 50, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://music.yandex.ru/) [YandexMusic (https://music.yandex.ru/)](https://music.yandex.ru/)*: top 50, music, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://yandex.ru/q/) [YandexZnatoki (https://yandex.ru/q/)](https://yandex.ru/q/)*: top 50, ru*
@@ -107,6 +108,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://www.mercadolivre.com.br) [mercadolivre (https://www.mercadolivre.com.br)](https://www.mercadolivre.com.br)*: top 500, br*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.crunchyroll.com/) [Crunchyroll (https://www.crunchyroll.com/)](https://www.crunchyroll.com/)*: top 500, forum, movies, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://wordpress.org/) [WordPressOrg (https://wordpress.org/)](https://wordpress.org/)*: top 500, in*
+1. ![](https://www.google.com/s2/favicons?domain=https://ameblo.jp) [Ameblo (https://ameblo.jp)](https://ameblo.jp)*: top 500, blog, jp*
 1. ![](https://www.google.com/s2/favicons?domain=https://unsplash.com/) [Unsplash (https://unsplash.com/)](https://unsplash.com/)*: top 500, art, photo*
 1. ![](https://www.google.com/s2/favicons?domain=https://steamcommunity.com/) [Steam (https://steamcommunity.com/)](https://steamcommunity.com/)*: top 500, gaming*
 1. ![](https://www.google.com/s2/favicons?domain=https://steamcommunity.com/) [Steam (by id) (https://steamcommunity.com/)](https://steamcommunity.com/)*: top 500, gaming*
@@ -131,12 +133,12 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://www.kickstarter.com) [Kickstarter (https://www.kickstarter.com)](https://www.kickstarter.com)*: top 1K, finance, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://forums.ea.com) [forums.ea.com (https://forums.ea.com)](https://forums.ea.com)*: top 1K, forum, gaming, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://forums.envato.com) [Envato (https://forums.envato.com)](https://forums.envato.com)*: top 1K, au, forum, in*
-1. ![](https://www.google.com/s2/favicons?domain=https://giphy.com/) [Giphy (https://giphy.com/)](https://giphy.com/)*: top 1K, photo, us, video*
 1. ![](https://www.google.com/s2/favicons?domain=https://ultimate-guitar.com/) [Ultimate-Guitar (https://ultimate-guitar.com/)](https://ultimate-guitar.com/)*: top 1K, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.freelancer.com/) [Freelancer.com (https://www.freelancer.com/)](https://www.freelancer.com/)*: top 1K, freelance, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://youporn.com) [YouPorn (https://youporn.com)](https://youporn.com)*: top 1K, porn, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.dreamstime.com) [Dreamstime (https://www.dreamstime.com)](https://www.dreamstime.com)*: top 1K, art, photo, stock*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.theverge.com) [TheVerge (https://www.theverge.com)](https://www.theverge.com)*: top 1K, us*
+1. ![](https://www.google.com/s2/favicons?domain=https://giphy.com) [giphy.com (https://giphy.com)](https://giphy.com)*: top 1K, video*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.championat.com/) [Championat (https://www.championat.com/)](https://www.championat.com/)*: top 1K, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.wattpad.com/) [Wattpad (https://www.wattpad.com/)](https://www.wattpad.com/)*: top 1K, reading, writing*
 1. ![](https://www.google.com/s2/favicons?domain=https://disqus.com/) [Disqus (https://disqus.com/)](https://disqus.com/)*: top 1K, discussion*
@@ -246,6 +248,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://4pda.ru/) [4pda (https://4pda.ru/)](https://4pda.ru/)*: top 5K, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.weforum.org) [Weforum (https://www.weforum.org)](https://www.weforum.org)*: top 5K, forum, us*
 1. ![](https://www.google.com/s2/favicons?domain=http://www.techspot.com/community/) [techspot.com (http://www.techspot.com/community/)](http://www.techspot.com/community/)*: top 5K, forum, us*
+1. ![](https://www.google.com/s2/favicons?domain=https://lyricstranslate.com) [lyricstranslate.com (https://lyricstranslate.com)](https://lyricstranslate.com)*: top 5K, music*
 1. ![](https://www.google.com/s2/favicons?domain=https://venmo.com/) [Venmo (https://venmo.com/)](https://venmo.com/)*: top 5K, finance, us*
 1. ![](https://www.google.com/s2/favicons?domain=http://www.wikidot.com/) [Wikidot (http://www.wikidot.com/)](http://www.wikidot.com/)*: top 5K, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://letterboxd.com/) [Letterboxd (https://letterboxd.com/)](https://letterboxd.com/)*: top 5K, us*
@@ -254,6 +257,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://weheartit.com/) [We Heart It (https://weheartit.com/)](https://weheartit.com/)*: top 5K, blog, in, photo*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.filmweb.pl/user/adam) [FilmWeb (https://www.filmweb.pl/user/adam)](https://www.filmweb.pl/user/adam)*: top 5K, movies, pl*
 1. ![](https://www.google.com/s2/favicons?domain=http://forums.bulbagarden.net) [forums.bulbagarden.net (http://forums.bulbagarden.net)](http://forums.bulbagarden.net)*: top 5K, forum, us*
+1. ![](https://www.google.com/s2/favicons?domain=https://videohive.net) [videohive.net (https://videohive.net)](https://videohive.net)*: top 5K, video*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.boardgamegeek.com) [BoardGameGeek (https://www.boardgamegeek.com)](https://www.boardgamegeek.com)*: top 5K, gaming, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://osu.ppy.sh/) [osu! (https://osu.ppy.sh/)](https://osu.ppy.sh/)*: top 5K, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://app.pluralsight.com) [Pluralsight (https://app.pluralsight.com)](https://app.pluralsight.com)*: top 5K, in, us*
@@ -392,8 +396,8 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://www.animenewsnetwork.com) [AnimeNewsNetwork (https://www.animenewsnetwork.com)](https://www.animenewsnetwork.com)*: top 100K, gb, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.smule.com/) [Smule (https://www.smule.com/)](https://www.smule.com/)*: top 100K, music*
 1. ![](https://www.google.com/s2/favicons?domain=https://tvtropes.org) [TVTropes (https://tvtropes.org)](https://tvtropes.org)*: top 100K, us*
-1. ![](https://www.google.com/s2/favicons?domain=https://author.today) [author.today (https://author.today)](https://author.today)*: top 100K, ru*
-1. ![](https://www.google.com/s2/favicons?domain=https://www.thesimsresource.com/) [TheSimsResource (https://www.thesimsresource.com/)](https://www.thesimsresource.com/)*: top 100K, de, gaming, it, us*
+1. ![](https://www.google.com/s2/favicons?domain=https://author.today) [author.today (https://author.today)](https://author.today)*: top 100K, reading, ru*
+1. ![](https://www.google.com/s2/favicons?domain=https://www.thesimsresource.com/) [TheSimsResource (https://www.thesimsresource.com/)](https://www.thesimsresource.com/)*: top 100K, gaming*
 1. ![](https://www.google.com/s2/favicons?domain=https://n4g.com/) [N4g (https://n4g.com/)](https://n4g.com/)*: top 100K, gaming, news, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://teletype.in) [Teletype (https://teletype.in)](https://teletype.in)*: top 100K, in, writing*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.empflix.com) [Empflix (https://www.empflix.com)](https://www.empflix.com)*: top 100K, de, fr, porn*
@@ -453,6 +457,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://jimdosite.com/) [Jimdo (https://jimdosite.com/)](https://jimdosite.com/)*: top 100K, jp*
 1. ![](https://www.google.com/s2/favicons?domain=https://club.cnews.ru/) [club.cnews.ru (https://club.cnews.ru/)](https://club.cnews.ru/)*: top 100K, blog, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://psnprofiles.com/) [PSNProfiles.com (https://psnprofiles.com/)](https://psnprofiles.com/)*: top 100K, gaming*
+1. ![](https://www.google.com/s2/favicons?domain=https://donorbox.org) [donorbox (https://donorbox.org)](https://donorbox.org)*: top 100K, finance*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.sbazar.cz/) [Sbazar.cz (https://www.sbazar.cz/)](https://www.sbazar.cz/)*: top 100K, cz, shopping*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.euro-football.ru) [EuroFootball (https://www.euro-football.ru)](https://www.euro-football.ru)*: top 100K, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://raidforums.com/) [Raidforums (https://raidforums.com/)](https://raidforums.com/)*: top 100K, cybercriminal, forum*
@@ -488,9 +493,10 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://forum.esetnod32.ru) [ESET (https://forum.esetnod32.ru)](https://forum.esetnod32.ru)*: top 100K, forum, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://dreamwidth.org/profile) [Dreamwidth (https://dreamwidth.org/profile)](https://dreamwidth.org/profile)*: top 100K, in, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.sparkpeople.com) [sparkpeople (https://www.sparkpeople.com)](https://www.sparkpeople.com)*: top 100K, us*
-1. ![](https://www.google.com/s2/favicons?domain=https://www.destructoid.com) [Destructoid (https://www.destructoid.com)](https://www.destructoid.com)*: top 100K, us*
+1. ![](https://www.google.com/s2/favicons?domain=https://www.destructoid.com) [Destructoid (https://www.destructoid.com)](https://www.destructoid.com)*: top 100K, us*, search is disabled
 1. ![](https://www.google.com/s2/favicons?domain=https://uid.me/) [uID.me (by username) (https://uid.me/)](https://uid.me/)*: top 100K, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://uid.me/) [uID.me (by uguid) (https://uid.me/)](https://uid.me/)*: top 100K, ru*
+1. ![](https://www.google.com/s2/favicons?domain=https://observablehq.com) [Observable (https://observablehq.com)](https://observablehq.com)*: top 100K, sharing*
 1. ![](https://www.google.com/s2/favicons?domain=https://overclockers.ru) [Overclockers (https://overclockers.ru)](https://overclockers.ru)*: top 100K, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.hackingwithswift.com) [HackingWithSwift (https://www.hackingwithswift.com)](https://www.hackingwithswift.com)*: top 100K, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.younow.com/) [YouNow (https://www.younow.com/)](https://www.younow.com/)*: top 100K, be, us*
@@ -526,6 +532,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://forums.battlefield.com) [forums.battlefield.com (https://forums.battlefield.com)](https://forums.battlefield.com)*: top 100K, forum, gaming, gb, us*, search is disabled
 1. ![](https://www.google.com/s2/favicons?domain=https://gotovim-doma.ru) [GotovimDoma (https://gotovim-doma.ru)](https://gotovim-doma.ru)*: top 100K, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://forums.prosportsdaily.com) [prosportsdaily (https://forums.prosportsdaily.com)](https://forums.prosportsdaily.com)*: top 100K, forum, in, us*
+1. ![](https://www.google.com/s2/favicons?domain=https://clarity.fm) [clarity.fm (https://clarity.fm)](https://clarity.fm)*: top 100K, business*
 1. ![](https://www.google.com/s2/favicons?domain=https://bukkit.org/) [Bukkit (https://bukkit.org/)](https://bukkit.org/)*: top 100K, at, forum, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://elakiri.com) [Elakiri (https://elakiri.com)](https://elakiri.com)*: top 100K, lk*
 1. ![](https://www.google.com/s2/favicons?domain=https://manutd.one) [Manutd (https://manutd.one)](https://manutd.one)*: top 100K, forum, sport*
@@ -778,6 +785,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://www.mobile-files.com/) [Mobile-files (https://www.mobile-files.com/)](https://www.mobile-files.com/)*: top 10M, forum, ru, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.fluther.com/) [Fluther (https://www.fluther.com/)](https://www.fluther.com/)*: top 10M, in, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.comedy.co.uk) [Comedy (https://www.comedy.co.uk)](https://www.comedy.co.uk)*: top 10M, gb, in, movies, pk, us*
+1. ![](https://www.google.com/s2/favicons?domain=https://sessionize.com) [sessionize.com (https://sessionize.com)](https://sessionize.com)*: top 10M, business*
 1. ![](https://www.google.com/s2/favicons?domain=https://fireworktv.com) [Fireworktv (https://fireworktv.com)](https://fireworktv.com)*: top 10M, in, jp*
 1. ![](https://www.google.com/s2/favicons?domain=https://forums.funcom.com) [funcom (https://forums.funcom.com)](https://forums.funcom.com)*: top 10M, forum, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://royalcams.com) [RoyalCams (https://royalcams.com)](https://royalcams.com)*: top 10M, gr, in, ng, ru, us, webcam*
@@ -817,6 +825,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=http://www.ustream.tv) [Ustream (http://www.ustream.tv)](http://www.ustream.tv)*: top 10M, eg, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://geodesist.ru) [Geodesist (https://geodesist.ru)](https://geodesist.ru)*: top 10M, forum, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://serveradmin.ru/) [Serveradmin (https://serveradmin.ru/)](https://serveradmin.ru/)*: top 10M, ru*
+1. ![](https://www.google.com/s2/favicons?domain=https://telescope.ac) [telescope.ac (https://telescope.ac)](https://telescope.ac)*: top 10M, blog*
 1. ![](https://www.google.com/s2/favicons?domain=https://ya-uchitel.ru/) [Ya-uchitel (https://ya-uchitel.ru/)](https://ya-uchitel.ru/)*: top 10M, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.residentadvisor.net) [ResidentAdvisor (https://www.residentadvisor.net)](https://www.residentadvisor.net)*: top 10M, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://weburg.net) [Weburg (https://weburg.net)](https://weburg.net)*: top 10M, ru*
@@ -840,6 +849,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=http://www.cqham.ru) [Cqham (http://www.cqham.ru)](http://www.cqham.ru)*: top 10M, ru, tech*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.kharkovforum.com/) [KharkovForum (https://www.kharkovforum.com/)](https://www.kharkovforum.com/)*: top 10M, forum, ua*
 1. ![](https://www.google.com/s2/favicons?domain=https://studwork.org/) [Studwork (https://studwork.org/)](https://studwork.org/)*: top 10M, ru*
+1. ![](https://www.google.com/s2/favicons?domain=https://forum.freeton.org) [forum.freeton.org (https://forum.freeton.org)](https://forum.freeton.org)*: top 10M, finance, forum*
 1. ![](https://www.google.com/s2/favicons?domain=https://playlists.net) [Playlists (https://playlists.net)](https://playlists.net)*: top 10M, in, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://liberapay.com) [Liberapay (https://liberapay.com)](https://liberapay.com)*: top 10M, eg, finance, in, pk, us, za*
 1. ![](https://www.google.com/s2/favicons?domain=https://forum.artinvestment.ru/) [artinvestment (https://forum.artinvestment.ru/)](https://forum.artinvestment.ru/)*: top 10M, forum, ru*
@@ -871,6 +881,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://www.paypal.me) [Paypal (https://www.paypal.me)](https://www.paypal.me)*: top 10M, finance*
 1. ![](https://www.google.com/s2/favicons?domain=https://seatracker.ru/) [Seatracker (https://seatracker.ru/)](https://seatracker.ru/)*: top 10M, ru*
 1. ![](https://www.google.com/s2/favicons?domain=http://hctorpedo.ru) [Hctorpedo (http://hctorpedo.ru)](http://hctorpedo.ru)*: top 10M, ru*
+1. ![](https://www.google.com/s2/favicons?domain=https://getmakerlog.com) [getmakerlog.com (https://getmakerlog.com)](https://getmakerlog.com)*: top 10M, business*
 1. ![](https://www.google.com/s2/favicons?domain=https://cmet4uk.ru) [Cmet4uk (https://cmet4uk.ru)](https://cmet4uk.ru)*: top 10M, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://forum.popjustice.com) [popjustice (https://forum.popjustice.com)](https://forum.popjustice.com)*: top 10M, co, forum, in, sg, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://rpggeek.com) [RPGGeek (https://rpggeek.com)](https://rpggeek.com)*: top 10M, gaming, us*
@@ -2518,5 +2529,39 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://discuss.hashicorp.com) [discuss.hashicorp.com (https://discuss.hashicorp.com)](https://discuss.hashicorp.com)*: top 100M, tech*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.blogger.com) [Blogger (by GAIA id) (https://www.blogger.com)](https://www.blogger.com)*: top 100M, blog*
 1. ![](https://www.google.com/s2/favicons?domain=http://weebly.com) [Weebly (http://weebly.com)](http://weebly.com)*: top 100M, business*
+1. ![](https://www.google.com/s2/favicons?domain=http://answerszuvs3gg2l64e6hmnryudl5zgrmwm3vh65hzszdghblddvfiqd.onion) [HiddenAnswers (http://answerszuvs3gg2l64e6hmnryudl5zgrmwm3vh65hzszdghblddvfiqd.onion)](http://answerszuvs3gg2l64e6hmnryudl5zgrmwm3vh65hzszdghblddvfiqd.onion)*: top 100M, tor*
+1. ![](https://www.google.com/s2/favicons?domain={username}.com) [.com ({username}.com)]({username}.com)*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=https://galactictalk.org) [galactictalk.org (https://galactictalk.org)](https://galactictalk.org)*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=https://discuss.bootstrapped.fm) [discuss.bootstrapped.fm (https://discuss.bootstrapped.fm)](https://discuss.bootstrapped.fm)*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=https://discourse.mozilla.org) [discourse.mozilla.org (https://discourse.mozilla.org)](https://discourse.mozilla.org)*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=http://ipinit.in) [ipinit.in (http://ipinit.in)](http://ipinit.in)*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=https://boards.theforce.net) [boards.theforce.net (https://boards.theforce.net)](https://boards.theforce.net)*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=https://aussiehomebrewer.com) [aussiehomebrewer.com (https://aussiehomebrewer.com)](https://aussiehomebrewer.com)*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=https://forum-ukraina.net) [forum-ukraina.net (https://forum-ukraina.net)](https://forum-ukraina.net)*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=http://forum-history.ru) [forum-history.ru (http://forum-history.ru)](http://forum-history.ru)*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=http://forum.vn.ua) [forum.vn.ua (http://forum.vn.ua)](http://forum.vn.ua)*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=https://forum.bestflowers.ru) [forum.bestflowers.ru (https://forum.bestflowers.ru)](https://forum.bestflowers.ru)*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=https://forum.alconar.ru) [forum.alconar.ru (https://forum.alconar.ru)](https://forum.alconar.ru)*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=https://forum.lancerx.ru) [forum.lancerx.ru (https://forum.lancerx.ru)](https://forum.lancerx.ru)*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=http://www.mfarmer.ru) [mfarmer.ru (http://www.mfarmer.ru)](http://www.mfarmer.ru)*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=https://forum.league17.ru) [forum.league17.ru (https://forum.league17.ru)](https://forum.league17.ru)*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=https://krskforum.com) [krskforum.com (https://krskforum.com)](https://krskforum.com)*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=https://forum.rarib.ag) [forum.rarib.ag (https://forum.rarib.ag)](https://forum.rarib.ag)*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=https://forum.oneclickchicks.com) [forum.oneclickchicks.com (https://forum.oneclickchicks.com)](https://forum.oneclickchicks.com)*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=http://forum.trade-print.ru) [forum.trade-print.ru (http://forum.trade-print.ru)](http://forum.trade-print.ru)*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=https://forum.setcombg.com) [forum.setcombg.com (https://forum.setcombg.com)](https://forum.setcombg.com)*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=https://vw-bus.ru) [vw-bus.ru (https://vw-bus.ru)](https://vw-bus.ru)*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=https://forum.ya1.ru) [forum.ya1.ru (https://forum.ya1.ru)](https://forum.ya1.ru)*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=https://forum.wordreference.com) [forum.wordreference.com (https://forum.wordreference.com)](https://forum.wordreference.com)*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=https://forums.zooclub.ru) [forums.zooclub.ru (https://forums.zooclub.ru)](https://forums.zooclub.ru)*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=https://www.homebrewtalk.com) [homebrewtalk.com (https://www.homebrewtalk.com)](https://www.homebrewtalk.com)*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=https://navimba.com) [navimba.com (https://navimba.com)](https://navimba.com)*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=https://www.niva-club.net) [niva-club.net (https://www.niva-club.net)](https://www.niva-club.net)*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=https://red-forum.com) [red-forum.com (https://red-forum.com)](https://red-forum.com)*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=http://www.scaleforum.ru) [scaleforum.ru (http://www.scaleforum.ru)](http://www.scaleforum.ru)*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=https://sign-forum.ru) [sign-forum.ru (https://sign-forum.ru)](https://sign-forum.ru)*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=https://rec.poker) [rec.poker (https://rec.poker)](https://rec.poker)*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=https://uforum.uz) [uforum.uz (https://uforum.uz)](https://uforum.uz)*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=http://dntrustmucd4mwec.onion) [DarkNet Trust (http://dntrustmucd4mwec.onion)](http://dntrustmucd4mwec.onion)*: top 100M, tor*
 
-Alexa.com rank data fetched at (2021-05-16 14:01:29.561381 UTC)
+Alexa.com rank data fetched at (2021-05-31 21:26:56.886650 UTC)

tests/test_activation.py

@@ -40,7 +40,7 @@ async def test_import_aiohttp_cookies():
     with open(cookies_filename, 'w') as f:
         f.write(COOKIES_TXT)
 
-    cookie_jar = await import_aiohttp_cookies(cookies_filename)
+    cookie_jar = import_aiohttp_cookies(cookies_filename)
     assert list(cookie_jar._cookies.keys()) == ['xss.is', 'httpbin.org']
 
     url = 'https://httpbin.org/cookies'

tests/test_checking.py

@@ -9,6 +9,7 @@ def site_result_except(server, username, **kwargs):
     server.expect_request('/url', query_string=query).respond_with_data(**kwargs)
 
 
+@pytest.mark.slow
 @pytest.mark.asyncio
 async def test_checking_by_status_code(httpserver, local_test_db):
     sites_dict = local_test_db.sites_dict
@@ -23,6 +24,7 @@ async def test_checking_by_status_code(httpserver, local_test_db):
     assert result['StatusCode']['status'].is_found() is False
 
 
+@pytest.mark.slow
 @pytest.mark.asyncio
 async def test_checking_by_message_positive_full(httpserver, local_test_db):
     sites_dict = local_test_db.sites_dict
@@ -37,6 +39,7 @@ async def test_checking_by_message_positive_full(httpserver, local_test_db):
     assert result['Message']['status'].is_found() is False
 
 
+@pytest.mark.slow
 @pytest.mark.asyncio
 async def test_checking_by_message_positive_part(httpserver, local_test_db):
     sites_dict = local_test_db.sites_dict
@@ -51,6 +54,7 @@ async def test_checking_by_message_positive_part(httpserver, local_test_db):
     assert result['Message']['status'].is_found() is False
 
 
+@pytest.mark.slow
 @pytest.mark.asyncio
 async def test_checking_by_message_negative(httpserver, local_test_db):
     sites_dict = local_test_db.sites_dict

tests/test_cli.py

@@ -25,17 +25,21 @@ DEFAULT_ARGS: Dict[str, Any] = {
     'print_check_errors': False,
     'print_not_found': False,
     'proxy': None,
+    'reports_sorting': 'default',
     'retries': 1,
     'self_check': False,
     'site_list': [],
     'stats': False,
     'tags': '',
     'timeout': 30,
+    'tor_proxy': 'socks5://127.0.0.1:9050',
+    'i2p_proxy': 'http://127.0.0.1:4444',
     'top_sites': 500,
     'txt': False,
     'use_disabled_sites': False,
     'username': [],
     'verbose': False,
+    'with_domains': False,
     'xmind': False,
 }
 

tests/test_maigret.py

@@ -138,6 +138,7 @@ def test_maigret_results(test_db):
 
     assert results['Reddit'].get('future') is None
     del results['GooglePlayStore']['future']
+    del results['GooglePlayStore']['checker']
 
     assert results == RESULTS_EXAMPLE
 

tests/test_report.py

@@ -45,6 +45,19 @@ EXAMPLE_RESULTS = {
     }
 }
 
+BROKEN_RESULTS = {
+    'GitHub': {
+        'username': 'test',
+        'parsing_enabled': True,
+        'url_main': 'https://www.github.com/',
+        'url_user': 'https://www.github.com/test',
+        'http_status': 200,
+        'is_similar': False,
+        'rank': 78,
+        'site': MaigretSite('test', {}),
+    }
+}
+
 GOOD_500PX_RESULT = copy.deepcopy(GOOD_RESULT)
 GOOD_500PX_RESULT.tags = ['photo', 'us', 'global']
 GOOD_500PX_RESULT.ids_data = {
@@ -239,10 +252,13 @@ TEST = [
 ]
 
 SUPPOSED_BRIEF = """Search by username alexaimephotographycars returned 1 accounts. Found target's other IDs: alexaimephotography, Alexaimephotogr. Search by username alexaimephotography returned 2 accounts. Search by username Alexaimephotogr returned 1 accounts. Extended info extracted from 3 accounts."""
-
-SUPPOSED_INTERESTS = "Interests: photo <span class=\"text-muted\">(2)</span>, news <span class=\"text-muted\">(1)</span>, social <span class=\"text-muted\">(1)</span>"
+SUPPOSED_BROKEN_BRIEF = """Search by username alexaimephotographycars returned 0 accounts. Search by username alexaimephotography returned 2 accounts. Search by username Alexaimephotogr returned 1 accounts. Extended info extracted from 2 accounts."""
 
 SUPPOSED_GEO = "Geo: us <span class=\"text-muted\">(3)</span>"
+SUPPOSED_BROKEN_GEO = "Geo: us <span class=\"text-muted\">(2)</span>"
+
+SUPPOSED_INTERESTS = "Interests: photo <span class=\"text-muted\">(2)</span>, news <span class=\"text-muted\">(1)</span>, social <span class=\"text-muted\">(1)</span>"
+SUPPOSED_BROKEN_INTERESTS = "Interests: news <span class=\"text-muted\">(1)</span>, photo <span class=\"text-muted\">(1)</span>, social <span class=\"text-muted\">(1)</span>"
 
 
 def test_generate_report_template():
@@ -270,6 +286,19 @@ def test_generate_csv_report():
     ]
 
 
+def test_generate_csv_report_broken():
+    csvfile = StringIO()
+    generate_csv_report('test', BROKEN_RESULTS, csvfile)
+
+    csvfile.seek(0)
+    data = csvfile.readlines()
+
+    assert data == [
+        'username,name,url_main,url_user,exists,http_status\r\n',
+        'test,GitHub,https://www.github.com/,https://www.github.com/test,Unknown,200\r\n',
+    ]
+
+
 def test_generate_txt_report():
     txtfile = StringIO()
     generate_txt_report('test', EXAMPLE_RESULTS, txtfile)
@@ -283,6 +312,18 @@ def test_generate_txt_report():
     ]
 
 
+def test_generate_txt_report_broken():
+    txtfile = StringIO()
+    generate_txt_report('test', BROKEN_RESULTS, txtfile)
+
+    txtfile.seek(0)
+    data = txtfile.readlines()
+
+    assert data == [
+        'Total Websites Username Detected On : 0',
+    ]
+
+
 def test_generate_json_simple_report():
     jsonfile = StringIO()
     MODIFIED_RESULTS = dict(EXAMPLE_RESULTS)
@@ -296,6 +337,19 @@ def test_generate_json_simple_report():
     assert list(json.loads(data[0]).keys()) == ['GitHub', 'GitHub2']
 
 
+def test_generate_json_simple_report_broken():
+    jsonfile = StringIO()
+    MODIFIED_RESULTS = dict(BROKEN_RESULTS)
+    MODIFIED_RESULTS['GitHub2'] = BROKEN_RESULTS['GitHub']
+    generate_json_report('test', BROKEN_RESULTS, jsonfile, 'simple')
+
+    jsonfile.seek(0)
+    data = jsonfile.readlines()
+
+    assert len(data) == 1
+    assert list(json.loads(data[0]).keys()) == []
+
+
 def test_generate_json_ndjson_report():
     jsonfile = StringIO()
     MODIFIED_RESULTS = dict(EXAMPLE_RESULTS)
@@ -329,6 +383,20 @@ def test_save_xmind_report():
     )
 
 
+def test_save_xmind_report_broken():
+    filename = 'report_test.xmind'
+    save_xmind_report(filename, 'test', BROKEN_RESULTS)
+
+    workbook = xmind.load(filename)
+    sheet = workbook.getPrimarySheet()
+    data = sheet.getData()
+
+    assert data['title'] == 'test Analysis'
+    assert data['topic']['title'] == 'test'
+    assert len(data['topic']['topics']) == 1
+    assert data['topic']['topics'][0]['title'] == 'Undefined'
+
+
 def test_html_report():
     report_name = 'report_test.html'
     context = generate_report_context(TEST)
@@ -341,6 +409,21 @@ def test_html_report():
     assert SUPPOSED_INTERESTS in report_text
 
 
+def test_html_report_broken():
+    report_name = 'report_test_broken.html'
+    BROKEN_DATA = copy.deepcopy(TEST)
+    BROKEN_DATA[0][2]['500px']['status'] = None
+
+    context = generate_report_context(BROKEN_DATA)
+    save_html_report(report_name, context)
+
+    report_text = open(report_name).read()
+
+    assert SUPPOSED_BROKEN_BRIEF in report_text
+    assert SUPPOSED_BROKEN_GEO in report_text
+    assert SUPPOSED_BROKEN_INTERESTS in report_text
+
+
 def test_pdf_report():
     report_name = 'report_test.pdf'
     context = generate_report_context(TEST)
@@ -357,3 +440,16 @@ def test_text_report():
         assert brief_part in report_text
     assert 'us' in report_text
     assert 'photo' in report_text
+
+
+def test_text_report_broken():
+    BROKEN_DATA = copy.deepcopy(TEST)
+    BROKEN_DATA[0][2]['500px']['status'] = None
+
+    context = generate_report_context(BROKEN_DATA)
+    report_text = get_plaintext_report(context)
+
+    for brief_part in SUPPOSED_BROKEN_BRIEF.split():
+        assert brief_part in report_text
+    assert 'us' in report_text
+    assert 'photo' in report_text