diff --git a/maigret/checking.py b/maigret/checking.py index 79e31df..4e4324a 100644 --- a/maigret/checking.py +++ b/maigret/checking.py @@ -9,6 +9,7 @@ from typing import Tuple, Optional, Dict, List from urllib.parse import quote import aiohttp +import aiodns import tqdm.asyncio from aiohttp_socks import ProxyConnector from python_socks import _errors as proxy_errors @@ -43,7 +44,11 @@ SUPPORTED_IDS = ( BAD_CHARS = "#" -class SimpleAiohttpChecker: +class CheckerBase: + pass + + +class SimpleAiohttpChecker(CheckerBase): def __init__(self, *args, **kwargs): proxy = kwargs.get('proxy') cookie_jar = kwargs.get('cookie_jar') @@ -135,6 +140,48 @@ class TorAiohttpChecker(SimpleAiohttpChecker): ) +class AiodnsDomainResolver(CheckerBase): + def __init__(self, *args, **kwargs): + loop = asyncio.get_event_loop() + self.logger = kwargs.get('logger', Mock()) + self.resolver = aiodns.DNSResolver(loop=loop) + + def prepare(self, url, headers=None, allow_redirects=True, timeout=0, method='get'): + return self.resolver.query(url, 'A') + + async def check(self, future) -> Tuple[str, int, Optional[CheckError]]: + status = 404 + error = None + text = '' + + try: + res = await future + text = str(res[0].host) + status = 200 + except aiodns.error.DNSError: + pass + except Exception as e: + self.logger.error(e, exc_info=True) + error = CheckError('DNS resolve error', str(e)) + + return text, status, error + + +class CheckerMock: + def __init__(self, *args, **kwargs): + pass + + def prepare(self, url, headers=None, allow_redirects=True, timeout=0, method='get'): + return None + + async def check(self, future) -> Tuple[str, int, Optional[CheckError]]: + await asyncio.sleep(0) + return '', 0, None + + async def close(self): + return + + # TODO: move to separate class def detect_error_page( html_text, status_code, fail_flags, ignore_403 @@ -370,7 +417,7 @@ def make_site_result( url = re.sub("(? QueryResultWrapper: """Main search func @@ -571,12 +619,17 @@ async def maigret( ) # TODO - tor_checker = Mock() + tor_checker = CheckerMock() if tor_proxy: tor_checker = TorAiohttpChecker( # type: ignore proxy=tor_proxy, cookie_jar=cookie_jar, logger=logger ) + # TODO + dns_checker = CheckerMock() + if check_domains: + dns_checker = AiodnsDomainResolver(logger=logger) # type: ignore + if logger.level == logging.DEBUG: await debug_ip_request(clearweb_checker, logger) @@ -595,6 +648,7 @@ async def maigret( options["checkers"] = { '': clearweb_checker, 'tor': tor_checker, + 'dns': dns_checker, } options["parsing"] = is_parsing_enabled options["timeout"] = timeout diff --git a/maigret/maigret.py b/maigret/maigret.py index 03b33ac..f969c8b 100755 --- a/maigret/maigret.py +++ b/maigret/maigret.py @@ -245,6 +245,12 @@ def setup_arguments_parser(): default='socks5://127.0.0.1:9050', help="Specify URL of your Tor gateway. Default is socks5://127.0.0.1:9050", ) + parser.add_argument( + "--with-domains", + action="store_true", + default=False, + help="Enable (experimental) feature of checking domains on usernames.", + ) filter_group = parser.add_argument_group( 'Site filtering', 'Options to set site search scope' @@ -602,6 +608,7 @@ async def main(): max_connections=args.connections, no_progressbar=args.no_progressbar, retries=args.retries, + check_domains=args.with_domains, ) notify_about_errors(results, query_notify) diff --git a/maigret/resources/data.json b/maigret/resources/data.json index fd620ea..f9f0db0 100644 --- a/maigret/resources/data.json +++ b/maigret/resources/data.json @@ -27749,7 +27749,7 @@ "tags": [ "tor" ], - "network": "tor", + "protocol": "tor", "url": "http://answerszuvs3gg2l64e6hmnryudl5zgrmwm3vh65hzszdghblddvfiqd.onion/user/{username}", "urlMain": "http://answerszuvs3gg2l64e6hmnryudl5zgrmwm3vh65hzszdghblddvfiqd.onion", "usernameClaimed": "theredqueen", @@ -27761,6 +27761,14 @@ "presenseStrs": [ "qa-part-form-profile" ] + }, + ".com": { + "protocol": "dns", + "url": "{username}.com", + "urlMain": "{username}.com", + "usernameClaimed": "soxoj", + "usernameUnclaimed": "noonewouldeverusethis7", + "checkType": "status_code" } }, "engines": { diff --git a/maigret/sites.py b/maigret/sites.py index 5d4cf23..c059de4 100644 --- a/maigret/sites.py +++ b/maigret/sites.py @@ -123,7 +123,7 @@ class MaigretSite: alexa_rank = None source = None - network = '' + protocol = '' def __init__(self, name, information): self.name = name diff --git a/requirements.txt b/requirements.txt index 58b110e..ddc2fb3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +aiodns==3.0.0 aiohttp==3.7.4 aiohttp-socks==0.5.5 arabic-reshaper==2.1.1 diff --git a/tests/test_cli.py b/tests/test_cli.py index c4600a3..3369164 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -37,6 +37,7 @@ DEFAULT_ARGS: Dict[str, Any] = { 'use_disabled_sites': False, 'username': [], 'verbose': False, + 'with_domains': False, 'xmind': False, }