Merge pull request #158 from soxoj/dns-resolving

Added DNS checker

maigret/checking.py

@@ -9,6 +9,7 @@ from typing import Tuple, Optional, Dict, List
 from urllib.parse import quote
 
 import aiohttp
+import aiodns
 import tqdm.asyncio
 from aiohttp_socks import ProxyConnector
 from python_socks import _errors as proxy_errors
@@ -43,7 +44,11 @@ SUPPORTED_IDS = (
 BAD_CHARS = "#"
 
 
-class SimpleAiohttpChecker:
+class CheckerBase:
+    pass
+
+
+class SimpleAiohttpChecker(CheckerBase):
     def __init__(self, *args, **kwargs):
         proxy = kwargs.get('proxy')
         cookie_jar = kwargs.get('cookie_jar')
@@ -135,6 +140,48 @@ class TorAiohttpChecker(SimpleAiohttpChecker):
         )
 
 
+class AiodnsDomainResolver(CheckerBase):
+    def __init__(self, *args, **kwargs):
+        loop = asyncio.get_event_loop()
+        self.logger = kwargs.get('logger', Mock())
+        self.resolver = aiodns.DNSResolver(loop=loop)
+
+    def prepare(self, url, headers=None, allow_redirects=True, timeout=0, method='get'):
+        return self.resolver.query(url, 'A')
+
+    async def check(self, future) -> Tuple[str, int, Optional[CheckError]]:
+        status = 404
+        error = None
+        text = ''
+
+        try:
+            res = await future
+            text = str(res[0].host)
+            status = 200
+        except aiodns.error.DNSError:
+            pass
+        except Exception as e:
+            self.logger.error(e, exc_info=True)
+            error = CheckError('DNS resolve error', str(e))
+
+        return text, status, error
+
+
+class CheckerMock:
+    def __init__(self, *args, **kwargs):
+        pass
+
+    def prepare(self, url, headers=None, allow_redirects=True, timeout=0, method='get'):
+        return None
+
+    async def check(self, future) -> Tuple[str, int, Optional[CheckError]]:
+        await asyncio.sleep(0)
+        return '', 0, None
+
+    async def close(self):
+        return
+
+
 # TODO: move to separate class
 def detect_error_page(
     html_text, status_code, fail_flags, ignore_403
@@ -370,7 +417,7 @@ def make_site_result(
     url = re.sub("(?<!:)/+", "/", url)
 
     # always clearweb_checker for now
-    checker = options["checkers"][site.network]
+    checker = options["checkers"][site.protocol]
 
     # site check is disabled
     if site.disabled and not options['forced']:
@@ -518,6 +565,7 @@ async def maigret(
     no_progressbar=False,
     cookies=None,
     retries=0,
+    check_domains=False,
 ) -> QueryResultWrapper:
     """Main search func
 
@@ -571,12 +619,17 @@ async def maigret(
     )
 
     # TODO
-    tor_checker = Mock()
+    tor_checker = CheckerMock()
     if tor_proxy:
         tor_checker = TorAiohttpChecker(  # type: ignore
             proxy=tor_proxy, cookie_jar=cookie_jar, logger=logger
         )
 
+    # TODO
+    dns_checker = CheckerMock()
+    if check_domains:
+        dns_checker = AiodnsDomainResolver(logger=logger)  # type: ignore
+
     if logger.level == logging.DEBUG:
         await debug_ip_request(clearweb_checker, logger)
 
@@ -595,6 +648,7 @@ async def maigret(
     options["checkers"] = {
         '': clearweb_checker,
         'tor': tor_checker,
+        'dns': dns_checker,
     }
     options["parsing"] = is_parsing_enabled
     options["timeout"] = timeout

maigret/maigret.py

@@ -245,6 +245,12 @@ def setup_arguments_parser():
         default='socks5://127.0.0.1:9050',
         help="Specify URL of your Tor gateway. Default is socks5://127.0.0.1:9050",
     )
+    parser.add_argument(
+        "--with-domains",
+        action="store_true",
+        default=False,
+        help="Enable (experimental) feature of checking domains on usernames.",
+    )
 
     filter_group = parser.add_argument_group(
         'Site filtering', 'Options to set site search scope'
@@ -602,6 +608,7 @@ async def main():
             max_connections=args.connections,
             no_progressbar=args.no_progressbar,
             retries=args.retries,
+            check_domains=args.with_domains,
         )
 
         notify_about_errors(results, query_notify)

maigret/resources/data.json

@@ -27749,7 +27749,7 @@
             "tags": [
                 "tor"
             ],
-            "network": "tor",
+            "protocol": "tor",
             "url": "http://answerszuvs3gg2l64e6hmnryudl5zgrmwm3vh65hzszdghblddvfiqd.onion/user/{username}",
             "urlMain": "http://answerszuvs3gg2l64e6hmnryudl5zgrmwm3vh65hzszdghblddvfiqd.onion",
             "usernameClaimed": "theredqueen",
@@ -27761,6 +27761,14 @@
             "presenseStrs": [
                 "qa-part-form-profile"
             ]
+        },
+        ".com": {
+            "protocol": "dns",
+            "url": "{username}.com",
+            "urlMain": "{username}.com",
+            "usernameClaimed": "soxoj",
+            "usernameUnclaimed": "noonewouldeverusethis7",
+            "checkType": "status_code"
         }
     },
     "engines": {

maigret/sites.py

@@ -123,7 +123,7 @@ class MaigretSite:
     alexa_rank = None
     source = None
 
-    network = ''
+    protocol = ''
 
     def __init__(self, name, information):
         self.name = name

requirements.txt

@@ -1,3 +1,4 @@
+aiodns==3.0.0
 aiohttp==3.7.4
 aiohttp-socks==0.5.5
 arabic-reshaper==2.1.1

tests/test_cli.py

@@ -37,6 +37,7 @@ DEFAULT_ARGS: Dict[str, Any] = {
     'use_disabled_sites': False,
     'username': [],
     'verbose': False,
+    'with_domains': False,
     'xmind': False,
 }