Merge pull request #36 from soxoj/cookies-support

Cookies support

cookies.txt

@@ -0,0 +1,15 @@
+# HTTP Cookie File downloaded with cookies.txt by Genuinous @genuinous
+# This file can be used by wget, curl, aria2c and other standard compliant tools.
+# Usage Examples:
+#   1) wget -x --load-cookies cookies.txt "https://xss.is/search/"
+#   2) curl --cookie cookies.txt "https://xss.is/search/"
+#   3) aria2c --load-cookies cookies.txt "https://xss.is/search/"
+#
+xss.is	FALSE	/	TRUE	0	xf_csrf	PMnZNsr42HETwYEr
+xss.is	FALSE	/	TRUE	0	xf_from_search	google
+xss.is	FALSE	/	TRUE	1642709308	xf_user	215268%2CZNKB_-64Wk-BOpsdtLYy-1UxfS5zGpxWaiEGUhmX
+xss.is	FALSE	/	TRUE	0	xf_session	sGdxJtP_sKV0LCG8vUQbr6cL670_EFWM
+.xss.is	TRUE	/	FALSE	0	muchacho_cache	["00fbb0f2772c9596b0483d6864563cce"]
+.xss.is	TRUE	/	FALSE	0	muchacho_png	["00fbb0f2772c9596b0483d6864563cce"]
+.xss.is	TRUE	/	FALSE	0	muchacho_etag	["00fbb0f2772c9596b0483d6864563cce"]
+.xss.is	TRUE	/	FALSE	1924905600	2e66e4dd94a7a237d0d1b4d50f01e179_evc	["00fbb0f2772c9596b0483d6864563cce"]

maigret/activation.py

@@ -1,8 +1,15 @@
+import aiohttp
+from aiohttp import CookieJar
+import asyncio
+import json
+from http.cookiejar import MozillaCookieJar
+from http.cookies import Morsel
+
 import requests
 
 class ParsingActivator:
     @staticmethod
-    def twitter(site, logger):
+    def twitter(site, logger, cookies={}):
         headers = dict(site.headers)
         del headers['x-guest-token']
         r = requests.post(site.activation['url'], headers=headers)
@@ -12,10 +19,46 @@ class ParsingActivator:
         site.headers['x-guest-token'] = guest_token
 
     @staticmethod
-    def vimeo(site, logger):
+    def vimeo(site, logger, cookies={}):
         headers = dict(site.headers)
         if 'Authorization' in headers:
             del headers['Authorization']
         r = requests.get(site.activation['url'], headers=headers)
         jwt_token = r.json()['jwt']
         site.headers['Authorization'] = 'jwt ' + jwt_token
+
+    @staticmethod
+    def xssis(site, logger, cookies={}):
+        if not cookies:
+            logger.debug('You must have cookies to activate xss.is parsing!')
+            return
+
+        headers = dict(site.headers)
+        post_data = {
+            '_xfResponseType': 'json',
+            '_xfToken': '1611177919,a2710362e45dad9aa1da381e21941a38'
+        }
+        headers['content-type'] = 'application/x-www-form-urlencoded; charset=UTF-8'
+        r = requests.post(site.activation['url'], headers=headers, cookies=cookies, data=post_data)
+        csrf = r.json()['csrf']
+        site.get_params['_xfToken'] = csrf
+
+
+async def import_aiohttp_cookies(cookiestxt_filename):
+    cookies_obj = MozillaCookieJar(cookiestxt_filename)
+    cookies_obj.load(ignore_discard=True, ignore_expires=True)
+
+    cookies = CookieJar()
+
+    cookies_list = []
+    for domain in cookies_obj._cookies.values():
+        for key, cookie in list(domain.values())[0].items():
+            c = Morsel()
+            c.set(key, cookie.value, cookie.value)
+            c['domain'] = cookie.domain
+            c['path'] = cookie.path
+            cookies_list.append((key, c))
+
+    cookies.update_cookies(cookies_list)
+
+    return cookies

maigret/maigret.py

@@ -2,28 +2,24 @@
 Maigret main module
 """
 
-import aiohttp
 import asyncio
-import csv
-import http.cookiejar as cookielib
-import json
 import logging
 import os
 import platform
 import re
-import requests
 import ssl
 import sys
-import tqdm.asyncio
-import xmind
-from aiohttp_socks import ProxyConnector
 from argparse import ArgumentParser, RawDescriptionHelpFormatter
-from http.cookies import SimpleCookie
+
+import aiohttp
+import requests
+import tqdm.asyncio
+from aiohttp_socks import ProxyConnector
 from mock import Mock
 from python_socks import _errors as proxy_errors
 from socid_extractor import parse, extract, __version__ as socid_version
 
-from .activation import ParsingActivator
+from .activation import ParsingActivator, import_aiohttp_cookies
 from .notify import QueryNotifyPrint
 from .report import save_csv_report, save_xmind_report, save_html_report, save_pdf_report, \
     generate_report_context, save_txt_report
@@ -53,9 +49,6 @@ common_errors = {
 
 unsupported_characters = '#'
 
-cookies_file = 'cookies.txt'
-
-
 async def get_response(request_future, site_name, logger):
     html_text = None
     status_code = 0
@@ -310,7 +303,8 @@ def process_site_result(response, query_notify, logger, results_info, site: Maig
 async def maigret(username, site_dict, query_notify, logger,
                   proxy=None, timeout=None, recursive_search=False,
                   id_type='username', debug=False, forced=False,
-                  max_connections=100, no_progressbar=False):
+                  max_connections=100, no_progressbar=False,
+                  cookies=None):
     """Main search func
 
     Checks for existence of username on various social media sites.
@@ -348,7 +342,12 @@ async def maigret(username, site_dict, query_notify, logger,
     connector = ProxyConnector.from_url(proxy) if proxy else aiohttp.TCPConnector(ssl=False)
     # connector = aiohttp.TCPConnector(ssl=False)
     connector.verify_ssl=False
-    session = aiohttp.ClientSession(connector=connector, trust_env=True)
+
+    cookie_jar = None
+    if cookies:
+        cookie_jar = await import_aiohttp_cookies(cookies)
+
+    session = aiohttp.ClientSession(connector=connector, trust_env=True, cookie_jar=cookie_jar)
 
     if logger.level == logging.DEBUG:
         future = session.get(url='https://icanhazip.com')
@@ -379,6 +378,7 @@ async def maigret(username, site_dict, query_notify, logger,
         results_site['username'] = username
         results_site['parsing_enabled'] = recursive_search
         results_site['url_main'] = site.url_main
+        results_site['cookies'] = cookie_jar and cookie_jar.filter_cookies(site.url_main) or None
 
         headers = {
             'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11.1; rv:55.0) Gecko/20100101 Firefox/55.0',
@@ -424,6 +424,8 @@ async def maigret(username, site_dict, query_notify, logger,
                     username=username,
                 )
 
+            for k, v in site.get_params.items():
+                url_probe += f'&{k}={v}'
 
             if site.check_type == 'status_code' and site.request_head_only:
                 # In most cases when we are detecting by status code,
@@ -446,16 +448,6 @@ async def maigret(username, site_dict, query_notify, logger,
                 # The final result of the request will be what is available.
                 allow_redirects = True
 
-            # TODO: cookies using
-            # def parse_cookies(cookies_str):
-            #     cookies = SimpleCookie()
-            #     cookies.load(cookies_str)
-            #     return {key: morsel.value for key, morsel in cookies.items()}
-            #
-            # if os.path.exists(cookies_file):
-            #     cookies_obj = cookielib.MozillaCookieJar(cookies_file)
-            #     cookies_obj.load(ignore_discard=True, ignore_expires=True)
-
             future = request_method(url=url_probe, headers=headers,
                                     allow_redirects=allow_redirects,
                                     timeout=timeout,
@@ -661,6 +653,9 @@ async def main():
     parser.add_argument("--json", "-j", metavar="JSON_FILE",
                         dest="json_file", default=None,
                         help="Load data from a JSON file or an online, valid, JSON file.")
+    parser.add_argument("--cookies-jar-file", metavar="COOKIE_FILE",
+                        dest="cookie_file", default=None,
+                        help="File with cookies.")
     parser.add_argument("--timeout",
                         action="store", metavar='TIMEOUT',
                         dest="timeout", type=timeout_check, default=10,
@@ -887,6 +882,7 @@ async def main():
                                 id_type=id_type,
                                 debug=args.verbose,
                                 logger=logger,
+                                cookies=args.cookie_file,
                                 forced=args.use_disabled_sites,
                                 max_connections=args.connections,
                                 )

maigret/resources/data.json

@@ -1585,6 +1585,25 @@
             "usernameClaimed": "adam",
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
+        "XSS.is": {
+            "activation": {
+                "method": "xssis",
+                "marks": [
+                    "errorHtml"
+                ],
+                "url": "https://xss.is/login/keep-alive",
+                "src": "csrf",
+                "dst": "x-guest-token"
+            },
+            "checkType": "status_code",
+            "getParams": {
+                "_xfToken": "1611179947,a2710362e45dad9aa1da381e21941a38"
+            },
+            "url": "https://xss.is/index.php?members/find&q={username}&_xfRequestUri=%2Fmembers%2F%3Fkey%3Dmost_messages&_xfWithData=1&_xfResponseType=json",
+            "urlMain": "https://xss.is",
+            "usernameClaimed": "adam",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
         "Battleraprus": {
             "tags": [
                 "ru",
@@ -13502,7 +13521,7 @@
                 "sec-ch-ua": "Google Chrome\";v=\"87\", \" Not;A Brand\";v=\"99\", \"Chromium\";v=\"87\"",
                 "authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
                 "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
-                "x-guest-token": "1352321534451470337"
+                "x-guest-token": "1351631725676388352"
             },
             "errors": {
                 "Bad guest token": "x-guest-token update required"
@@ -16181,6 +16200,16 @@
             "usernameClaimed": "alex",
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
+        "Codeby.net": {
+            "tags": [
+                "ru",
+                "hacking"
+            ],
+            "engine": "XenForo",
+            "urlMain": "https://codeby.net",
+            "usernameClaimed": "pragmalion",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
         "freelance.codeby.net": {
             "tags": [
                 "ru"

maigret/sites.py

@@ -2,10 +2,10 @@
 """Maigret Sites Information"""
 import copy
 import json
-import operator
-import requests
 import sys
 
+import requests
+
 from .utils import CaseConverter
 
 
@@ -38,6 +38,7 @@ class MaigretSite:
         self.url_probe = None
         self.check_type = ''
         self.request_head_only = ''
+        self.get_params = {}
 
         self.presense_strs = []
         self.absence_strs = []

tests/test_activation.py

@@ -1,8 +1,24 @@
 """Maigret activation test functions"""
+import json
+import aiohttp
 import pytest
 from mock import Mock
 
-from maigret.activation import ParsingActivator
+from maigret.activation import ParsingActivator, import_aiohttp_cookies
+
+COOKIES_TXT = """# HTTP Cookie File downloaded with cookies.txt by Genuinous @genuinous
+# This file can be used by wget, curl, aria2c and other standard compliant tools.
+# Usage Examples:
+#   1) wget -x --load-cookies cookies.txt "https://xss.is/search/"
+#   2) curl --cookie cookies.txt "https://xss.is/search/"
+#   3) aria2c --load-cookies cookies.txt "https://xss.is/search/"
+#
+xss.is	FALSE	/	TRUE	0	xf_csrf	test
+xss.is	FALSE	/	TRUE	1642709308	xf_user	tset
+.xss.is	TRUE	/	FALSE	0	muchacho_cache	test
+.xss.is	TRUE	/	FALSE	1924905600	132_evc	test
+httpbin.org	FALSE	/	FALSE	0	a	b
+"""
 
 
 @pytest.mark.slow
@@ -14,3 +30,24 @@ def test_twitter_activation(default_db):
     token2 = twitter_site.headers['x-guest-token']
 
     assert token1 != token2
+
+
+@pytest.mark.asyncio
+async def test_import_aiohttp_cookies():
+    cookies_filename = 'cookies_test.txt'
+    with open(cookies_filename, 'w') as f:
+        f.write(COOKIES_TXT)
+
+    cookie_jar = await import_aiohttp_cookies(cookies_filename)
+    assert list(cookie_jar._cookies.keys()) == ['xss.is', 'httpbin.org']
+
+    url = 'https://httpbin.org/cookies'
+    connector = aiohttp.TCPConnector(ssl=False)
+    session = aiohttp.ClientSession(connector=connector, trust_env=True,
+                                    cookie_jar=cookie_jar)
+
+    response = await session.get(url=url)
+    result = json.loads(await response.content.read())
+    await session.close()
+
+    assert result == {'cookies': {'a': 'b'}}