From 69a3d17608ea54aced4eb366f4f22e85b410573a Mon Sep 17 00:00:00 2001 From: Soxoj Date: Thu, 21 Jan 2021 00:21:04 +0300 Subject: [PATCH 1/3] Cookies loading MVP for XSS.is --- cookies.txt | 15 +++++++++++++++ maigret/maigret.py | 31 ++++++++++++++++--------------- maigret/resources/data.json | 7 +++++++ 3 files changed, 38 insertions(+), 15 deletions(-) create mode 100644 cookies.txt diff --git a/cookies.txt b/cookies.txt new file mode 100644 index 0000000..83a253b --- /dev/null +++ b/cookies.txt @@ -0,0 +1,15 @@ +# HTTP Cookie File downloaded with cookies.txt by Genuinous @genuinous +# This file can be used by wget, curl, aria2c and other standard compliant tools. +# Usage Examples: +# 1) wget -x --load-cookies cookies.txt "https://xss.is/search/" +# 2) curl --cookie cookies.txt "https://xss.is/search/" +# 3) aria2c --load-cookies cookies.txt "https://xss.is/search/" +# +xss.is FALSE / TRUE 0 xf_csrf PMnZNsr42HETwYEr +xss.is FALSE / TRUE 0 xf_from_search google +xss.is FALSE / TRUE 1642709308 xf_user 215268%2CZNKB_-64Wk-BOpsdtLYy-1UxfS5zGpxWaiEGUhmX +xss.is FALSE / TRUE 0 xf_session sGdxJtP_sKV0LCG8vUQbr6cL670_EFWM +.xss.is TRUE / FALSE 0 muchacho_cache ["00fbb0f2772c9596b0483d6864563cce"] +.xss.is TRUE / FALSE 0 muchacho_png ["00fbb0f2772c9596b0483d6864563cce"] +.xss.is TRUE / FALSE 0 muchacho_etag ["00fbb0f2772c9596b0483d6864563cce"] +.xss.is TRUE / FALSE 1924905600 2e66e4dd94a7a237d0d1b4d50f01e179_evc ["00fbb0f2772c9596b0483d6864563cce"] diff --git a/maigret/maigret.py b/maigret/maigret.py index 3877150..8ea7384 100755 --- a/maigret/maigret.py +++ b/maigret/maigret.py @@ -53,9 +53,6 @@ common_errors = { unsupported_characters = '#' -cookies_file = 'cookies.txt' - - async def get_response(request_future, site_name, logger): html_text = None status_code = 0 @@ -310,7 +307,8 @@ def process_site_result(response, query_notify, logger, results_info, site: Maig async def maigret(username, site_dict, query_notify, logger, proxy=None, timeout=None, recursive_search=False, id_type='username', debug=False, forced=False, - max_connections=100, no_progressbar=False): + max_connections=100, no_progressbar=False, + cookies=None): """Main search func Checks for existence of username on various social media sites. @@ -348,7 +346,16 @@ async def maigret(username, site_dict, query_notify, logger, connector = ProxyConnector.from_url(proxy) if proxy else aiohttp.TCPConnector(ssl=False) # connector = aiohttp.TCPConnector(ssl=False) connector.verify_ssl=False - session = aiohttp.ClientSession(connector=connector, trust_env=True) + + cookies_dict = {} + if cookies: + cookies_obj = cookielib.MozillaCookieJar(cookies) + cookies_obj.load(ignore_discard=True, ignore_expires=True) + + for c in cookies_obj: + cookies_dict[c.name] = c.value + + session = aiohttp.ClientSession(connector=connector, trust_env=True, cookies=cookies_dict) if logger.level == logging.DEBUG: future = session.get(url='https://icanhazip.com') @@ -446,16 +453,6 @@ async def maigret(username, site_dict, query_notify, logger, # The final result of the request will be what is available. allow_redirects = True - # TODO: cookies using - # def parse_cookies(cookies_str): - # cookies = SimpleCookie() - # cookies.load(cookies_str) - # return {key: morsel.value for key, morsel in cookies.items()} - # - # if os.path.exists(cookies_file): - # cookies_obj = cookielib.MozillaCookieJar(cookies_file) - # cookies_obj.load(ignore_discard=True, ignore_expires=True) - future = request_method(url=url_probe, headers=headers, allow_redirects=allow_redirects, timeout=timeout, @@ -660,6 +657,9 @@ async def main(): parser.add_argument("--json", "-j", metavar="JSON_FILE", dest="json_file", default=None, help="Load data from a JSON file or an online, valid, JSON file.") + parser.add_argument("--cookie", metavar="COOKIE_FILE", + dest="cookie_file", default=None, + help="File with cookies.") parser.add_argument("--timeout", action="store", metavar='TIMEOUT', dest="timeout", type=timeout_check, default=10, @@ -886,6 +886,7 @@ async def main(): id_type=id_type, debug=args.verbose, logger=logger, + cookies=args.cookie_file, forced=args.use_disabled_sites, max_connections=args.connections, ) diff --git a/maigret/resources/data.json b/maigret/resources/data.json index 1137c54..b19c863 100644 --- a/maigret/resources/data.json +++ b/maigret/resources/data.json @@ -1587,6 +1587,13 @@ "usernameClaimed": "adam", "usernameUnclaimed": "noonewouldeverusethis7" }, + "XSS.is": { + "checkType": "status_code", + "url": "https://xss.is/index.php?members/find&q={username}&_xfToken=1611176826%2Ce821e74f39e8436e2b599758f6fa5387&_xfRequestUri=%2Fmembers%2F%3Fkey%3Dmost_messages&_xfWithData=1&_xfResponseType=json", + "urlMain": "https://xss.is", + "usernameClaimed": "adam", + "usernameUnclaimed": "noonewouldeverusethis7" + }, "Battleraprus": { "tags": [ "ru", From 36ccafbb3df3f5c28a7666ea617784a5564fb204 Mon Sep 17 00:00:00 2001 From: Soxoj Date: Thu, 21 Jan 2021 01:08:29 +0300 Subject: [PATCH 2/3] Added XSS.is activation method and GET params support --- maigret/activation.py | 20 ++++++++++++++++++-- maigret/maigret.py | 5 ++++- maigret/resources/data.json | 16 ++++++++++++++-- maigret/sites.py | 1 + 4 files changed, 37 insertions(+), 5 deletions(-) diff --git a/maigret/activation.py b/maigret/activation.py index b9f42cf..78c8a1c 100644 --- a/maigret/activation.py +++ b/maigret/activation.py @@ -2,7 +2,7 @@ import requests class ParsingActivator: @staticmethod - def twitter(site, logger): + def twitter(site, logger, cookies={}): headers = dict(site.headers) del headers['x-guest-token'] r = requests.post(site.activation['url'], headers=headers) @@ -12,10 +12,26 @@ class ParsingActivator: site.headers['x-guest-token'] = guest_token @staticmethod - def vimeo(site, logger): + def vimeo(site, logger, cookies={}): headers = dict(site.headers) if 'Authorization' in headers: del headers['Authorization'] r = requests.get(site.activation['url'], headers=headers) jwt_token = r.json()['jwt'] site.headers['Authorization'] = 'jwt ' + jwt_token + + @staticmethod + def xssis(site, logger, cookies={}): + if not cookies: + logger.debug('You must have cookies to activate xss.is parsing!') + return + + headers = dict(site.headers) + post_data = { + '_xfResponseType': 'json', + '_xfToken': '1611177919,a2710362e45dad9aa1da381e21941a38' + } + headers['content-type'] = 'application/x-www-form-urlencoded; charset=UTF-8' + r = requests.post(site.activation['url'], headers=headers, cookies=cookies, data=post_data) + csrf = r.json()['csrf'] + site.get_params['_xfToken'] = csrf diff --git a/maigret/maigret.py b/maigret/maigret.py index 8ea7384..627399d 100755 --- a/maigret/maigret.py +++ b/maigret/maigret.py @@ -386,6 +386,7 @@ async def maigret(username, site_dict, query_notify, logger, results_site['username'] = username results_site['parsing_enabled'] = recursive_search results_site['url_main'] = site.url_main + results_site['cookies'] = cookies_dict headers = { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11.1; rv:55.0) Gecko/20100101 Firefox/55.0', @@ -431,6 +432,8 @@ async def maigret(username, site_dict, query_notify, logger, username=username, ) + for k, v in site.get_params.items(): + url_probe += f'&{k}={v}' if site.check_type == 'status_code' and site.request_head_only: # In most cases when we are detecting by status code, @@ -657,7 +660,7 @@ async def main(): parser.add_argument("--json", "-j", metavar="JSON_FILE", dest="json_file", default=None, help="Load data from a JSON file or an online, valid, JSON file.") - parser.add_argument("--cookie", metavar="COOKIE_FILE", + parser.add_argument("--cookies-jar-file", metavar="COOKIE_FILE", dest="cookie_file", default=None, help="File with cookies.") parser.add_argument("--timeout", diff --git a/maigret/resources/data.json b/maigret/resources/data.json index b19c863..67498cc 100644 --- a/maigret/resources/data.json +++ b/maigret/resources/data.json @@ -1588,8 +1588,20 @@ "usernameUnclaimed": "noonewouldeverusethis7" }, "XSS.is": { + "activation": { + "method": "xssis", + "marks": [ + "errorHtml" + ], + "url": "https://xss.is/login/keep-alive", + "src": "csrf", + "dst": "x-guest-token" + }, "checkType": "status_code", - "url": "https://xss.is/index.php?members/find&q={username}&_xfToken=1611176826%2Ce821e74f39e8436e2b599758f6fa5387&_xfRequestUri=%2Fmembers%2F%3Fkey%3Dmost_messages&_xfWithData=1&_xfResponseType=json", + "getParams": { + "_xfToken": "1611179947,a2710362e45dad9aa1da381e21941a38" + }, + "url": "https://xss.is/index.php?members/find&q={username}&_xfRequestUri=%2Fmembers%2F%3Fkey%3Dmost_messages&_xfWithData=1&_xfResponseType=json", "urlMain": "https://xss.is", "usernameClaimed": "adam", "usernameUnclaimed": "noonewouldeverusethis7" @@ -13436,7 +13448,7 @@ "sec-ch-ua": "Google Chrome\";v=\"87\", \" Not;A Brand\";v=\"99\", \"Chromium\";v=\"87\"", "authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA", "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", - "x-guest-token": "1350800018744160259" + "x-guest-token": "1351631725676388352" }, "errors": { "Bad guest token": "x-guest-token update required" diff --git a/maigret/sites.py b/maigret/sites.py index f16777e..05c171a 100644 --- a/maigret/sites.py +++ b/maigret/sites.py @@ -38,6 +38,7 @@ class MaigretSite: self.url_probe = None self.check_type = '' self.request_head_only = '' + self.get_params = {} self.presense_strs = [] self.absence_strs = [] From cf8d0958ed8bf66b3cc5309af9d87b1c6858d8ea Mon Sep 17 00:00:00 2001 From: Soxoj Date: Wed, 27 Jan 2021 01:44:21 +0300 Subject: [PATCH 3/3] Full CookieJar support with test, +1 site --- maigret/activation.py | 27 +++++++++++++++++++++++++ maigret/maigret.py | 30 +++++++++++----------------- maigret/resources/data.json | 10 ++++++++++ maigret/sites.py | 4 ++-- tests/test_activation.py | 39 ++++++++++++++++++++++++++++++++++++- 5 files changed, 88 insertions(+), 22 deletions(-) diff --git a/maigret/activation.py b/maigret/activation.py index 78c8a1c..e4cb66b 100644 --- a/maigret/activation.py +++ b/maigret/activation.py @@ -1,3 +1,10 @@ +import aiohttp +from aiohttp import CookieJar +import asyncio +import json +from http.cookiejar import MozillaCookieJar +from http.cookies import Morsel + import requests class ParsingActivator: @@ -35,3 +42,23 @@ class ParsingActivator: r = requests.post(site.activation['url'], headers=headers, cookies=cookies, data=post_data) csrf = r.json()['csrf'] site.get_params['_xfToken'] = csrf + + +async def import_aiohttp_cookies(cookiestxt_filename): + cookies_obj = MozillaCookieJar(cookiestxt_filename) + cookies_obj.load(ignore_discard=True, ignore_expires=True) + + cookies = CookieJar() + + cookies_list = [] + for domain in cookies_obj._cookies.values(): + for key, cookie in list(domain.values())[0].items(): + c = Morsel() + c.set(key, cookie.value, cookie.value) + c['domain'] = cookie.domain + c['path'] = cookie.path + cookies_list.append((key, c)) + + cookies.update_cookies(cookies_list) + + return cookies diff --git a/maigret/maigret.py b/maigret/maigret.py index 627399d..124f9a4 100755 --- a/maigret/maigret.py +++ b/maigret/maigret.py @@ -2,31 +2,27 @@ Maigret main module """ -import aiohttp import asyncio -import csv -import http.cookiejar as cookielib -import json import logging import os import platform import re -import requests import ssl import sys -import tqdm.asyncio -import xmind -from aiohttp_socks import ProxyConnector from argparse import ArgumentParser, RawDescriptionHelpFormatter -from http.cookies import SimpleCookie + +import aiohttp +import requests +import tqdm.asyncio +from aiohttp_socks import ProxyConnector from mock import Mock from python_socks import _errors as proxy_errors from socid_extractor import parse, extract, __version__ as socid_version -from .activation import ParsingActivator +from .activation import ParsingActivator, import_aiohttp_cookies from .notify import QueryNotifyPrint from .report import save_csv_report, save_xmind_report, save_html_report, save_pdf_report, \ - generate_report_context, save_txt_report + generate_report_context, save_txt_report from .result import QueryResult, QueryStatus from .sites import MaigretDatabase, MaigretSite @@ -347,15 +343,11 @@ async def maigret(username, site_dict, query_notify, logger, # connector = aiohttp.TCPConnector(ssl=False) connector.verify_ssl=False - cookies_dict = {} + cookie_jar = None if cookies: - cookies_obj = cookielib.MozillaCookieJar(cookies) - cookies_obj.load(ignore_discard=True, ignore_expires=True) + cookie_jar = await import_aiohttp_cookies(cookies) - for c in cookies_obj: - cookies_dict[c.name] = c.value - - session = aiohttp.ClientSession(connector=connector, trust_env=True, cookies=cookies_dict) + session = aiohttp.ClientSession(connector=connector, trust_env=True, cookie_jar=cookie_jar) if logger.level == logging.DEBUG: future = session.get(url='https://icanhazip.com') @@ -386,7 +378,7 @@ async def maigret(username, site_dict, query_notify, logger, results_site['username'] = username results_site['parsing_enabled'] = recursive_search results_site['url_main'] = site.url_main - results_site['cookies'] = cookies_dict + results_site['cookies'] = cookie_jar and cookie_jar.filter_cookies(site.url_main) or None headers = { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11.1; rv:55.0) Gecko/20100101 Firefox/55.0', diff --git a/maigret/resources/data.json b/maigret/resources/data.json index 67498cc..6f7d7c5 100644 --- a/maigret/resources/data.json +++ b/maigret/resources/data.json @@ -16120,6 +16120,16 @@ "usernameClaimed": "alex", "usernameUnclaimed": "noonewouldeverusethis7" }, + "Codeby.net": { + "tags": [ + "ru", + "hacking" + ], + "engine": "XenForo", + "urlMain": "https://codeby.net", + "usernameClaimed": "pragmalion", + "usernameUnclaimed": "noonewouldeverusethis7" + }, "freelance.codeby.net": { "tags": [ "ru" diff --git a/maigret/sites.py b/maigret/sites.py index 05c171a..7448914 100644 --- a/maigret/sites.py +++ b/maigret/sites.py @@ -2,10 +2,10 @@ """Maigret Sites Information""" import copy import json -import operator -import requests import sys +import requests + from .utils import CaseConverter diff --git a/tests/test_activation.py b/tests/test_activation.py index efe30a9..e37a9a1 100644 --- a/tests/test_activation.py +++ b/tests/test_activation.py @@ -1,8 +1,24 @@ """Maigret activation test functions""" +import json +import aiohttp import pytest from mock import Mock -from maigret.activation import ParsingActivator +from maigret.activation import ParsingActivator, import_aiohttp_cookies + +COOKIES_TXT = """# HTTP Cookie File downloaded with cookies.txt by Genuinous @genuinous +# This file can be used by wget, curl, aria2c and other standard compliant tools. +# Usage Examples: +# 1) wget -x --load-cookies cookies.txt "https://xss.is/search/" +# 2) curl --cookie cookies.txt "https://xss.is/search/" +# 3) aria2c --load-cookies cookies.txt "https://xss.is/search/" +# +xss.is FALSE / TRUE 0 xf_csrf test +xss.is FALSE / TRUE 1642709308 xf_user tset +.xss.is TRUE / FALSE 0 muchacho_cache test +.xss.is TRUE / FALSE 1924905600 132_evc test +httpbin.org FALSE / FALSE 0 a b +""" @pytest.mark.slow @@ -14,3 +30,24 @@ def test_twitter_activation(default_db): token2 = twitter_site.headers['x-guest-token'] assert token1 != token2 + + +@pytest.mark.asyncio +async def test_import_aiohttp_cookies(): + cookies_filename = 'cookies_test.txt' + with open(cookies_filename, 'w') as f: + f.write(COOKIES_TXT) + + cookie_jar = await import_aiohttp_cookies(cookies_filename) + assert list(cookie_jar._cookies.keys()) == ['xss.is', 'httpbin.org'] + + url = 'https://httpbin.org/cookies' + connector = aiohttp.TCPConnector(ssl=False) + session = aiohttp.ClientSession(connector=connector, trust_env=True, + cookie_jar=cookie_jar) + + response = await session.get(url=url) + result = json.loads(await response.content.read()) + await session.close() + + assert result == {'cookies': {'a': 'b'}}