diff --git a/maigret/activation.py b/maigret/activation.py index 78c8a1c..e4cb66b 100644 --- a/maigret/activation.py +++ b/maigret/activation.py @@ -1,3 +1,10 @@ +import aiohttp +from aiohttp import CookieJar +import asyncio +import json +from http.cookiejar import MozillaCookieJar +from http.cookies import Morsel + import requests class ParsingActivator: @@ -35,3 +42,23 @@ class ParsingActivator: r = requests.post(site.activation['url'], headers=headers, cookies=cookies, data=post_data) csrf = r.json()['csrf'] site.get_params['_xfToken'] = csrf + + +async def import_aiohttp_cookies(cookiestxt_filename): + cookies_obj = MozillaCookieJar(cookiestxt_filename) + cookies_obj.load(ignore_discard=True, ignore_expires=True) + + cookies = CookieJar() + + cookies_list = [] + for domain in cookies_obj._cookies.values(): + for key, cookie in list(domain.values())[0].items(): + c = Morsel() + c.set(key, cookie.value, cookie.value) + c['domain'] = cookie.domain + c['path'] = cookie.path + cookies_list.append((key, c)) + + cookies.update_cookies(cookies_list) + + return cookies diff --git a/maigret/maigret.py b/maigret/maigret.py index 627399d..124f9a4 100755 --- a/maigret/maigret.py +++ b/maigret/maigret.py @@ -2,31 +2,27 @@ Maigret main module """ -import aiohttp import asyncio -import csv -import http.cookiejar as cookielib -import json import logging import os import platform import re -import requests import ssl import sys -import tqdm.asyncio -import xmind -from aiohttp_socks import ProxyConnector from argparse import ArgumentParser, RawDescriptionHelpFormatter -from http.cookies import SimpleCookie + +import aiohttp +import requests +import tqdm.asyncio +from aiohttp_socks import ProxyConnector from mock import Mock from python_socks import _errors as proxy_errors from socid_extractor import parse, extract, __version__ as socid_version -from .activation import ParsingActivator +from .activation import ParsingActivator, import_aiohttp_cookies from .notify import QueryNotifyPrint from .report import save_csv_report, save_xmind_report, save_html_report, save_pdf_report, \ - generate_report_context, save_txt_report + generate_report_context, save_txt_report from .result import QueryResult, QueryStatus from .sites import MaigretDatabase, MaigretSite @@ -347,15 +343,11 @@ async def maigret(username, site_dict, query_notify, logger, # connector = aiohttp.TCPConnector(ssl=False) connector.verify_ssl=False - cookies_dict = {} + cookie_jar = None if cookies: - cookies_obj = cookielib.MozillaCookieJar(cookies) - cookies_obj.load(ignore_discard=True, ignore_expires=True) + cookie_jar = await import_aiohttp_cookies(cookies) - for c in cookies_obj: - cookies_dict[c.name] = c.value - - session = aiohttp.ClientSession(connector=connector, trust_env=True, cookies=cookies_dict) + session = aiohttp.ClientSession(connector=connector, trust_env=True, cookie_jar=cookie_jar) if logger.level == logging.DEBUG: future = session.get(url='https://icanhazip.com') @@ -386,7 +378,7 @@ async def maigret(username, site_dict, query_notify, logger, results_site['username'] = username results_site['parsing_enabled'] = recursive_search results_site['url_main'] = site.url_main - results_site['cookies'] = cookies_dict + results_site['cookies'] = cookie_jar and cookie_jar.filter_cookies(site.url_main) or None headers = { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11.1; rv:55.0) Gecko/20100101 Firefox/55.0', diff --git a/maigret/resources/data.json b/maigret/resources/data.json index 67498cc..6f7d7c5 100644 --- a/maigret/resources/data.json +++ b/maigret/resources/data.json @@ -16120,6 +16120,16 @@ "usernameClaimed": "alex", "usernameUnclaimed": "noonewouldeverusethis7" }, + "Codeby.net": { + "tags": [ + "ru", + "hacking" + ], + "engine": "XenForo", + "urlMain": "https://codeby.net", + "usernameClaimed": "pragmalion", + "usernameUnclaimed": "noonewouldeverusethis7" + }, "freelance.codeby.net": { "tags": [ "ru" diff --git a/maigret/sites.py b/maigret/sites.py index 05c171a..7448914 100644 --- a/maigret/sites.py +++ b/maigret/sites.py @@ -2,10 +2,10 @@ """Maigret Sites Information""" import copy import json -import operator -import requests import sys +import requests + from .utils import CaseConverter diff --git a/tests/test_activation.py b/tests/test_activation.py index efe30a9..e37a9a1 100644 --- a/tests/test_activation.py +++ b/tests/test_activation.py @@ -1,8 +1,24 @@ """Maigret activation test functions""" +import json +import aiohttp import pytest from mock import Mock -from maigret.activation import ParsingActivator +from maigret.activation import ParsingActivator, import_aiohttp_cookies + +COOKIES_TXT = """# HTTP Cookie File downloaded with cookies.txt by Genuinous @genuinous +# This file can be used by wget, curl, aria2c and other standard compliant tools. +# Usage Examples: +# 1) wget -x --load-cookies cookies.txt "https://xss.is/search/" +# 2) curl --cookie cookies.txt "https://xss.is/search/" +# 3) aria2c --load-cookies cookies.txt "https://xss.is/search/" +# +xss.is FALSE / TRUE 0 xf_csrf test +xss.is FALSE / TRUE 1642709308 xf_user tset +.xss.is TRUE / FALSE 0 muchacho_cache test +.xss.is TRUE / FALSE 1924905600 132_evc test +httpbin.org FALSE / FALSE 0 a b +""" @pytest.mark.slow @@ -14,3 +30,24 @@ def test_twitter_activation(default_db): token2 = twitter_site.headers['x-guest-token'] assert token1 != token2 + + +@pytest.mark.asyncio +async def test_import_aiohttp_cookies(): + cookies_filename = 'cookies_test.txt' + with open(cookies_filename, 'w') as f: + f.write(COOKIES_TXT) + + cookie_jar = await import_aiohttp_cookies(cookies_filename) + assert list(cookie_jar._cookies.keys()) == ['xss.is', 'httpbin.org'] + + url = 'https://httpbin.org/cookies' + connector = aiohttp.TCPConnector(ssl=False) + session = aiohttp.ClientSession(connector=connector, trust_env=True, + cookie_jar=cookie_jar) + + response = await session.get(url=url) + result = json.loads(await response.content.read()) + await session.close() + + assert result == {'cookies': {'a': 'b'}}