Full CookieJar support with test, +1 site

maigret/activation.py

@@ -1,3 +1,10 @@
+import aiohttp
+from aiohttp import CookieJar
+import asyncio
+import json
+from http.cookiejar import MozillaCookieJar
+from http.cookies import Morsel
+
 import requests
 
 class ParsingActivator:
@@ -35,3 +42,23 @@ class ParsingActivator:
         r = requests.post(site.activation['url'], headers=headers, cookies=cookies, data=post_data)
         csrf = r.json()['csrf']
         site.get_params['_xfToken'] = csrf
+
+
+async def import_aiohttp_cookies(cookiestxt_filename):
+    cookies_obj = MozillaCookieJar(cookiestxt_filename)
+    cookies_obj.load(ignore_discard=True, ignore_expires=True)
+
+    cookies = CookieJar()
+
+    cookies_list = []
+    for domain in cookies_obj._cookies.values():
+        for key, cookie in list(domain.values())[0].items():
+            c = Morsel()
+            c.set(key, cookie.value, cookie.value)
+            c['domain'] = cookie.domain
+            c['path'] = cookie.path
+            cookies_list.append((key, c))
+
+    cookies.update_cookies(cookies_list)
+
+    return cookies

maigret/maigret.py

@@ -2,28 +2,24 @@
 Maigret main module
 """
 
-import aiohttp
 import asyncio
-import csv
-import http.cookiejar as cookielib
-import json
 import logging
 import os
 import platform
 import re
-import requests
 import ssl
 import sys
-import tqdm.asyncio
-import xmind
-from aiohttp_socks import ProxyConnector
 from argparse import ArgumentParser, RawDescriptionHelpFormatter
-from http.cookies import SimpleCookie
+
+import aiohttp
+import requests
+import tqdm.asyncio
+from aiohttp_socks import ProxyConnector
 from mock import Mock
 from python_socks import _errors as proxy_errors
 from socid_extractor import parse, extract, __version__ as socid_version
 
-from .activation import ParsingActivator
+from .activation import ParsingActivator, import_aiohttp_cookies
 from .notify import QueryNotifyPrint
 from .report import save_csv_report, save_xmind_report, save_html_report, save_pdf_report, \
     generate_report_context, save_txt_report
@@ -347,15 +343,11 @@ async def maigret(username, site_dict, query_notify, logger,
     # connector = aiohttp.TCPConnector(ssl=False)
     connector.verify_ssl=False
 
-    cookies_dict = {}
+    cookie_jar = None
     if cookies:
-        cookies_obj = cookielib.MozillaCookieJar(cookies)
-        cookies_obj.load(ignore_discard=True, ignore_expires=True)
+        cookie_jar = await import_aiohttp_cookies(cookies)
 
-        for c in cookies_obj:
-            cookies_dict[c.name] = c.value
-
-    session = aiohttp.ClientSession(connector=connector, trust_env=True, cookies=cookies_dict)
+    session = aiohttp.ClientSession(connector=connector, trust_env=True, cookie_jar=cookie_jar)
 
     if logger.level == logging.DEBUG:
         future = session.get(url='https://icanhazip.com')
@@ -386,7 +378,7 @@ async def maigret(username, site_dict, query_notify, logger,
         results_site['username'] = username
         results_site['parsing_enabled'] = recursive_search
         results_site['url_main'] = site.url_main
-        results_site['cookies'] = cookies_dict
+        results_site['cookies'] = cookie_jar and cookie_jar.filter_cookies(site.url_main) or None
 
         headers = {
             'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11.1; rv:55.0) Gecko/20100101 Firefox/55.0',

maigret/resources/data.json

@@ -16120,6 +16120,16 @@
             "usernameClaimed": "alex",
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
+        "Codeby.net": {
+            "tags": [
+                "ru",
+                "hacking"
+            ],
+            "engine": "XenForo",
+            "urlMain": "https://codeby.net",
+            "usernameClaimed": "pragmalion",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
         "freelance.codeby.net": {
             "tags": [
                 "ru"

maigret/sites.py

@@ -2,10 +2,10 @@
 """Maigret Sites Information"""
 import copy
 import json
-import operator
-import requests
 import sys
 
+import requests
+
 from .utils import CaseConverter
 
 

tests/test_activation.py

@@ -1,8 +1,24 @@
 """Maigret activation test functions"""
+import json
+import aiohttp
 import pytest
 from mock import Mock
 
-from maigret.activation import ParsingActivator
+from maigret.activation import ParsingActivator, import_aiohttp_cookies
+
+COOKIES_TXT = """# HTTP Cookie File downloaded with cookies.txt by Genuinous @genuinous
+# This file can be used by wget, curl, aria2c and other standard compliant tools.
+# Usage Examples:
+#   1) wget -x --load-cookies cookies.txt "https://xss.is/search/"
+#   2) curl --cookie cookies.txt "https://xss.is/search/"
+#   3) aria2c --load-cookies cookies.txt "https://xss.is/search/"
+#
+xss.is	FALSE	/	TRUE	0	xf_csrf	test
+xss.is	FALSE	/	TRUE	1642709308	xf_user	tset
+.xss.is	TRUE	/	FALSE	0	muchacho_cache	test
+.xss.is	TRUE	/	FALSE	1924905600	132_evc	test
+httpbin.org	FALSE	/	FALSE	0	a	b
+"""
 
 
 @pytest.mark.slow
@@ -14,3 +30,24 @@ def test_twitter_activation(default_db):
     token2 = twitter_site.headers['x-guest-token']
 
     assert token1 != token2
+
+
+@pytest.mark.asyncio
+async def test_import_aiohttp_cookies():
+    cookies_filename = 'cookies_test.txt'
+    with open(cookies_filename, 'w') as f:
+        f.write(COOKIES_TXT)
+
+    cookie_jar = await import_aiohttp_cookies(cookies_filename)
+    assert list(cookie_jar._cookies.keys()) == ['xss.is', 'httpbin.org']
+
+    url = 'https://httpbin.org/cookies'
+    connector = aiohttp.TCPConnector(ssl=False)
+    session = aiohttp.ClientSession(connector=connector, trust_env=True,
+                                    cookie_jar=cookie_jar)
+
+    response = await session.get(url=url)
+    result = json.loads(await response.content.read())
+    await session.close()
+
+    assert result == {'cookies': {'a': 'b'}}