mirror/maigret
1
0
Fork 0
mirror of https://github.com/soxoj/maigret.git synced 2026-05-06 22:19:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #36 from soxoj/cookies-support

Browse Source 
Cookies support
This commit is contained in:
soxoj
2021-01-27 01:48:19 +03:00
committed by GitHub
parent 2a9558ca46 745dcda260
commit 2a0737735b
6 changed files with 153 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cookies.txt
+15
View File
@@ -0,0 +1,15 @@
# HTTP Cookie File downloaded with cookies.txt by Genuinous @genuinous
# This file can be used by wget, curl, aria2c and other standard compliant tools.
# Usage Examples:
# 1) wget -x --load-cookies cookies.txt "https://xss.is/search/"
# 2) curl --cookie cookies.txt "https://xss.is/search/"
# 3) aria2c --load-cookies cookies.txt "https://xss.is/search/"
#
xss.is FALSE / TRUE 0 xf_csrf PMnZNsr42HETwYEr
xss.is FALSE / TRUE 0 xf_from_search google
xss.is FALSE / TRUE 1642709308 xf_user 215268%2CZNKB_-64Wk-BOpsdtLYy-1UxfS5zGpxWaiEGUhmX
xss.is FALSE / TRUE 0 xf_session sGdxJtP_sKV0LCG8vUQbr6cL670_EFWM
.xss.is TRUE / FALSE 0 muchacho_cache ["00fbb0f2772c9596b0483d6864563cce"]
.xss.is TRUE / FALSE 0 muchacho_png ["00fbb0f2772c9596b0483d6864563cce"]
.xss.is TRUE / FALSE 0 muchacho_etag ["00fbb0f2772c9596b0483d6864563cce"]
.xss.is TRUE / FALSE 1924905600 2e66e4dd94a7a237d0d1b4d50f01e179_evc ["00fbb0f2772c9596b0483d6864563cce"]
maigret/activation.py
+45 -2
View File
@@ -1,8 +1,15 @@
import aiohttp
from aiohttp import CookieJar
import asyncio
import json
from http.cookiejar import MozillaCookieJar
from http.cookies import Morsel
import requests
class ParsingActivator:
@staticmethod
def twitter(site, logger):
def twitter(site, logger, cookies={}):
headers = dict(site.headers)
del headers['x-guest-token']
r = requests.post(site.activation['url'], headers=headers)
@@ -12,10 +19,46 @@ class ParsingActivator:
site.headers['x-guest-token'] = guest_token
@staticmethod
def vimeo(site, logger):
def vimeo(site, logger, cookies={}):
headers = dict(site.headers)
if 'Authorization' in headers:
del headers['Authorization']
r = requests.get(site.activation['url'], headers=headers)
jwt_token = r.json()['jwt']
site.headers['Authorization'] = 'jwt ' + jwt_token
@staticmethod
def xssis(site, logger, cookies={}):
if not cookies:
logger.debug('You must have cookies to activate xss.is parsing!')
return
headers = dict(site.headers)
post_data = {
'_xfResponseType': 'json',
'_xfToken': '1611177919,a2710362e45dad9aa1da381e21941a38'
}
headers['content-type'] = 'application/x-www-form-urlencoded; charset=UTF-8'
r = requests.post(site.activation['url'], headers=headers, cookies=cookies, data=post_data)
csrf = r.json()['csrf']
site.get_params['_xfToken'] = csrf
async def import_aiohttp_cookies(cookiestxt_filename):
cookies_obj = MozillaCookieJar(cookiestxt_filename)
cookies_obj.load(ignore_discard=True, ignore_expires=True)
cookies = CookieJar()
cookies_list = []
for domain in cookies_obj._cookies.values():
for key, cookie in list(domain.values())[0].items():
c = Morsel()
c.set(key, cookie.value, cookie.value)
c['domain'] = cookie.domain
c['path'] = cookie.path
cookies_list.append((key, c))
cookies.update_cookies(cookies_list)
return cookies
maigret/maigret.py
+22 -26
View File
@@ -2,31 +2,27 @@
Maigret main module
"""
import aiohttp
import asyncio
import csv
import http.cookiejar as cookielib
import json
import logging
import os
import platform
import re
import requests
import ssl
import sys
import tqdm.asyncio
import xmind
from aiohttp_socks import ProxyConnector
from argparse import ArgumentParser, RawDescriptionHelpFormatter
from http.cookies import SimpleCookie
import aiohttp
import requests
import tqdm.asyncio
from aiohttp_socks import ProxyConnector
from mock import Mock
from python_socks import _errors as proxy_errors
from socid_extractor import parse, extract, __version__ as socid_version
from .activation import ParsingActivator
from .activation import ParsingActivator, import_aiohttp_cookies
from .notify import QueryNotifyPrint
from .report import save_csv_report, save_xmind_report, save_html_report, save_pdf_report, \
generate_report_context, save_txt_report
generate_report_context, save_txt_report
from .result import QueryResult, QueryStatus
from .sites import MaigretDatabase, MaigretSite
@@ -53,9 +49,6 @@ common_errors = {
unsupported_characters = '#'
cookies_file = 'cookies.txt'
async def get_response(request_future, site_name, logger):
html_text = None
status_code = 0
@@ -310,7 +303,8 @@ def process_site_result(response, query_notify, logger, results_info, site: Maig
async def maigret(username, site_dict, query_notify, logger,
proxy=None, timeout=None, recursive_search=False,
id_type='username', debug=False, forced=False,
max_connections=100, no_progressbar=False):
max_connections=100, no_progressbar=False,
cookies=None):
"""Main search func
Checks for existence of username on various social media sites.
@@ -348,7 +342,12 @@ async def maigret(username, site_dict, query_notify, logger,
connector = ProxyConnector.from_url(proxy) if proxy else aiohttp.TCPConnector(ssl=False)
# connector = aiohttp.TCPConnector(ssl=False)
connector.verify_ssl=False
session = aiohttp.ClientSession(connector=connector, trust_env=True)
cookie_jar = None
if cookies:
cookie_jar = await import_aiohttp_cookies(cookies)
session = aiohttp.ClientSession(connector=connector, trust_env=True, cookie_jar=cookie_jar)
if logger.level == logging.DEBUG:
future = session.get(url='https://icanhazip.com')
@@ -379,6 +378,7 @@ async def maigret(username, site_dict, query_notify, logger,
results_site['username'] = username
results_site['parsing_enabled'] = recursive_search
results_site['url_main'] = site.url_main
results_site['cookies'] = cookie_jar and cookie_jar.filter_cookies(site.url_main) or None
headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11.1; rv:55.0) Gecko/20100101 Firefox/55.0',
@@ -424,6 +424,8 @@ async def maigret(username, site_dict, query_notify, logger,
username=username,
)
for k, v in site.get_params.items():
url_probe += f'&{k}={v}'
if site.check_type == 'status_code' and site.request_head_only:
# In most cases when we are detecting by status code,
@@ -446,16 +448,6 @@ async def maigret(username, site_dict, query_notify, logger,
# The final result of the request will be what is available.
allow_redirects = True
# TODO: cookies using
# def parse_cookies(cookies_str):
# cookies = SimpleCookie()
# cookies.load(cookies_str)
# return {key: morsel.value for key, morsel in cookies.items()}
#
# if os.path.exists(cookies_file):
# cookies_obj = cookielib.MozillaCookieJar(cookies_file)
# cookies_obj.load(ignore_discard=True, ignore_expires=True)
future = request_method(url=url_probe, headers=headers,
allow_redirects=allow_redirects,
timeout=timeout,
@@ -661,6 +653,9 @@ async def main():
parser.add_argument("--json", "-j", metavar="JSON_FILE",
dest="json_file", default=None,
help="Load data from a JSON file or an online, valid, JSON file.")
parser.add_argument("--cookies-jar-file", metavar="COOKIE_FILE",
dest="cookie_file", default=None,
help="File with cookies.")
parser.add_argument("--timeout",
action="store", metavar='TIMEOUT',
dest="timeout", type=timeout_check, default=10,
@@ -887,6 +882,7 @@ async def main():
id_type=id_type,
debug=args.verbose,
logger=logger,
cookies=args.cookie_file,
forced=args.use_disabled_sites,
max_connections=args.connections,
)
maigret/resources/data.json
+30 -1
View File
@@ -1585,6 +1585,25 @@
"usernameClaimed": "adam",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"XSS.is": {
"activation": {
"method": "xssis",
"marks": [
"errorHtml"
],
"url": "https://xss.is/login/keep-alive",
"src": "csrf",
"dst": "x-guest-token"
},
"checkType": "status_code",
"getParams": {
"_xfToken": "1611179947,a2710362e45dad9aa1da381e21941a38"
},
"url": "https://xss.is/index.php?members/find&q={username}&_xfRequestUri=%2Fmembers%2F%3Fkey%3Dmost_messages&_xfWithData=1&_xfResponseType=json",
"urlMain": "https://xss.is",
"usernameClaimed": "adam",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"Battleraprus": {
"tags": [
"ru",
@@ -13502,7 +13521,7 @@
"sec-ch-ua": "Google Chrome\";v=\"87\", \" Not;A Brand\";v=\"99\", \"Chromium\";v=\"87\"",
"authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
"x-guest-token": "1352321534451470337"
"x-guest-token": "1351631725676388352"
},
"errors": {
"Bad guest token": "x-guest-token update required"
@@ -16181,6 +16200,16 @@
"usernameClaimed": "alex",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"Codeby.net": {
"tags": [
"ru",
"hacking"
],
"engine": "XenForo",
"urlMain": "https://codeby.net",
"usernameClaimed": "pragmalion",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"freelance.codeby.net": {
"tags": [
"ru"
maigret/sites.py
+3 -2
View File
@@ -2,10 +2,10 @@
"""Maigret Sites Information"""
import copy
import json
import operator
import requests
import sys
import requests
from .utils import CaseConverter
@@ -38,6 +38,7 @@ class MaigretSite:
self.url_probe = None
self.check_type = ''
self.request_head_only = ''
self.get_params = {}
self.presense_strs = []
self.absence_strs = []
tests/test_activation.py
+38 -1
View File
@@ -1,8 +1,24 @@
"""Maigret activation test functions"""
import json
import aiohttp
import pytest
from mock import Mock
from maigret.activation import ParsingActivator
from maigret.activation import ParsingActivator, import_aiohttp_cookies
COOKIES_TXT = """# HTTP Cookie File downloaded with cookies.txt by Genuinous @genuinous
# This file can be used by wget, curl, aria2c and other standard compliant tools.
# Usage Examples:
# 1) wget -x --load-cookies cookies.txt "https://xss.is/search/"
# 2) curl --cookie cookies.txt "https://xss.is/search/"
# 3) aria2c --load-cookies cookies.txt "https://xss.is/search/"
#
xss.is FALSE / TRUE 0 xf_csrf test
xss.is FALSE / TRUE 1642709308 xf_user tset
.xss.is TRUE / FALSE 0 muchacho_cache test
.xss.is TRUE / FALSE 1924905600 132_evc test
httpbin.org FALSE / FALSE 0 a b
"""
@pytest.mark.slow
@@ -14,3 +30,24 @@ def test_twitter_activation(default_db):
token2 = twitter_site.headers['x-guest-token']
assert token1 != token2
@pytest.mark.asyncio
async def test_import_aiohttp_cookies():
cookies_filename = 'cookies_test.txt'
with open(cookies_filename, 'w') as f:
f.write(COOKIES_TXT)
cookie_jar = await import_aiohttp_cookies(cookies_filename)
assert list(cookie_jar._cookies.keys()) == ['xss.is', 'httpbin.org']
url = 'https://httpbin.org/cookies'
connector = aiohttp.TCPConnector(ssl=False)
session = aiohttp.ClientSession(connector=connector, trust_env=True,
cookie_jar=cookie_jar)
response = await session.get(url=url)
result = json.loads(await response.content.read())
await session.close()
assert result == {'cookies': {'a': 'b'}}