diff --git a/maigret/activation.py b/maigret/activation.py new file mode 100644 index 0000000..77e6975 --- /dev/null +++ b/maigret/activation.py @@ -0,0 +1,12 @@ +import requests + +class ParsingActivator: + @staticmethod + def twitter(site, logger): + headers = dict(site.headers) + del headers['x-guest-token'] + r = requests.post(site.activation['url'], headers=headers) + logger.info(r) + j = r.json() + guest_token = j[site.activation['src']] + site.headers['x-guest-token'] = guest_token diff --git a/maigret/maigret.py b/maigret/maigret.py index 0aea456..aa4a4be 100755 --- a/maigret/maigret.py +++ b/maigret/maigret.py @@ -23,6 +23,7 @@ import tqdm.asyncio from mock import Mock from socid_extractor import parse, extract +from .activation import ParsingActivator from .notify import QueryNotifyPrint from .result import QueryResult, QueryStatus from .sites import MaigretDatabase, MaigretSite @@ -183,6 +184,17 @@ def process_site_result(response, query_notify, logger, results_info, site: Maig error_text, site_error_text = detect_error_page(html_text, status_code, failure_errors, site.ignore_403) + if site.activation: + is_need_activation = any([s for s in site.activation['marks'] if s in html_text]) + if is_need_activation: + method = site.activation['method'] + try: + activate_fun = getattr(ParsingActivator(), method) + # TODO: async call + activate_fun(site, logger) + except AttributeError: + logger.warning(f'Activation method {method} for site {site.name} not found!') + # presense flags # True by default presense_flags = site.presense_strs @@ -888,6 +900,8 @@ async def main(): if pathPDF or pathHTML: save_html_pdf_report(general_results,pathHTML,pathPDF) + db.save_to_file(args.json_file) + def run(): try: diff --git a/maigret/resources/data.json b/maigret/resources/data.json index 5ab47fd..f30deda 100644 --- a/maigret/resources/data.json +++ b/maigret/resources/data.json @@ -13416,7 +13416,7 @@ "sec-ch-ua": "Google Chrome\";v=\"87\", \" Not;A Brand\";v=\"99\", \"Chromium\";v=\"87\"", "authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA", "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", - "x-guest-token": "1348265069432479753" + "x-guest-token": "1349509919867854849" }, "errors": { "Bad guest token": "x-guest-token update required" @@ -13425,6 +13425,15 @@ "checkType": "message", "absenceStrs": "Not found", "alexaRank": 55, + "activation": { + "method": "twitter", + "marks": [ + "Bad guest token." + ], + "url": "https://api.twitter.com/1.1/guest/activate.json", + "src": "guest_token", + "dst": "x-guest-token" + }, "url": "https://twitter.com/{username}", "urlMain": "https://www.twitter.com/", "usernameClaimed": "blue", diff --git a/maigret/sites.py b/maigret/sites.py index cc31c09..f7db004 100644 --- a/maigret/sites.py +++ b/maigret/sites.py @@ -33,6 +33,7 @@ class MaigretSite: self.type = 'username' self.headers = {} self.errors = {} + self.activation = {} self.url_subpath = '' self.regex_check = None self.url_probe = None diff --git a/tests/conftest.py b/tests/conftest.py index 133e2bc..fe4ebca 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,11 @@ from _pytest.mark import Mark +from mock import Mock +import os +import pytest +from maigret.sites import MaigretDatabase, MaigretSite +JSON_FILE = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../maigret/resources/data.json') empty_mark = Mark('', [], {}) @@ -9,4 +14,11 @@ def by_slow_marker(item): def pytest_collection_modifyitems(items): - items.sort(key=by_slow_marker, reverse=False) \ No newline at end of file + items.sort(key=by_slow_marker, reverse=False) + + +@pytest.fixture(scope='session') +def default_db(): + db = MaigretDatabase().load_from_file(JSON_FILE) + + return db diff --git a/tests/test_activation.py b/tests/test_activation.py new file mode 100644 index 0000000..b733cbd --- /dev/null +++ b/tests/test_activation.py @@ -0,0 +1,16 @@ +"""Maigret activation test functions""" +from mock import Mock +import pytest + +from maigret.activation import ParsingActivator + + +@pytest.mark.slow +def test_twitter_activation(default_db): + twitter_site = default_db.sites_dict['Twitter'] + token1 = twitter_site.headers['x-guest-token'] + + ParsingActivator.twitter(twitter_site, Mock()) + token2 = twitter_site.headers['x-guest-token'] + + assert token1 != token2