Twitter parsing activation

maigret/activation.py

@@ -0,0 +1,12 @@
+import requests
+
+class ParsingActivator:
+    @staticmethod
+    def twitter(site, logger):
+        headers = dict(site.headers)
+        del headers['x-guest-token']
+        r = requests.post(site.activation['url'], headers=headers)
+        logger.info(r)
+        j = r.json()
+        guest_token = j[site.activation['src']]
+        site.headers['x-guest-token'] = guest_token

maigret/maigret.py

@@ -23,6 +23,7 @@ import tqdm.asyncio
 from mock import Mock
 from socid_extractor import parse, extract
 
+from .activation import ParsingActivator
 from .notify import QueryNotifyPrint
 from .result import QueryResult, QueryStatus
 from .sites import MaigretDatabase, MaigretSite
@@ -183,6 +184,17 @@ def process_site_result(response, query_notify, logger, results_info, site: Maig
         error_text, site_error_text = detect_error_page(html_text, status_code, failure_errors,
                                                         site.ignore_403)
 
+    if site.activation:
+        is_need_activation = any([s for s in site.activation['marks'] if s in html_text])
+        if is_need_activation:
+            method = site.activation['method']
+            try:
+                activate_fun = getattr(ParsingActivator(), method)
+                # TODO: async call
+                activate_fun(site, logger)
+            except AttributeError:
+                logger.warning(f'Activation method {method} for site {site.name} not found!')
+
     # presense flags
     # True by default
     presense_flags = site.presense_strs
@@ -888,6 +900,8 @@ async def main():
         if pathPDF or pathHTML:
             save_html_pdf_report(general_results,pathHTML,pathPDF)
 
+    db.save_to_file(args.json_file)
+
 
 def run():
     try:

maigret/resources/data.json

@@ -13416,7 +13416,7 @@
                 "sec-ch-ua": "Google Chrome\";v=\"87\", \" Not;A Brand\";v=\"99\", \"Chromium\";v=\"87\"",
                 "authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
                 "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
-                "x-guest-token": "1348265069432479753"
+                "x-guest-token": "1349509919867854849"
             },
             "errors": {
                 "Bad guest token": "x-guest-token update required"
@@ -13425,6 +13425,15 @@
             "checkType": "message",
             "absenceStrs": "Not found",
             "alexaRank": 55,
+            "activation": {
+                "method": "twitter",
+                "marks": [
+                    "Bad guest token."
+                ],
+                "url": "https://api.twitter.com/1.1/guest/activate.json",
+                "src": "guest_token",
+                "dst": "x-guest-token"
+            },
             "url": "https://twitter.com/{username}",
             "urlMain": "https://www.twitter.com/",
             "usernameClaimed": "blue",

maigret/sites.py

@@ -33,6 +33,7 @@ class MaigretSite:
         self.type = 'username'
         self.headers = {}
         self.errors = {}
+        self.activation = {}
         self.url_subpath = ''
         self.regex_check = None
         self.url_probe = None

tests/conftest.py

@@ -1,6 +1,11 @@
 from _pytest.mark import Mark
+from mock import Mock
+import os
+import pytest
 
+from maigret.sites import MaigretDatabase, MaigretSite
 
+JSON_FILE = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../maigret/resources/data.json')
 empty_mark = Mark('', [], {})
 
 
@@ -9,4 +14,11 @@ def by_slow_marker(item):
 
 
 def pytest_collection_modifyitems(items):
-    items.sort(key=by_slow_marker, reverse=False)
+    items.sort(key=by_slow_marker, reverse=False)
+
+
+@pytest.fixture(scope='session')
+def default_db():
+    db = MaigretDatabase().load_from_file(JSON_FILE)
+
+    return db

tests/test_activation.py

@@ -0,0 +1,16 @@
+"""Maigret activation test functions"""
+from mock import Mock
+import pytest
+
+from maigret.activation import ParsingActivator
+
+
+@pytest.mark.slow
+def test_twitter_activation(default_db):
+    twitter_site = default_db.sites_dict['Twitter']
+    token1 = twitter_site.headers['x-guest-token']
+
+    ParsingActivator.twitter(twitter_site, Mock())
+    token2 = twitter_site.headers['x-guest-token']
+
+    assert token1 != token2