Mirrors (#630)

* Update checking.py

* Added attempts check and mirrors

Co-authored-by: Soxoj <soxoj@protonmail.com>
Co-authored-by: Soxoj <31013580+soxoj@users.noreply.github.com>

maigret/checking.py

@@ -10,6 +10,7 @@ import re
 import ssl
 import sys
 import tqdm
+import random
 from typing import Tuple, Optional, Dict, List
 from urllib.parse import quote
 
@@ -397,7 +398,7 @@ def process_site_result(
 
 
 def make_site_result(
-    site: MaigretSite, username: str, options: QueryOptions, logger
+    site: MaigretSite, username: str, options: QueryOptions, logger, *args, **kwargs
 ) -> QueryResultWrapper:
     results_site: QueryResultWrapper = {}
 
@@ -421,6 +422,10 @@ def make_site_result(
     if "url" not in site.__dict__:
         logger.error("No URL for site %s", site.name)
 
+    if kwargs.get('retry') and hasattr(site, "mirrors"):
+        site.url_main = random.choice(site.mirrors)
+        logger.info(f"Use {site.url_main} as a main url of site {site}")
+
     # URL of user on site (if it exists)
     url = site.url.format(
         urlMain=site.url_main, urlSubpath=site.url_subpath, username=quote(username)
@@ -524,7 +529,7 @@ def make_site_result(
 async def check_site_for_username(
     site, username, options: QueryOptions, logger, query_notify, *args, **kwargs
 ) -> Tuple[str, QueryResultWrapper]:
-    default_result = make_site_result(site, username, options, logger)
+    default_result = make_site_result(site, username, options, logger, retry=kwargs.get('retry'))
     future = default_result.get("future")
     if not future:
         return site.name, default_result
@@ -705,7 +710,7 @@ async def maigret(
             tasks_dict[sitename] = (
                 check_site_for_username,
                 [site, username, options, logger, query_notify],
-                {'default': (sitename, default_result)},
+                {'default': (sitename, default_result), 'retry': retries-attempts+1},
             )
 
         cur_results = await executor.run(tasks_dict.values())

maigret/resources/data.json

@@ -13809,8 +13809,12 @@
                 "\u041f\u0440\u043e\u0444\u0438\u043b\u044c \u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044f"
             ],
             "alexaRank": 495,
-            "urlMain": "https://rutracker.org/",
-            "url": "https://rutracker.org/forum/profile.php?mode=viewprofile&u={username}",
+            "urlMain": "http://37.1.216.121/",
+            "mirrors": [
+                "https://rutracker.org/",
+                "http://37.1.216.121/"
+            ],
+            "url": "{urlMain}forum/profile.php?mode=viewprofile&u={username}",
             "usernameClaimed": "adam",
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
@@ -16524,10 +16528,16 @@
             "presenseStrs": [
                 "<div class=\"profile-tabs\">"
             ],
+            "mirrors": [
+                "https://nitter.net/",
+                "https://nitter.42l.fr/",
+                "https://nitter.1d4.us/",
+                "https://nitter.kavin.rocks/"
+            ],
             "source": "Twitter",
             "alexaRank": 48,
-            "urlMain": "https://www.nitter.net/",
-            "url": "https://nitter.net/{username}",
+            "urlMain": "https://nitter.kavin.rocks/",
+            "url": "{urlMain}{username}",
             "usernameClaimed": "blue",
             "usernameUnclaimed": "noonewould123"
         },