Improve site-check quality: fix broken site configs, add diagnostic utilities, and make self-check report-only by default with opt-in auto-disable. (#2301)

- Fix VK and TradingView checkType; add Reddit and Microsoft Learn API-style probes where appropriate; adjust or disable entries that are unreliable under anti-bot protection.
- Self-check: stop aggressive auto-disable; default to reporting issues only; add --auto-disable and --diagnose for optional fixes and deeper output.
- Tooling: add utils/site_check.py and utils/check_top_n.py (and related helpers) to inspect and rank site behavior against the top-N list
- Scope: aligns with fixing top-traffic / high-impact sites and making diagnostics repeatable without silently flipping disabled flags

tests/test_cli.py

@@ -5,11 +5,13 @@ from typing import Dict, Any
 
 DEFAULT_ARGS: Dict[str, Any] = {
     'all_sites': False,
+    'auto_disable': False,
     'connections': 100,
     'cookie_file': None,
     'csv': False,
     'db_file': 'resources/data.json',
     'debug': False,
+    'diagnose': False,
     'disable_extracting': False,
     'disable_recursive_search': False,
     'folderoutput': 'reports',

tests/test_maigret.py

@@ -27,7 +27,9 @@ async def test_self_check_db(test_db):
     assert test_db.sites_dict['ValidActive'].disabled is False
     assert test_db.sites_dict['InvalidInactive'].disabled is True
 
-    await self_check(test_db, test_db.sites_dict, logger, silent=False)
+    await self_check(
+        test_db, test_db.sites_dict, logger, silent=False, auto_disable=True
+    )
 
     assert test_db.sites_dict['InvalidActive'].disabled is True
     assert test_db.sites_dict['ValidInactive'].disabled is False

tests/test_twitter.py

@@ -0,0 +1,63 @@
+"""Tests for the Twitter / X site entry and GraphQL probe."""
+
+import re
+
+import pytest
+import requests
+
+from maigret.sites import MaigretSite
+
+
+def _twitter_site(site: MaigretSite) -> None:
+    assert site.name == "Twitter"
+    assert site.disabled is False
+    assert site.check_type == "message"
+    assert site.url_probe and "{username}" in site.url_probe
+    assert "UserByScreenName" in site.url_probe or "graphql" in site.url_probe
+    assert site.regex_check
+    assert re.fullmatch(site.regex_check, site.username_claimed)
+    assert re.fullmatch(site.regex_check, site.username_unclaimed)
+    assert site.absence_strs
+    assert site.activation.get("method") == "twitter"
+    assert site.activation.get("url")
+    assert "authorization" in {k.lower() for k in site.headers.keys()}
+
+
+def test_twitter_site_entry_config(default_db):
+    """Twitter entry in data.json must define probe URL, regex, and activation."""
+    site = default_db.sites_dict["Twitter"]
+    assert isinstance(site, MaigretSite)
+    _twitter_site(site)
+
+
+@pytest.mark.slow
+def test_twitter_graphql_probe_claimed_vs_unclaimed(default_db):
+    """
+    Live check: guest activation + UserByScreenName GraphQL returns a user for
+    usernameClaimed and no user for usernameUnclaimed (same flow as urlProbe).
+    """
+    site = default_db.sites_dict["Twitter"]
+    _twitter_site(site)
+
+    headers = dict(site.headers)
+    headers.pop("x-guest-token", None)
+
+    act = requests.post(site.activation["url"], headers=headers, timeout=45)
+    assert act.status_code == 200, act.text[:500]
+    body = act.json()
+    assert "guest_token" in body
+    headers["x-guest-token"] = body["guest_token"]
+
+    def fetch(username: str) -> dict:
+        url = site.url_probe.format(username=username)
+        resp = requests.get(url, headers=headers, timeout=45)
+        resp.raise_for_status()
+        return resp.json()
+
+    claimed_json = fetch(site.username_claimed)
+    assert "data" in claimed_json
+    assert claimed_json["data"].get("user") is not None
+
+    unclaimed_json = fetch(site.username_unclaimed)
+    data = unclaimed_json.get("data") or {}
+    assert data == {} or data.get("user") is None