From 25026e21ea5829cf6dfb22811cbd7622145f7165 Mon Sep 17 00:00:00 2001 From: Soxoj <31013580+soxoj@users.noreply.github.com> Date: Thu, 23 Apr 2026 21:17:07 +0200 Subject: [PATCH] =?UTF-8?q?Fix=20site=20checks:=204=20=E2=86=92=20ip=5Frep?= =?UTF-8?q?utation,=209=20fixed,=2016=20disabled,=203=20dead=20dele?= =?UTF-8?q?=E2=80=A6=20(#2555)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix site checks: 4 → ip_reputation, 9 fixed, 16 disabled, 3 dead deleted; clarify ip_reputation tag semantics * Improved test coverage --- docs/source/development.rst | 28 ++-- maigret/report.py | 12 +- maigret/resources/data.json | 158 +++++++++++++--------- maigret/resources/db_meta.json | 6 +- sites.md | 63 +++++---- tests/test_activation.py | 107 +++++++++++++++ tests/test_checking.py | 240 +++++++++++++++++++++++++++++++++ tests/test_report.py | 227 +++++++++++++++++++++++++++++++ 8 files changed, 730 insertions(+), 111 deletions(-) diff --git a/docs/source/development.rst b/docs/source/development.rst index 80cd1b5..e54c619 100644 --- a/docs/source/development.rst +++ b/docs/source/development.rst @@ -142,18 +142,28 @@ There are few options for sites data.json helpful in various cases: ``protection`` (site protection tracking) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The ``protection`` field records what kind of anti-bot protection a site uses. Maigret reads this field and automatically applies the appropriate bypass mechanism. +The ``protection`` field records what kind of anti-bot protection a site uses. Maigret reads this field and automatically applies the appropriate bypass mechanism where one exists. + +Two categories of tag: + +- **Load-bearing.** Maigret changes its HTTP client or headers based on the tag. Currently only ``tls_fingerprint`` (switches to ``curl_cffi`` with Chrome-class TLS). +- **Documentation-only.** Maigret does **not** change behavior based on the tag; it records *why* the site is hard so a future solver can target the right set of sites without re-auditing. + +Within the documentation-only tags, there is a further split that dictates whether the site is ``disabled: true``: + +- ``ip_reputation`` is the **only** doc-tag that **keeps the site enabled**. It means "works for most users, fails from datacenter/cloud IPs." Disabling would silently hide a working site from anyone with a clean IP. The fix is **external** to Maigret (residential IP or ``--proxy``). +- ``cf_js_challenge``, ``cf_firewall``, ``aws_waf_js_challenge``, ``ddos_guard_challenge``, ``custom_bot_protection``, ``js_challenge`` all pair with ``disabled: true``. They mean "does not work for anyone right now"; the tag identifies the provider so that when a bypass ships, every site with that tag can be re-enabled in one pass. Supported values: -- ``tls_fingerprint`` — the site fingerprints the TLS handshake (JA3/JA4) and blocks non-browser clients. Maigret automatically uses ``curl_cffi`` with Chrome browser emulation to bypass this. Requires the ``curl_cffi`` package (included as a dependency). Examples: Instagram, NPM, Codepen, Kickstarter, Letterboxd. -- ``ip_reputation`` — the site blocks requests from datacenter/cloud IPs regardless of headers or TLS. Cannot be bypassed automatically; run Maigret from a regular internet connection (not a datacenter) or use a proxy (``--proxy``). Examples: Reddit, Patreon, Figma. -- ``cf_js_challenge`` — Cloudflare Managed Challenge / Turnstile JS challenge. Symptom: HTTP 403 with ``cf-mitigated: challenge`` header; body contains ``challenges.cloudflare.com``, ``_cf_chl_opt``, ``window._cf_chl``, or "Just a moment". Not bypassable via ``curl_cffi`` TLS impersonation (verified across Chrome 123/124/131, Safari 17/18, Firefox 133/135, Edge 101 — all return the same 403 challenge page); a real browser executing the challenge JS is required to obtain the clearance cookie. Documentation-only flag; sites stay ``disabled: true`` until a CF-challenge solver is integrated. Examples: DMOJ, Elakiri, Fanlore, Bdoutdoors, TheStudentRoom, forum.hr. -- ``cf_firewall`` — Cloudflare firewall rule / bot score block (WAF action=block, **not** action=challenge). Symptom: HTTP 403 served by Cloudflare (``server: cloudflare``, ``cf-ray`` header) **without** JS-challenge markers — body typically shows "Access denied", "Attention Required", or just a bare 1015/1016/1020 error page. Unlike ``ip_reputation``, residential IPs are **not** sufficient to bypass — Cloudflare decides based on a composite of bot score, TLS fingerprint, UA, ASN, and custom site-owner rules, so ``curl_cffi`` Chrome impersonation from a residential line still returns 403. Documentation-only flag; sites stay ``disabled: true`` until a per-site bypass (cookies, real browser, or residential+clean session) is found. Examples: Fark, Fodors, Huntingnet, Hunttalk. -- ``aws_waf_js_challenge`` — the site is protected by AWS WAF with a JavaScript challenge. Symptom: HTTP 202 with empty body and ``x-amzn-waf-action: challenge`` header (a token-granting challenge that requires executing the CAPTCHA/challenge JS bundle). Neither ``curl_cffi`` TLS impersonation nor User-Agent changes bypass this — a real browser or the official AWS WAF challenge-solver SDK is required. Currently marked for documentation only; sites using this protection stay ``disabled: true`` until a solver is integrated. Example: Dreamwidth. -- ``ddos_guard_challenge`` — DDoS-Guard (ddos-guard.net) anti-bot page. Symptom: HTTP 403 with ``server: ddos-guard`` header; body contains "DDoS-Guard". DDoS-Guard fingerprints different UAs per source IP, so a single User-Agent override does not work across environments; a JS-capable bypass or DDoS-Guard-aware solver is required. Documentation-only flag; sites stay ``disabled: true`` until a solver is integrated. Example: ForumHouse. -- ``js_challenge`` — **fallback** for JavaScript-challenge systems whose provider cannot be identified (custom in-house challenge pages that are not Cloudflare, AWS WAF, or any other recognized vendor). Prefer a provider-specific tag whenever the provider can be pinned down from response headers or body signatures. -- ``custom_bot_protection`` — **fallback** for non-JS-challenge bot protection served by a custom/in-house system (not Cloudflare, not AWS WAF, not DDoS-Guard). Typical symptom: HTTP 403 from the site's own origin server (``server: nginx``, AWS ELB, etc.) with a branded block page, returned regardless of TLS fingerprint or residential IP. Not generically bypassable; investigate per site (cookies, session, proxy geography). Examples: Hackerearth ("HackerEarth Guardian"), FreelanceJob (nginx-level block). +- ``tls_fingerprint`` *(load-bearing; site stays enabled)* — the site fingerprints the TLS handshake (JA3/JA4) and blocks non-browser clients. Maigret automatically uses ``curl_cffi`` with Chrome browser emulation to bypass this. Requires the ``curl_cffi`` package (included as a dependency). Examples: Instagram, NPM, Codepen, Kickstarter, Letterboxd. +- ``ip_reputation`` *(documentation-only; site stays enabled)* — the site blocks requests from datacenter/cloud IPs regardless of headers or TLS. Cannot be bypassed automatically; run Maigret from a regular internet connection (not a datacenter) or use a proxy (``--proxy``). The site is **not** marked ``disabled`` because it continues to work for users on residential IPs. Examples: Reddit, Patreon, Figma, OnlyFans. +- ``cf_js_challenge`` *(documentation-only; pair with ``disabled: true``)* — Cloudflare Managed Challenge / Turnstile JS challenge. Symptom: HTTP 403 with ``cf-mitigated: challenge`` header; body contains ``challenges.cloudflare.com``, ``_cf_chl_opt``, ``window._cf_chl``, or "Just a moment". Not bypassable via ``curl_cffi`` TLS impersonation (verified across Chrome 123/124/131, Safari 17/18, Firefox 133/135, Edge 101 — all return the same 403 challenge page); a real browser executing the challenge JS is required to obtain the clearance cookie. Sites stay ``disabled: true`` until a CF-challenge solver is integrated. Examples: DMOJ, Elakiri, Fanlore, Bdoutdoors, TheStudentRoom, forum.hr. +- ``cf_firewall`` *(documentation-only; pair with ``disabled: true``)* — Cloudflare firewall rule / bot score block (WAF action=block, **not** action=challenge). Symptom: HTTP 403 served by Cloudflare (``server: cloudflare``, ``cf-ray`` header) **without** JS-challenge markers — body typically shows "Access denied", "Attention Required", or just a bare 1015/1016/1020 error page. Unlike ``ip_reputation``, residential IPs are **not** sufficient to bypass — Cloudflare decides based on a composite of bot score, TLS fingerprint, UA, ASN, and custom site-owner rules, so ``curl_cffi`` Chrome impersonation from a residential line still returns 403. Sites stay ``disabled: true`` until a per-site bypass (cookies, real browser, or residential+clean session) is found. Examples: Fark, Fodors, Huntingnet, Hunttalk. +- ``aws_waf_js_challenge`` *(documentation-only; pair with ``disabled: true``)* — the site is protected by AWS WAF with a JavaScript challenge. Symptom: HTTP 202 with empty body and ``x-amzn-waf-action: challenge`` header (a token-granting challenge that requires executing the CAPTCHA/challenge JS bundle). Neither ``curl_cffi`` TLS impersonation nor User-Agent changes bypass this — a real browser or the official AWS WAF challenge-solver SDK is required. Sites stay ``disabled: true`` until a solver is integrated. Example: Dreamwidth. +- ``ddos_guard_challenge`` *(documentation-only; pair with ``disabled: true``)* — DDoS-Guard (ddos-guard.net) anti-bot page. Symptom: HTTP 403 with ``server: ddos-guard`` header; body contains "DDoS-Guard". DDoS-Guard fingerprints different UAs per source IP, so a single User-Agent override does not work across environments; a JS-capable bypass or DDoS-Guard-aware solver is required. Sites stay ``disabled: true`` until a solver is integrated. Example: ForumHouse. +- ``js_challenge`` *(documentation-only; pair with ``disabled: true``)* — **fallback** for JavaScript-challenge systems whose provider cannot be identified (custom in-house challenge pages that are not Cloudflare, AWS WAF, or any other recognized vendor). Prefer a provider-specific tag whenever the provider can be pinned down from response headers or body signatures. +- ``custom_bot_protection`` *(documentation-only; pair with ``disabled: true``)* — **fallback** for non-JS-challenge bot protection served by a custom/in-house system (not Cloudflare, not AWS WAF, not DDoS-Guard). Typical symptom: HTTP 403 from the site's own origin server (``server: nginx``, AWS ELB, etc.) with a branded block page, returned regardless of TLS fingerprint or residential IP. Not generically bypassable; investigate per site (cookies, session, proxy geography). Examples: Hackerearth ("HackerEarth Guardian"), FreelanceJob (nginx-level block). **Rule: prefer provider-specific protection tags.** When a site is blocked by an identifiable anti-bot vendor, always record the vendor in the tag (``cf_js_challenge``, ``cf_firewall``, ``aws_waf_js_challenge``, ``ddos_guard_challenge``, and future additions such as ``sucuri_challenge``, ``incapsula_challenge``). The generic ``js_challenge`` and ``custom_bot_protection`` tags are reserved for custom/unknown systems. Rationale: bypass solvers are inherently provider-specific (a Cloudflare Turnstile solver does not help with AWS WAF); recording the provider in advance lets us fan out fixes the moment a per-provider solver is added, without re-auditing every disabled site. The same principle applies to other protection categories when the provider is identifiable. diff --git a/maigret/report.py b/maigret/report.py index 4e75be6..1393678 100644 --- a/maigret/report.py +++ b/maigret/report.py @@ -30,14 +30,18 @@ UTILS def filter_supposed_data(data): - # interesting fields allowed_fields = ["fullname", "gender", "location", "age"] - filtered_supposed_data = { - CaseConverter.snake_to_title(k): v[0] + + def _first(v): + if isinstance(v, (list, tuple)): + return v[0] if v else "" + return v + + return { + CaseConverter.snake_to_title(k): _first(v) for k, v in data.items() if k in allowed_fields } - return filtered_supposed_data def sort_report_by_data_points(results): diff --git a/maigret/resources/data.json b/maigret/resources/data.json index 5955e3b..5ef5dc1 100644 --- a/maigret/resources/data.json +++ b/maigret/resources/data.json @@ -491,6 +491,10 @@ "usernameUnclaimed": "noonewouldeverusethis7" }, "Reddit": { + "disabled": true, + "protection": [ + "custom_bot_protection" + ], "tags": [ "discussion", "news", @@ -511,10 +515,7 @@ "url": "https://www.reddit.com/user/{username}", "urlProbe": "https://api.reddit.com/user/{username}/about", "usernameClaimed": "blue", - "usernameUnclaimed": "noonewouldeverusethis7", - "protection": [ - "tls_fingerprint" - ] + "usernameUnclaimed": "noonewouldeverusethis7" }, "Tumblr": { "tags": [ @@ -1613,6 +1614,10 @@ "usernameUnclaimed": "noonewouldeverusethis7" }, "Quora": { + "protection": [ + "cf_js_challenge", + "tls_fingerprint" + ], "tags": [ "education" ], @@ -1779,6 +1784,10 @@ "usernameUnclaimed": "noonewouldeverusethis7" }, "Patreon": { + "disabled": true, + "protection": [ + "cf_js_challenge" + ], "tags": [ "finance" ], @@ -2044,6 +2053,10 @@ "usernameUnclaimed": "noonewouldeverusethis7" }, "Shutterstock": { + "disabled": true, + "protection": [ + "custom_bot_protection" + ], "tags": [ "music", "photo", @@ -2807,6 +2820,10 @@ "usernameUnclaimed": "noonewouldeverusethis7" }, "PyPi": { + "disabled": true, + "protection": [ + "custom_bot_protection" + ], "tags": [ "coding" ], @@ -2818,10 +2835,7 @@ "urlMain": "https://pypi.org/", "url": "https://pypi.org/user/{username}", "usernameClaimed": "adam", - "usernameUnclaimed": "noonewouldeverusethis7", - "protection": [ - "tls_fingerprint" - ] + "usernameUnclaimed": "noonewouldeverusethis7" }, "Pastebin": { "tags": [ @@ -3490,8 +3504,7 @@ "usernameUnclaimed": "noonewouldeverusethis7", "alexaRank": 1426, "absenceStrs": [ - "not found", - "404" + "false | GeeksforGeeks Profile" ], "tags": [ "coding", @@ -3632,6 +3645,10 @@ "disabled": true }, "Redbubble": { + "disabled": true, + "protection": [ + "cf_js_challenge" + ], "tags": [ "shopping" ], @@ -3640,10 +3657,7 @@ "urlMain": "https://www.redbubble.com/", "url": "https://www.redbubble.com/people/{username}", "usernameClaimed": "blue", - "usernameUnclaimed": "noonewouldeverusethis77777", - "protection": [ - "tls_fingerprint" - ] + "usernameUnclaimed": "noonewouldeverusethis77777" }, "codeberg.org": { "tags": [ @@ -5613,6 +5627,9 @@ "alexaRank": 2472 }, "OnlyFans": { + "protection": [ + "ip_reputation" + ], "tags": [ "porn" ], @@ -5622,8 +5639,8 @@ "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", "user-id": "0", "x-bc": "0a106d301866494c873ae3a05bc3c97cee59a749", - "time": "1776790550214", - "sign": "57203:31541b62efa9f19fafc79ca8002b1d0f12335c1d:6d2:69cfa6d8", + "time": "1776959404882", + "sign": "57203:46ddb95bceab303946739ba884f008f6a2118657:646:69cfa6d8", "referer": "https://onlyfans.com/", "cookie": "__cf_bm=YovfzPN0T_wg6F60L5eZKPOQvlGESws3UDGgEkmPb9A-1776790253-1.0.1.1-KRZgptNe5P9epBZSdITa12VfTEDlDdLckPY3I2FDAacvCPxOj0PqeK86J5mcC7UQ_TM8_O24bAh21ElYINovqk2386EoJYyLmknHJ5UsFts" }, @@ -6995,6 +7012,10 @@ ] }, "LibraryThing": { + "protection": [ + "cf_js_challenge", + "tls_fingerprint" + ], "tags": [ "books" ], @@ -7168,6 +7189,10 @@ ] }, "Speedrun.com": { + "protection": [ + "cf_js_challenge", + "tls_fingerprint" + ], "tags": [ "gaming" ], @@ -7922,11 +7947,14 @@ "alexaRank": 6720 }, "Kick": { + "protection": [ + "tls_fingerprint" + ], "url": "https://kick.com/{username}", "urlMain": "https://kick.com/", "urlProbe": "https://kick.com/api/v2/channels/{username}", "checkType": "status_code", - "usernameClaimed": "blue", + "usernameClaimed": "xqc", "usernameUnclaimed": "noonewouldeverusethis7", "alexaRank": 6474, "tags": [ @@ -8368,6 +8396,10 @@ "usernameUnclaimed": "noonewouldeverusethis7" }, "PlanetMinecraft": { + "protection": [ + "cf_js_challenge", + "tls_fingerprint" + ], "tags": [ "gaming" ], @@ -9354,6 +9386,10 @@ "alexaRank": 8514 }, "Rate Your Music": { + "disabled": true, + "protection": [ + "cf_js_challenge" + ], "tags": [ "music" ], @@ -9890,6 +9926,10 @@ "usernameUnclaimed": "noonewouldeverusethis7" }, "JeuxVideo": { + "protection": [ + "cf_js_challenge", + "tls_fingerprint" + ], "tags": [ "fr", "gaming" @@ -9983,7 +10023,8 @@ }, "Anime-planet": { "protection": [ - "tls_fingerprint" + "tls_fingerprint", + "ip_reputation" ], "tags": [ "anime" @@ -10475,17 +10516,6 @@ "usernameClaimed": "blue", "usernameUnclaimed": "noonewouldeverusethis7" }, - "Fotolog": { - "tags": [ - "photo" - ], - "engine": "engine404get", - "urlMain": "http://fotolog.com", - "url": "http://fotolog.com/{username}", - "usernameUnclaimed": "noonewouldeverusethis7", - "usernameClaimed": "red", - "alexaRank": 11693 - }, "PushSquare": { "tags": [ "gaming", @@ -10615,6 +10645,10 @@ "usernameUnclaimed": "noonewouldeverusethis7" }, "Lomography": { + "disabled": true, + "protection": [ + "cf_js_challenge" + ], "absenceStrs": [ "<title>404 · Lomography" ], @@ -10874,6 +10908,10 @@ "usernameUnclaimed": "noonewouldeverusethis7" }, "Liberapay": { + "disabled": true, + "protection": [ + "cf_js_challenge" + ], "tags": [ "finance" ], @@ -11034,6 +11072,7 @@ "usernameUnclaimed": "noonewouldeverusethis7" }, "Joomlart": { + "disabled": true, "tags": [ "coding" ], @@ -11151,7 +11190,8 @@ }, "Flyertalk": { "protection": [ - "tls_fingerprint" + "tls_fingerprint", + "ip_reputation" ], "tags": [ "travel" @@ -11798,6 +11838,10 @@ "alexaRank": 20421 }, "Smule": { + "protection": [ + "cf_js_challenge", + "tls_fingerprint" + ], "tags": [ "music" ], @@ -13285,6 +13329,10 @@ "usernameUnclaimed": "noonewouldeverusethis7" }, "Smogon": { + "disabled": true, + "protection": [ + "custom_bot_protection" + ], "tags": [ "gaming" ], @@ -13336,6 +13384,9 @@ "usernameUnclaimed": "noonewouldeverusethis7" }, "PromptBase": { + "protection": [ + "ip_reputation" + ], "absenceStrs": [ "NotFound" ], @@ -15289,6 +15340,7 @@ "usernameUnclaimed": "noonewouldeverusethis7" }, "Knowem": { + "disabled": true, "tags": [ "business" ], @@ -15558,6 +15610,7 @@ "usernameUnclaimed": "noonewouldeverusethis7" }, "Polywork": { + "disabled": true, "checkType": "message", "absenceStrs": [ ">404", @@ -15700,9 +15753,13 @@ "usernameUnclaimed": "noonewouldeverusethis7" }, "Designspiration": { + "protection": [ + "cf_js_challenge", + "tls_fingerprint" + ], "checkType": "status_code", - "urlMain": "https://www.designspiration.net/", - "url": "https://www.designspiration.net/{username}/", + "urlMain": "https://designspiration.com/", + "url": "https://designspiration.com/{username}/", "usernameClaimed": "blue", "usernameUnclaimed": "noonewouldeverusethis7", "alexaRank": 89022, @@ -17640,6 +17697,7 @@ "usernameUnclaimed": "noonewouldeverusethis7" }, "the-mainboard.com": { + "disabled": true, "tags": [ "forum", "us" @@ -17863,6 +17921,7 @@ "usernameUnclaimed": "noonewouldeverusethis7" }, "Onlyfinder": { + "disabled": true, "absenceStrs": [ "\"rows\":[]" ], @@ -18029,18 +18088,6 @@ ], "alexaRank": 379171 }, - "Pitomec": { - "tags": [ - "ru", - "ua" - ], - "checkType": "status_code", - "alexaRank": 228310, - "urlMain": "https://www.pitomec.ru", - "url": "https://www.pitomec.ru/{username}", - "usernameClaimed": "adam", - "usernameUnclaimed": "noonewouldeverusethis7" - }, "Loveplanet": { "disabled": true, "tags": [ @@ -18717,24 +18764,6 @@ "ua" ] }, - "SQL.ru": { - "tags": [ - "ru" - ], - "checkType": "message", - "presenseStrs": [ - "По вашему запросу найдено" - ], - "absenceStrs": [ - "Извините", - " но по вашему запросу ничего не найдено" - ], - "url": "https://www.sql.ru/forum/actualsearch.aspx?a={username}&ma=0", - "urlMain": "https://www.sql.ru", - "usernameClaimed": "buser", - "usernameUnclaimed": "noonewouldeverusethis7", - "alexaRank": 285351 - }, "Pepper PL": { "url": "https://www.pepper.pl/profile/{username}", "urlMain": "https://www.pepper.pl/", @@ -18828,6 +18857,10 @@ }, "Math10": { "urlSubpath": "/forum", + "disabled": true, + "protection": [ + "cf_js_challenge" + ], "tags": [ "forum", "ru" @@ -19042,6 +19075,7 @@ "usernameUnclaimed": "noonewouldeverusethis7" }, "mcfc-fan.ru": { + "disabled": true, "engine": "uCoz", "urlMain": "http://mcfc-fan.ru", "usernameUnclaimed": "noonewouldeverusethis7", diff --git a/maigret/resources/db_meta.json b/maigret/resources/db_meta.json index 10537a0..b7fdcda 100644 --- a/maigret/resources/db_meta.json +++ b/maigret/resources/db_meta.json @@ -1,8 +1,8 @@ { "version": 1, - "updated_at": "2026-04-22T16:15:02Z", - "sites_count": 3142, + "updated_at": "2026-04-23T17:41:19Z", + "sites_count": 3139, "min_maigret_version": "0.6.0", - "data_sha256": "1e1ed6da2aa9db0f34171f61a044c20bbd1ed53a0430dec4a9ce8f8543655d1a", + "data_sha256": "35bfbb1271a50890c78a03d8e9d9f8d07f78de0e140c8232626de2f6eb124bae", "data_url": "https://raw.githubusercontent.com/soxoj/maigret/main/maigret/resources/data.json" } \ No newline at end of file diff --git a/sites.md b/sites.md index 0b58e9f..75a757e 100644 --- a/sites.md +++ b/sites.md @@ -1,5 +1,5 @@ -## List of supported sites (search methods): total 3142 +## List of supported sites (search methods): total 3139 Rank data fetched from Majestic Million by domains. @@ -22,7 +22,7 @@ Rank data fetched from Majestic Million by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://wordpress.com) [WordPress (https://wordpress.com)](https://wordpress.com)*: top 50, blog* 1. ![](https://www.google.com/s2/favicons?domain=https://plus.google.com) [Google Plus (archived) (https://plus.google.com)](https://plus.google.com)*: top 50, social* 1. ![](https://www.google.com/s2/favicons?domain=https://t.me/) [Telegram (https://t.me/)](https://t.me/)*: top 50, messaging* -1. ![](https://www.google.com/s2/favicons?domain=https://www.reddit.com/) [Reddit (https://www.reddit.com/)](https://www.reddit.com/)*: top 50, discussion, news, social* +1. ![](https://www.google.com/s2/favicons?domain=https://www.reddit.com/) [Reddit (https://www.reddit.com/)](https://www.reddit.com/)*: top 50, discussion, news, social*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://www.tumblr.com) [Tumblr (https://www.tumblr.com)](https://www.tumblr.com)*: top 100, blog, social* 1. ![](https://www.google.com/s2/favicons?domain=https://open.spotify.com/) [Spotify (https://open.spotify.com/)](https://open.spotify.com/)*: top 100, music* 1. ![](https://www.google.com/s2/favicons?domain=https://archive.org) [Archive.org (https://archive.org)](https://archive.org)*: top 100, archive*, search is disabled @@ -101,7 +101,7 @@ Rank data fetched from Majestic Million by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://www.op.gg/) [OP.GG LoL Vietnam (https://www.op.gg/)](https://www.op.gg/)*: top 500, gaming, vn* 1. ![](https://www.google.com/s2/favicons?domain=https://www.op.gg/) [OP.GG LoL Thailand (https://www.op.gg/)](https://www.op.gg/)*: top 500, gaming, th* 1. ![](https://www.google.com/s2/favicons?domain=https://www.xing.com/) [Xing (https://www.xing.com/)](https://www.xing.com/)*: top 500, de, eu* -1. ![](https://www.google.com/s2/favicons?domain=https://www.patreon.com/) [Patreon (https://www.patreon.com/)](https://www.patreon.com/)*: top 500, finance* +1. ![](https://www.google.com/s2/favicons?domain=https://www.patreon.com/) [Patreon (https://www.patreon.com/)](https://www.patreon.com/)*: top 500, finance*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://deviantart.com) [DeviantART (https://deviantart.com)](https://deviantart.com)*: top 500, art, photo* 1. ![](https://www.google.com/s2/favicons?domain=https://www.gofundme.com) [Gofundme (https://www.gofundme.com)](https://www.gofundme.com)*: top 500, finance* 1. ![](https://www.google.com/s2/favicons?domain=https://www.zhihu.com/) [Zhihu (https://www.zhihu.com/)](https://www.zhihu.com/)*: top 500, cn*, search is disabled @@ -117,7 +117,7 @@ Rank data fetched from Majestic Million by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://ok.ru/) [OK (https://ok.ru/)](https://ok.ru/)*: top 1K, ru, social* 1. ![](https://www.google.com/s2/favicons?domain=https://photobucket.com/) [Photobucket (https://photobucket.com/)](https://photobucket.com/)*: top 1K, photo*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://www.udemy.com) [Udemy (https://www.udemy.com)](https://www.udemy.com)*: top 1K, education*, search is disabled -1. ![](https://www.google.com/s2/favicons?domain=https://www.shutterstock.com) [Shutterstock (https://www.shutterstock.com)](https://www.shutterstock.com)*: top 1K, music, photo, stock* +1. ![](https://www.google.com/s2/favicons?domain=https://www.shutterstock.com) [Shutterstock (https://www.shutterstock.com)](https://www.shutterstock.com)*: top 1K, music, photo, stock*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://www.mixcloud.com/) [MixCloud (https://www.mixcloud.com/)](https://www.mixcloud.com/)*: top 1K, music* 1. ![](https://www.google.com/s2/favicons?domain=https://www.npmjs.com/) [NPM (https://www.npmjs.com/)](https://www.npmjs.com/)*: top 1K, coding* 1. ![](https://www.google.com/s2/favicons?domain=https://www.npmjs.com/) [NPM-Package (https://www.npmjs.com/)](https://www.npmjs.com/)*: top 1K, coding* @@ -139,7 +139,7 @@ Rank data fetched from Majestic Million by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://www.gumroad.com/) [Gumroad (https://www.gumroad.com/)](https://www.gumroad.com/)*: top 1K, shopping* 1. ![](https://www.google.com/s2/favicons?domain=https://upwork.com) [Upwork (https://upwork.com)](https://upwork.com)*: top 1K, freelance* 1. ![](https://www.google.com/s2/favicons?domain=https://www.yumpu.com) [Yumpu (https://www.yumpu.com)](https://www.yumpu.com)*: top 1K, stock*, search is disabled -1. ![](https://www.google.com/s2/favicons?domain=https://pypi.org/) [PyPi (https://pypi.org/)](https://pypi.org/)*: top 1K, coding* +1. ![](https://www.google.com/s2/favicons?domain=https://pypi.org/) [PyPi (https://pypi.org/)](https://pypi.org/)*: top 1K, coding*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://www.douban.com) [Douban (https://www.douban.com)](https://www.douban.com)*: top 1K, cn* 1. ![](https://www.google.com/s2/favicons?domain=https://www.lonelyplanet.com) [LonelyPlanet (https://www.lonelyplanet.com)](https://www.lonelyplanet.com)*: top 1K, travel*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://www.figma.com/) [Figma (https://www.figma.com/)](https://www.figma.com/)*: top 1K, design* @@ -183,7 +183,7 @@ Rank data fetched from Majestic Million by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://www.alltrails.com/) [AllTrails (https://www.alltrails.com/)](https://www.alltrails.com/)*: top 5K, sport, travel*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://habr.com/) [Habr (https://habr.com/)](https://habr.com/)*: top 5K, blog, discussion, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://www.allrecipes.com/) [AllRecipes (https://www.allrecipes.com/)](https://www.allrecipes.com/)*: top 5K, hobby* -1. ![](https://www.google.com/s2/favicons?domain=https://www.redbubble.com/) [Redbubble (https://www.redbubble.com/)](https://www.redbubble.com/)*: top 5K, shopping* +1. ![](https://www.google.com/s2/favicons?domain=https://www.redbubble.com/) [Redbubble (https://www.redbubble.com/)](https://www.redbubble.com/)*: top 5K, shopping*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://www.diigo.com/) [Diigo (https://www.diigo.com/)](https://www.diigo.com/)*: top 5K, bookmarks* 1. ![](https://www.google.com/s2/favicons?domain=https://windy.com/) [Windy (https://windy.com/)](https://windy.com/)*: top 5K, maps* 1. ![](https://www.google.com/s2/favicons?domain=https://codecanyon.net) [Codecanyon (https://codecanyon.net)](https://codecanyon.net)*: top 5K, coding, shopping* @@ -360,7 +360,7 @@ Rank data fetched from Majestic Million by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://www.paltalk.com) [Paltalk (https://www.paltalk.com)](https://www.paltalk.com)*: top 10K, messaging* 1. ![](https://www.google.com/s2/favicons?domain=https://www.native-instruments.com/forum/) [NICommunityForum (https://www.native-instruments.com/forum/)](https://www.native-instruments.com/forum/)*: top 10K, forum* 1. ![](https://www.google.com/s2/favicons?domain=https://ccm.net) [Ccm (https://ccm.net)](https://ccm.net)*: top 10K, fr* -1. ![](https://www.google.com/s2/favicons?domain=https://rateyourmusic.com/) [Rate Your Music (https://rateyourmusic.com/)](https://rateyourmusic.com/)*: top 10K, music* +1. ![](https://www.google.com/s2/favicons?domain=https://rateyourmusic.com/) [Rate Your Music (https://rateyourmusic.com/)](https://rateyourmusic.com/)*: top 10K, music*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://videohive.net) [VideoHive (https://videohive.net)](https://videohive.net)*: top 10K, video* 1. ![](https://www.google.com/s2/favicons?domain=http://www.authorstream.com/) [authorSTREAM (http://www.authorstream.com/)](http://www.authorstream.com/)*: top 10K, documents, in, sharing*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://www.airliners.net/) [Airliners (https://www.airliners.net/)](https://www.airliners.net/)*: top 10K, hobby, photo*, search is disabled @@ -407,7 +407,6 @@ Rank data fetched from Majestic Million by domains. 1. ![](https://www.google.com/s2/favicons?domain=http://www.hi5.com) [hi5 (http://www.hi5.com)](http://www.hi5.com)*: top 100K, social*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://diary.ru) [Diary.ru (https://diary.ru)](https://diary.ru)*: top 100K, blog, ru*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://mirtesen.ru) [MirTesen (https://mirtesen.ru)](https://mirtesen.ru)*: top 100K, news, ru*, search is disabled -1. ![](https://www.google.com/s2/favicons?domain=http://fotolog.com) [Fotolog (http://fotolog.com)](http://fotolog.com)*: top 100K, photo* 1. ![](https://www.google.com/s2/favicons?domain=https://www.aufeminin.com) [Aufeminin (https://www.aufeminin.com)](https://www.aufeminin.com)*: top 100K, fr* 1. ![](https://www.google.com/s2/favicons?domain=https://coderwall.com/) [Coderwall (https://coderwall.com/)](https://coderwall.com/)*: top 100K, coding* 1. ![](https://www.google.com/s2/favicons?domain=https://pcpartpicker.com) [PCPartPicker (https://pcpartpicker.com)](https://pcpartpicker.com)*: top 100K, shopping, tech*, search is disabled @@ -417,13 +416,13 @@ Rank data fetched from Majestic Million by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://www.thestudentroom.co.uk) [TheStudentRoom (https://www.thestudentroom.co.uk)](https://www.thestudentroom.co.uk)*: top 100K, forum, gb*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://www.codementor.io/) [Codementor (https://www.codementor.io/)](https://www.codementor.io/)*: top 100K, coding* 1. ![](https://www.google.com/s2/favicons?domain=https://n4g.com/) [N4g (https://n4g.com/)](https://n4g.com/)*: top 100K, gaming, news* -1. ![](https://www.google.com/s2/favicons?domain=https://www.lomography.com) [Lomography (https://www.lomography.com)](https://www.lomography.com)*: top 100K, photo* +1. ![](https://www.google.com/s2/favicons?domain=https://www.lomography.com) [Lomography (https://www.lomography.com)](https://www.lomography.com)*: top 100K, photo*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://pixelfed.social/) [pixelfed.social (https://pixelfed.social/)](https://pixelfed.social/)*: top 100K, art, photo* 1. ![](https://www.google.com/s2/favicons?domain=https://www.hackerearth.com) [Hackerearth (https://www.hackerearth.com)](https://www.hackerearth.com)*: top 100K, freelance*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://weedmaps.com) [Weedmaps (https://weedmaps.com)](https://weedmaps.com)*: top 100K, us* 1. ![](https://www.google.com/s2/favicons?domain=https://www.redtube.com/) [Redtube (https://www.redtube.com/)](https://www.redtube.com/)*: top 100K, porn* 1. ![](https://www.google.com/s2/favicons?domain=https://www.neoseeker.com) [Neoseeker (https://www.neoseeker.com)](https://www.neoseeker.com)*: top 100K, forum, gaming* -1. ![](https://www.google.com/s2/favicons?domain=https://liberapay.com) [Liberapay (https://liberapay.com)](https://liberapay.com)*: top 100K, finance* +1. ![](https://www.google.com/s2/favicons?domain=https://liberapay.com) [Liberapay (https://liberapay.com)](https://liberapay.com)*: top 100K, finance*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://www.sythe.org) [Sythe (https://www.sythe.org)](https://www.sythe.org)*: top 100K, forum* 1. ![](https://www.google.com/s2/favicons?domain=https://www.filmweb.pl/user/adam) [FilmWeb (https://www.filmweb.pl/user/adam)](https://www.filmweb.pl/user/adam)*: top 100K, movies, pl* 1. ![](https://www.google.com/s2/favicons?domain=https://listal.com/) [Listal (https://listal.com/)](https://listal.com/)*: top 100K, movies, music* @@ -438,7 +437,7 @@ Rank data fetched from Majestic Million by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://chaos.social/) [mastodon.social (https://chaos.social/)](https://chaos.social/)*: top 100K, social* 1. ![](https://www.google.com/s2/favicons?domain=https://notabug.org/) [notabug.org (https://notabug.org/)](https://notabug.org/)*: top 100K, coding* 1. ![](https://www.google.com/s2/favicons?domain=https://www.livemaster.ru) [Livemaster (https://www.livemaster.ru)](https://www.livemaster.ru)*: top 100K, ru*, search is disabled -1. ![](https://www.google.com/s2/favicons?domain=https://www.joomlart.com) [Joomlart (https://www.joomlart.com)](https://www.joomlart.com)*: top 100K, coding* +1. ![](https://www.google.com/s2/favicons?domain=https://www.joomlart.com) [Joomlart (https://www.joomlart.com)](https://www.joomlart.com)*: top 100K, coding*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://trinixy.ru) [Trinixy (https://trinixy.ru)](https://trinixy.ru)*: top 100K, news, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://tripit.com) [TripIt (https://tripit.com)](https://tripit.com)*: top 100K, travel*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://mydramalist.com) [Mydramalist (https://mydramalist.com)](https://mydramalist.com)*: top 100K, kr, movies* @@ -578,7 +577,7 @@ Rank data fetched from Majestic Million by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://www.babyblog.ru/) [BabyBlog.ru (https://www.babyblog.ru/)](https://www.babyblog.ru/)*: top 100K, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://www.7cups.com/) [7Cups (https://www.7cups.com/)](https://www.7cups.com/)*: top 100K, medicine*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://ctftime.org/) [CTFtime (https://ctftime.org/)](https://ctftime.org/)*: top 100K, hacking* -1. ![](https://www.google.com/s2/favicons?domain=https://www.smogon.com) [Smogon (https://www.smogon.com)](https://www.smogon.com)*: top 100K, gaming* +1. ![](https://www.google.com/s2/favicons?domain=https://www.smogon.com) [Smogon (https://www.smogon.com)](https://www.smogon.com)*: top 100K, gaming*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://linux.org.ru/) [LOR (https://linux.org.ru/)](https://linux.org.ru/)*: top 100K, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://www.mouthshut.com/) [Mouthshut (https://www.mouthshut.com/)](https://www.mouthshut.com/)*: top 100K, in* 1. ![](https://www.google.com/s2/favicons?domain=https://eva.ru/) [Eva (https://eva.ru/)](https://eva.ru/)*: top 100K, ru*, search is disabled @@ -679,7 +678,7 @@ Rank data fetched from Majestic Million by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://partyflock.nl) [Partyflock (https://partyflock.nl)](https://partyflock.nl)*: top 100K, nl* 1. ![](https://www.google.com/s2/favicons?domain=https://trisquel.info) [Trisquel (https://trisquel.info)](https://trisquel.info)*: top 100K, eu* 1. ![](https://www.google.com/s2/favicons?domain=https://pokemonshowdown.com) [Pokemon Showdown (https://pokemonshowdown.com)](https://pokemonshowdown.com)*: top 100K, gaming* -1. ![](https://www.google.com/s2/favicons?domain=https://knowem.com/) [Knowem (https://knowem.com/)](https://knowem.com/)*: top 100K, business* +1. ![](https://www.google.com/s2/favicons?domain=https://knowem.com/) [Knowem (https://knowem.com/)](https://knowem.com/)*: top 100K, business*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://moikrug.ru/) [MoiKrug (https://moikrug.ru/)](https://moikrug.ru/)*: top 100K, career* 1. ![](https://www.google.com/s2/favicons?domain=https://www.medikforum.ru) [Medikforum (https://www.medikforum.ru)](https://www.medikforum.ru)*: top 100K, de, forum, nl, ru, ua*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://mynickname.com) [mynickname.com (https://mynickname.com)](https://mynickname.com)*: top 100K, social* @@ -694,7 +693,7 @@ Rank data fetched from Majestic Million by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://www.govloop.com) [Govloop (https://www.govloop.com)](https://www.govloop.com)*: top 100K, education* 1. ![](https://www.google.com/s2/favicons?domain=https://wakatime.com) [Wakatime (https://wakatime.com)](https://wakatime.com)*: top 100K, ng, ve* 1. ![](https://www.google.com/s2/favicons?domain=http://www.cqham.ru) [Cqham (http://www.cqham.ru)](http://www.cqham.ru)*: top 100K, ru, tech* -1. ![](https://www.google.com/s2/favicons?domain=https://www.designspiration.net/) [Designspiration (https://www.designspiration.net/)](https://www.designspiration.net/)*: top 100K, design* +1. ![](https://www.google.com/s2/favicons?domain=https://designspiration.com/) [Designspiration (https://designspiration.com/)](https://designspiration.com/)*: top 100K, design* 1. ![](https://www.google.com/s2/favicons?domain=https://www.politforums.net/) [Politforums (https://www.politforums.net/)](https://www.politforums.net/)*: top 100K, forum, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://namemc.com/) [NameMC (https://namemc.com/)](https://namemc.com/)*: top 100K, gaming* 1. ![](https://www.google.com/s2/favicons?domain=https://www.euro-football.ru) [EuroFootball (https://www.euro-football.ru)](https://www.euro-football.ru)*: top 100K, ru* @@ -702,7 +701,7 @@ Rank data fetched from Majestic Million by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://www.truelancer.com) [Truelancer (https://www.truelancer.com)](https://www.truelancer.com)*: top 100K, in* 1. ![](https://www.google.com/s2/favicons?domain=https://www.icheckmovies.com/) [Icheckmovies (https://www.icheckmovies.com/)](https://www.icheckmovies.com/)*: top 100K, movies* 1. ![](https://www.google.com/s2/favicons?domain=https://likee.video) [Likee (https://likee.video)](https://likee.video)*: top 100K, video* -1. ![](https://www.google.com/s2/favicons?domain=https://www.polywork.com) [Polywork (https://www.polywork.com)](https://www.polywork.com)*: top 100K, career* +1. ![](https://www.google.com/s2/favicons?domain=https://www.polywork.com) [Polywork (https://www.polywork.com)](https://www.polywork.com)*: top 100K, career*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://www.forumhouse.ru/) [ForumHouse (https://www.forumhouse.ru/)](https://www.forumhouse.ru/)*: top 100K, forum, ru*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://animesuperhero.com) [AnimeSuperHero (https://animesuperhero.com)](https://animesuperhero.com)*: top 100K, forum*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://www.sports-tracker.com/) [SportsTracker (https://www.sports-tracker.com/)](https://www.sports-tracker.com/)*: top 100K, pt, ru* @@ -805,7 +804,7 @@ Rank data fetched from Majestic Million by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://www.jigidi.com/) [Jigidi (https://www.jigidi.com/)](https://www.jigidi.com/)*: top 10M, hobby* 1. ![](https://www.google.com/s2/favicons?domain=https://allhockey.ru/) [Allhockey (https://allhockey.ru/)](https://allhockey.ru/)*: top 10M, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://www.runitonce.com/) [Runitonce (https://www.runitonce.com/)](https://www.runitonce.com/)*: top 10M, ca* -1. ![](https://www.google.com/s2/favicons?domain=https://onlyfinder.com) [Onlyfinder (https://onlyfinder.com)](https://onlyfinder.com)*: top 10M, webcam* +1. ![](https://www.google.com/s2/favicons?domain=https://onlyfinder.com) [Onlyfinder (https://onlyfinder.com)](https://onlyfinder.com)*: top 10M, webcam*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://postila.ru/) [Postila (https://postila.ru/)](https://postila.ru/)*: top 10M, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://www.chemport.ru) [Chemport (https://www.chemport.ru)](https://www.chemport.ru)*: top 10M, forum, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://vapenews.ru/) [Vapenews (https://vapenews.ru/)](https://vapenews.ru/)*: top 10M, ru* @@ -824,7 +823,7 @@ Rank data fetched from Majestic Million by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://loveplanet.ru) [Loveplanet (https://loveplanet.ru)](https://loveplanet.ru)*: top 10M, dating, ru*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://sevenstring.org) [sevenstring.org (https://sevenstring.org)](https://sevenstring.org)*: top 10M, forum*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://bikepost.ru) [Bikepost (https://bikepost.ru)](https://bikepost.ru)*: top 10M, ru* -1. ![](https://www.google.com/s2/favicons?domain=http://the-mainboard.com/index.php) [the-mainboard.com (http://the-mainboard.com/index.php)](http://the-mainboard.com/index.php)*: top 10M, forum, us* +1. ![](https://www.google.com/s2/favicons?domain=http://the-mainboard.com/index.php) [the-mainboard.com (http://the-mainboard.com/index.php)](http://the-mainboard.com/index.php)*: top 10M, forum, us*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://www.australianfrequentflyer.com.au/community/) [australianfrequentflyer.com.au (https://www.australianfrequentflyer.com.au/community/)](https://www.australianfrequentflyer.com.au/community/)*: top 10M, au, forum* 1. ![](https://www.google.com/s2/favicons?domain=https://4stor.ru) [4stor (https://4stor.ru)](https://4stor.ru)*: top 10M, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://subaruoutback.org) [subaruoutback.org (https://subaruoutback.org)](https://subaruoutback.org)*: top 10M, forum, us* @@ -834,7 +833,6 @@ Rank data fetched from Majestic Million by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://www.snooth.com/) [Snooth (https://www.snooth.com/)](https://www.snooth.com/)*: top 10M, news* 1. ![](https://www.google.com/s2/favicons?domain=https://svtperformance.com) [svtperformance.com (https://svtperformance.com)](https://svtperformance.com)*: top 10M, forum, us* 1. ![](https://www.google.com/s2/favicons?domain=https://www.defensivecarry.com) [DefensiveCarry (https://www.defensivecarry.com)](https://www.defensivecarry.com)*: top 10M, us* -1. ![](https://www.google.com/s2/favicons?domain=https://www.pitomec.ru) [Pitomec (https://www.pitomec.ru)](https://www.pitomec.ru)*: top 10M, ru, ua* 1. ![](https://www.google.com/s2/favicons?domain=https://gotovim-doma.ru) [GotovimDoma (https://gotovim-doma.ru)](https://gotovim-doma.ru)*: top 10M, ru*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://www.chollometro.com/) [Chollometro (https://www.chollometro.com/)](https://www.chollometro.com/)*: top 10M, es, shopping* 1. ![](https://www.google.com/s2/favicons?domain=https://hpc.ru) [Hpc (https://hpc.ru)](https://hpc.ru)*: top 10M, ru* @@ -870,9 +868,8 @@ Rank data fetched from Majestic Million by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://www.affiliatefix.com) [Affiliatefix (https://www.affiliatefix.com)](https://www.affiliatefix.com)*: top 10M, forum* 1. ![](https://www.google.com/s2/favicons?domain=https://shophelp.ru/) [Shophelp (https://shophelp.ru/)](https://shophelp.ru/)*: top 10M, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://www.beermoneyforum.com) [BeerMoneyForum (https://www.beermoneyforum.com)](https://www.beermoneyforum.com)*: top 10M, finance, forum, gambling*, search is disabled -1. ![](https://www.google.com/s2/favicons?domain=https://www.math10.com/) [Math10 (https://www.math10.com/)](https://www.math10.com/)*: top 10M, forum, ru* +1. ![](https://www.google.com/s2/favicons?domain=https://www.math10.com/) [Math10 (https://www.math10.com/)](https://www.math10.com/)*: top 10M, forum, ru*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://www.pepper.pl/) [Pepper PL (https://www.pepper.pl/)](https://www.pepper.pl/)*: top 10M, pl* -1. ![](https://www.google.com/s2/favicons?domain=https://www.sql.ru) [SQL.ru (https://www.sql.ru)](https://www.sql.ru)*: top 10M, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://sigtalk.com) [sigtalk.com (https://sigtalk.com)](https://sigtalk.com)*: top 10M, forum, us* 1. ![](https://www.google.com/s2/favicons?domain=http://mir-stalkera.ru) [mir-stalkera.ru (http://mir-stalkera.ru)](http://mir-stalkera.ru)*: top 10M, gaming, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://pedsovet.su/) [Pedsovet (https://pedsovet.su/)](https://pedsovet.su/)*: top 10M, ru*, search is disabled @@ -890,7 +887,7 @@ Rank data fetched from Majestic Million by domains. 1. ![](https://www.google.com/s2/favicons?domain=http://www.lada-vesta.net) [lada-vesta.net (http://www.lada-vesta.net)](http://www.lada-vesta.net)*: top 10M, auto, forum, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://sysadmins.ru) [Sysadmins (https://sysadmins.ru)](https://sysadmins.ru)*: top 10M, forum, ru, tech* 1. ![](https://www.google.com/s2/favicons?domain=https://plug.dj/) [Plug.DJ (https://plug.dj/)](https://plug.dj/)*: top 10M, music*, search is disabled -1. ![](https://www.google.com/s2/favicons?domain=http://mcfc-fan.ru) [mcfc-fan.ru (http://mcfc-fan.ru)](http://mcfc-fan.ru)*: top 10M, ru, sport* +1. ![](https://www.google.com/s2/favicons?domain=http://mcfc-fan.ru) [mcfc-fan.ru (http://mcfc-fan.ru)](http://mcfc-fan.ru)*: top 10M, ru, sport*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://www.hipforums.com/) [Hipforums (https://www.hipforums.com/)](https://www.hipforums.com/)*: top 10M, forum, ru*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://www.rusfishing.ru) [Rusfishing (https://www.rusfishing.ru)](https://www.rusfishing.ru)*: top 10M, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://jeepgarage.org) [jeepgarage.org (https://jeepgarage.org)](https://jeepgarage.org)*: top 10M, forum, us* @@ -3146,24 +3143,24 @@ Rank data fetched from Majestic Million by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://flarum.es) [flarum.es (https://flarum.es)](https://flarum.es)*: top 100M, es, forum* 1. ![](https://www.google.com/s2/favicons?domain=https://forum.fibra.click) [forum.fibra.click (https://forum.fibra.click)](https://forum.fibra.click)*: top 100M, forum, it* -The list was updated at (2026-04-22) +The list was updated at (2026-04-23) ## Statistics -Enabled/total sites: 2529/3142 = 80.49% +Enabled/total sites: 2510/3139 = 79.96% -Incomplete message checks: 320/2529 = 12.65% (false positive risks) +Incomplete message checks: 317/2510 = 12.63% (false positive risks) -Status code checks: 632/2529 = 24.99% (false positive risks) +Status code checks: 625/2510 = 24.9% (false positive risks) -False positive risk (total): 37.64% +False positive risk (total): 37.53% -Sites with probing: 500px, Armchairgm, BinarySearch (disabled), BleachFandom, Bluesky, BongaCams, Boosty, BuyMeACoffee, Calendly, Cent, Chess, Code Sandbox, Code Snippet Wiki, DailyMotion, Discord, Diskusjon.no, Disqus, Docker Hub, Duolingo, FandomCommunityCentral, GitHub, GitLab, Google Plus (archived), Gravatar, HackTheBox, Hackerrank, Hashnode, Holopin, Imgur, Issuu, Keybase, Kick, Kvinneguiden, LeetCode, Lesswrong, Livejasmin, LocalCryptos (disabled), Medium, MicrosoftLearn, MixCloud, Monkeytype, NPM, Niftygateway, Omg.lol, OnlyFans, Paragraph, Picsart, Plurk, Polarsteps, Rarible, Reddit, Reddit Search (Pushshift) (disabled), Revolut.me, RoyalCams, Scratch, Soop, SportsTracker, Spotify, StackOverflow, Substack, TAP'D, Topcoder, Trello, Twitch, Twitter, Twitter Shadowban (disabled), UnstoppableDomains, Vimeo, Warframe Market, Warpcast, Weibo, Wikipedia, Yapisal (disabled), YouNow, en.brickimedia.org, nightbot, notabug.org, qiwi.me (disabled) +Sites with probing: 500px, Armchairgm, BinarySearch (disabled), BleachFandom, Bluesky, BongaCams, Boosty, BuyMeACoffee, Calendly, Cent, Chess, Code Sandbox, Code Snippet Wiki, DailyMotion, Discord, Diskusjon.no, Disqus, Docker Hub, Duolingo, FandomCommunityCentral, GitHub, GitLab, Google Plus (archived), Gravatar, HackTheBox, Hackerrank, Hashnode, Holopin, Imgur, Issuu, Keybase, Kick, Kvinneguiden, LeetCode, Lesswrong, Livejasmin, LocalCryptos (disabled), Medium, MicrosoftLearn, MixCloud, Monkeytype, NPM, Niftygateway, Omg.lol, OnlyFans, Paragraph, Picsart, Plurk, Polarsteps, Rarible, Reddit (disabled), Reddit Search (Pushshift) (disabled), Revolut.me, RoyalCams, Scratch, Soop, SportsTracker, Spotify, StackOverflow, Substack, TAP'D, Topcoder, Trello, Twitch, Twitter, Twitter Shadowban (disabled), UnstoppableDomains, Vimeo, Warframe Market, Warpcast, Weibo, Wikipedia, Yapisal (disabled), YouNow, en.brickimedia.org, nightbot, notabug.org, qiwi.me (disabled) Sites with activation: OnlyFans, Twitter, Vimeo, Weibo Top 20 profile URLs: - (709) `{urlMain}/index/8-0-{username} (uCoz)` -- (314) `/{username}` +- (312) `/{username}` - (223) `{urlMain}{urlSubpath}/members/?username={username} (XenForo)` - (170) `/user/{username}` - (138) `/profile/{username}` @@ -3185,19 +3182,19 @@ Top 20 profile URLs: Sites by engine: -- `uCoz`: 635/709 (89.6%) -- `XenForo`: 182/223 (81.6%) +- `uCoz`: 634/709 (89.4%) +- `XenForo`: 181/223 (81.2%) - `phpBB/Search`: 120/127 (94.5%) - `vBulletin`: 31/120 (25.8%) - `Discourse`: 81/87 (93.1%) -- `phpBB`: 22/27 (81.5%) +- `phpBB`: 21/27 (77.8%) - `engine404`: 19/23 (82.6%) - `op.gg`: 17/17 (100.0%) - `Flarum`: 15/15 (100.0%) - `Wordpress/Author`: 7/9 (77.8%) - `engineRedirect`: 3/4 (75.0%) -- `engine404get`: 3/3 (100.0%) - `phpBB2/Search`: 2/3 (66.7%) +- `engine404get`: 2/2 (100.0%) Top 20 tags: @@ -3205,7 +3202,7 @@ Top 20 tags: - (750) `forum` - (128) `gaming` - (80) `coding` -- (58) `photo` +- (57) `photo` - (46) `tech` - (45) `social` - (41) `news` diff --git a/tests/test_activation.py b/tests/test_activation.py index 46ed332..76e60f4 100644 --- a/tests/test_activation.py +++ b/tests/test_activation.py @@ -56,3 +56,110 @@ async def test_import_aiohttp_cookies(cookie_test_server): print(f"Server response: {result}") assert result == {'cookies': {'a': 'b'}} + + +# ---- OnlyFans signing tests (pure-compute, no network) ---- + +class _FakeSite: + """Minimal stand-in for MaigretSite with the attributes onlyfans() touches.""" + + def __init__(self, headers=None, activation=None): + self.headers = headers or {} + self.activation = activation or { + "static_param": "jLM8LXHU1CGcuCzPMNwWX9osCScVuP4D", + "checksum_indexes": [28, 3, 16, 32, 25, 24, 23, 0, 26], + "checksum_constant": -180, + "format": "57203:{}:{:x}:69cfa6d8", + "url": "https://onlyfans.com/api2/v2/init", + } + + +class _FakeResponse: + def __init__(self, cookies=None): + self.cookies = cookies or {} + + +def test_onlyfans_sets_xbc_when_zero(monkeypatch): + site = _FakeSite(headers={"x-bc": "0", "cookie": "existing=1"}) + + # Prevent any real network. If _sign path still fires requests.get, fail loudly. + import maigret.activation as act_mod + + def boom(*a, **kw): # pragma: no cover - sanity + raise AssertionError("requests.get should not run when cookie is present") + + monkeypatch.setattr(act_mod.__dict__.get("requests", None) or __import__("requests"), "get", boom, raising=False) + + logger = Mock() + ParsingActivator.onlyfans(site, logger, url="https://onlyfans.com/api2/v2/users/adam") + + # x-bc must be rewritten to a non-zero hex token + assert site.headers["x-bc"] != "0" + assert len(site.headers["x-bc"]) == 40 # 20 bytes → 40 hex chars + # time / sign headers set for target URL + assert "time" in site.headers and site.headers["time"].isdigit() + assert site.headers["sign"].startswith("57203:") + + +def test_onlyfans_fetches_init_cookie_when_missing(monkeypatch): + """When cookie header is absent, init endpoint is called and its cookies stored.""" + site = _FakeSite(headers={"x-bc": "already_set_token", "user-id": "0"}) + + import requests + + captured = {} + + def fake_get(url, headers=None, timeout=15): + captured["url"] = url + captured["headers"] = dict(headers or {}) + return _FakeResponse(cookies={"sess": "abc123", "csrf": "xyz"}) + + monkeypatch.setattr(requests, "get", fake_get) + + logger = Mock() + ParsingActivator.onlyfans(site, logger, url="https://onlyfans.com/api2/v2/users/adam") + + # init request made + assert captured["url"] == site.activation["url"] + # headers passed to init include freshly generated time/sign + assert "time" in captured["headers"] + assert captured["headers"]["sign"].startswith("57203:") + # cookie header populated from response + assert site.headers["cookie"] == "sess=abc123; csrf=xyz" + + +def test_onlyfans_signature_is_deterministic_for_same_time(monkeypatch): + """Two calls with patched time produce identical signatures.""" + site1 = _FakeSite(headers={"x-bc": "token", "cookie": "c=1"}) + site2 = _FakeSite(headers={"x-bc": "token", "cookie": "c=1"}) + + import maigret.activation + monkeypatch.setattr(maigret.activation, "_time", __import__("time"), raising=False) + + fixed = 1_700_000_000.123 + import time as time_mod + monkeypatch.setattr(time_mod, "time", lambda: fixed) + + logger = Mock() + ParsingActivator.onlyfans(site1, logger, url="https://onlyfans.com/api2/v2/users/adam") + ParsingActivator.onlyfans(site2, logger, url="https://onlyfans.com/api2/v2/users/adam") + + assert site1.headers["time"] == site2.headers["time"] + assert site1.headers["sign"] == site2.headers["sign"] + + +def test_onlyfans_sign_differs_per_path(monkeypatch): + """Different target URLs must yield different signatures.""" + site = _FakeSite(headers={"x-bc": "token", "cookie": "c=1"}) + + import time as time_mod + monkeypatch.setattr(time_mod, "time", lambda: 1_700_000_000.0) + + logger = Mock() + ParsingActivator.onlyfans(site, logger, url="https://onlyfans.com/api2/v2/users/adam") + sig_adam = site.headers["sign"] + + ParsingActivator.onlyfans(site, logger, url="https://onlyfans.com/api2/v2/users/bob") + sig_bob = site.headers["sign"] + + assert sig_adam != sig_bob diff --git a/tests/test_checking.py b/tests/test_checking.py index 98a354f..25ac2f0 100644 --- a/tests/test_checking.py +++ b/tests/test_checking.py @@ -1,7 +1,22 @@ +from argparse import ArgumentTypeError + from mock import Mock import pytest from maigret import search +from maigret.checking import ( + detect_error_page, + extract_ids_data, + parse_usernames, + update_results_info, + get_failed_sites, + timeout_check, + debug_response_logging, + process_site_result, +) +from maigret.errors import CheckError +from maigret.result import MaigretCheckResult, MaigretCheckStatus +from maigret.sites import MaigretSite def site_result_except(server, username, **kwargs): @@ -67,3 +82,228 @@ async def test_checking_by_message_negative(httpserver, local_test_db): result = await search('unclaimed', site_dict=sites_dict, logger=Mock()) assert result['Message']['status'].is_found() is True + + +# ---- Pure-function unit tests (no network) ---- + + +def test_detect_error_page_site_specific(): + err = detect_error_page( + "Please enable JavaScript to proceed", + 200, + {"Please enable JavaScript to proceed": "Scraping protection"}, + ignore_403=False, + ) + assert err is not None + assert err.type == "Site-specific" + assert err.desc == "Scraping protection" + + +def test_detect_error_page_403(): + err = detect_error_page("some body", 403, {}, ignore_403=False) + assert err is not None + assert err.type == "Access denied" + + +def test_detect_error_page_403_ignored(): + # XenForo engine uses ignore403 because member-not-found also returns 403 + assert detect_error_page("not found body", 403, {}, ignore_403=True) is None + + +def test_detect_error_page_999_linkedin(): + # LinkedIn returns 999 on bot suspicion — must NOT be reported as Server error + assert detect_error_page("", 999, {}, ignore_403=False) is None + + +def test_detect_error_page_500(): + err = detect_error_page("", 503, {}, ignore_403=False) + assert err is not None + assert err.type == "Server" + assert "503" in err.desc + + +def test_detect_error_page_ok(): + assert detect_error_page("hello world", 200, {}, ignore_403=False) is None + + +def test_parse_usernames_single_username(): + logger = Mock() + result = parse_usernames({"profile_username": "alice"}, logger) + assert result == {"alice": "username"} + + +def test_parse_usernames_list_of_usernames(): + logger = Mock() + result = parse_usernames({"other_usernames": "['alice', 'bob']"}, logger) + assert result == {"alice": "username", "bob": "username"} + + +def test_parse_usernames_malformed_list(): + logger = Mock() + result = parse_usernames({"other_usernames": "not-a-list"}, logger) + # should swallow the error and just return empty + assert result == {} + assert logger.warning.called + + +def test_parse_usernames_supported_id(): + logger = Mock() + # "telegram" is in SUPPORTED_IDS per socid_extractor + from maigret.checking import SUPPORTED_IDS + if SUPPORTED_IDS: + key = next(iter(SUPPORTED_IDS)) + result = parse_usernames({key: "some_value"}, logger) + assert result.get("some_value") == key + + +def test_update_results_info_links(): + info = {"username": "test"} + result = update_results_info( + info, + {"links": "['https://example.com/a', 'https://example.com/b']", "website": "https://example.com/w"}, + {"alice": "username"}, + ) + assert result["ids_usernames"] == {"alice": "username"} + assert "https://example.com/w" in result["ids_links"] + assert "https://example.com/a" in result["ids_links"] + + +def test_update_results_info_no_website(): + info = {} + result = update_results_info(info, {"links": "[]"}, {}) + assert result["ids_links"] == [] + + +def test_extract_ids_data_bad_html_returns_empty(): + logger = Mock() + # Random HTML should not raise — returns {} if nothing matches + out = extract_ids_data("nothing special", logger, Mock(name="Site")) + assert isinstance(out, dict) + + +def test_get_failed_sites_filters_permanent_errors(): + # Temporary errors (Request timeout, Connecting failure, etc.) are retryable → returned. + # Permanent ones (Captcha, Access denied, etc.) and results without error → filtered out. + good_status = MaigretCheckResult("u", "S1", "https://s1", MaigretCheckStatus.CLAIMED) + timeout_err = MaigretCheckResult( + "u", "S2", "https://s2", MaigretCheckStatus.UNKNOWN, + error=CheckError("Request timeout", "slow server"), + ) + captcha_err = MaigretCheckResult( + "u", "S3", "https://s3", MaigretCheckStatus.UNKNOWN, + error=CheckError("Captcha", "Cloudflare"), + ) + results = { + "S1": {"status": good_status}, + "S2": {"status": timeout_err}, + "S3": {"status": captcha_err}, + "S4": {}, # no status at all + } + failed = get_failed_sites(results) + # Only the temporary-error site is retry-worthy + assert failed == ["S2"] + + +def test_timeout_check_valid(): + assert timeout_check("2.5") == 2.5 + assert timeout_check("30") == 30.0 + + +def test_timeout_check_invalid(): + with pytest.raises(ArgumentTypeError): + timeout_check("abc") + with pytest.raises(ArgumentTypeError): + timeout_check("0") + with pytest.raises(ArgumentTypeError): + timeout_check("-1") + + +def test_debug_response_logging_writes(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + debug_response_logging("https://example.com", "hi", 200, None) + out = (tmp_path / "debug.log").read_text() + assert "https://example.com" in out + assert "200" in out + + +def test_debug_response_logging_no_response(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + debug_response_logging("https://example.com", None, None, CheckError("Timeout")) + out = (tmp_path / "debug.log").read_text() + assert "No response" in out + + +def _make_site(data_overrides=None): + base = { + "url": "https://x/{username}", + "urlMain": "https://x", + "checkType": "status_code", + "usernameClaimed": "a", + "usernameUnclaimed": "b", + } + if data_overrides: + base.update(data_overrides) + return MaigretSite("TestSite", base) + + +def test_process_site_result_no_response_returns_info(): + site = _make_site() + info = {"username": "a", "parsing_enabled": False, "url_user": "https://x/a"} + out = process_site_result(None, Mock(), Mock(), info, site) + assert out is info + + +def test_process_site_result_status_already_set(): + site = _make_site() + pre = MaigretCheckResult("a", "S", "u", MaigretCheckStatus.ILLEGAL) + info = {"username": "a", "parsing_enabled": False, "status": pre, "url_user": "u"} + # Since status is already set, function returns without changes + out = process_site_result(("", 200, None), Mock(), Mock(), info, site) + assert out["status"] is pre + + +def test_process_site_result_status_code_claimed(): + site = _make_site({"checkType": "status_code"}) + info = {"username": "a", "parsing_enabled": False, "url_user": "https://x/a"} + out = process_site_result(("", 200, None), Mock(), Mock(), info, site) + assert out["status"].status == MaigretCheckStatus.CLAIMED + assert out["http_status"] == 200 + + +def test_process_site_result_status_code_available(): + site = _make_site({"checkType": "status_code"}) + info = {"username": "a", "parsing_enabled": False, "url_user": "https://x/a"} + out = process_site_result(("", 404, None), Mock(), Mock(), info, site) + assert out["status"].status == MaigretCheckStatus.AVAILABLE + + +def test_process_site_result_message_claimed(): + site = _make_site({ + "checkType": "message", + "presenseStrs": ["profile-name"], + "absenceStrs": ["not found"], + }) + info = {"username": "a", "parsing_enabled": False, "url_user": "https://x/a"} + out = process_site_result(("
Alice
", 200, None), Mock(), Mock(), info, site) + assert out["status"].status == MaigretCheckStatus.CLAIMED + + +def test_process_site_result_message_available_by_absence(): + site = _make_site({ + "checkType": "message", + "presenseStrs": ["profile-name"], + "absenceStrs": ["not found"], + }) + info = {"username": "a", "parsing_enabled": False, "url_user": "https://x/a"} + out = process_site_result(("

not found

profile-name too", 200, None), Mock(), Mock(), info, site) + # absence marker wins even if presence marker also appears + assert out["status"].status == MaigretCheckStatus.AVAILABLE + + +def test_process_site_result_with_error_is_unknown(): + site = _make_site({"checkType": "status_code"}) + info = {"username": "a", "parsing_enabled": False, "url_user": "https://x/a"} + resp = ("body", 403, CheckError("Captcha", "Cloudflare")) + out = process_site_result(resp, Mock(), Mock(), info, site) + assert out["status"].status == MaigretCheckStatus.UNKNOWN + assert out["status"].error is not None diff --git a/tests/test_report.py b/tests/test_report.py index e5fb88b..8e12408 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -10,8 +10,15 @@ import xmind # type: ignore[import-untyped] from jinja2 import Template from maigret.report import ( + filter_supposed_data, + sort_report_by_data_points, + _md_format_value, generate_csv_report, generate_txt_report, + save_csv_report, + save_txt_report, + save_json_report, + save_markdown_report, save_xmind_report, save_html_report, save_pdf_report, @@ -456,3 +463,223 @@ def test_text_report_broken(): assert brief_part in report_text assert 'us' in report_text assert 'photo' in report_text + + +def test_filter_supposed_data(): + data = { + 'fullname': ['Alice'], + 'gender': ['female'], + 'location': ['Berlin'], + 'age': ['30'], + 'email': ['x@y.z'], # not allowed, must be dropped + 'bio': ['hi'], # not allowed + } + result = filter_supposed_data(data) + assert result == { + 'Fullname': 'Alice', + 'Gender': 'female', + 'Location': 'Berlin', + 'Age': '30', + } + + +def test_filter_supposed_data_empty(): + assert filter_supposed_data({}) == {} + assert filter_supposed_data({'nope': ['v']}) == {} + + +def test_filter_supposed_data_scalar_values(): + # Strings and scalars must be kept whole — previously v[0] on "Alice" + # silently returned "A" instead of "Alice". + data = { + 'fullname': 'Alice', + 'gender': 'female', + 'location': 'Berlin', + 'age': 30, + } + assert filter_supposed_data(data) == { + 'Fullname': 'Alice', + 'Gender': 'female', + 'Location': 'Berlin', + 'Age': 30, + } + + +def test_filter_supposed_data_empty_list_yields_empty_string(): + # Edge case: list value present but empty should not crash with IndexError. + assert filter_supposed_data({'fullname': []}) == {'Fullname': ''} + + +def test_filter_supposed_data_mixed_values(): + # List and scalar mixed in the same payload. + data = {'fullname': ['Alice', 'Alicia'], 'gender': 'female'} + assert filter_supposed_data(data) == { + 'Fullname': 'Alice', + 'Gender': 'female', + } + + +def test_sort_report_by_data_points(): + status_many = MaigretCheckResult('', '', '', MaigretCheckStatus.CLAIMED) + status_many.ids_data = {'a': 1, 'b': 2, 'c': 3} + status_one = MaigretCheckResult('', '', '', MaigretCheckStatus.CLAIMED) + status_one.ids_data = {'a': 1} + status_none = MaigretCheckResult('', '', '', MaigretCheckStatus.CLAIMED) + + results = { + 'few': {'status': status_one}, + 'many': {'status': status_many}, + 'zero': {'status': status_none}, + 'nostatus': {}, + } + sorted_out = sort_report_by_data_points(results) + keys = list(sorted_out.keys()) + # site with 3 ids_data fields must come first + assert keys[0] == 'many' + # site with 1 field next + assert keys[1] == 'few' + + +def test_md_format_value_list(): + assert _md_format_value(['a', 'b', 'c']) == 'a, b, c' + + +def test_md_format_value_url(): + assert _md_format_value('https://example.com') == '[https://example.com](https://example.com)' + assert _md_format_value('http://x.y') == '[http://x.y](http://x.y)' + + +def test_md_format_value_plain(): + assert _md_format_value('hello') == 'hello' + assert _md_format_value(42) == '42' + + +def test_save_csv_report(): + filename = 'report_test.csv' + save_csv_report(filename, 'test', EXAMPLE_RESULTS) + with open(filename) as f: + content = f.read() + assert 'username,name,url_main' in content + assert 'test,GitHub' in content + + +def test_save_txt_report(): + filename = 'report_test.txt' + save_txt_report(filename, 'test', EXAMPLE_RESULTS) + with open(filename) as f: + content = f.read() + assert 'https://www.github.com/test' in content + assert 'Total Websites Username Detected On : 1' in content + + +def test_save_json_report_simple(): + filename = 'report_test.json' + save_json_report(filename, 'test', EXAMPLE_RESULTS, 'simple') + with open(filename) as f: + data = json.load(f) + assert 'GitHub' in data + + +def test_save_json_report_ndjson(): + filename = 'report_test_ndjson.json' + save_json_report(filename, 'test', EXAMPLE_RESULTS, 'ndjson') + with open(filename) as f: + lines = f.readlines() + assert len(lines) == 1 + assert json.loads(lines[0])['sitename'] == 'GitHub' + + +def _markdown_context_with_rich_ids(): + """Build a context with found accounts, ids_data (incl. image, url, list) to exercise all branches.""" + found_result = copy.deepcopy(GOOD_RESULT) + found_result.tags = ['photo', 'us'] + found_result.ids_data = { + "fullname": "Alice", + "name": "Alice A.", + "location": "Berlin", + "bio": "Photographer", + "external_url": "https://example.com/profile", + "image": "https://example.com/avatar.png", # must be skipped + "aliases": ["alice", "alicea"], # list value + "last_online": "2024-01-02 10:00:00", + } + data = { + 'Github': { + 'username': 'alice', + 'parsing_enabled': True, + 'url_main': 'https://github.com/', + 'url_user': 'https://github.com/alice', + 'status': found_result, + 'http_status': 200, + 'is_similar': False, + 'rank': 1, + 'site': MaigretSite('Github', {}), + 'found': True, + 'ids_data': found_result.ids_data, + }, + 'Similar': { + 'username': 'alice', + 'url_user': 'https://other.com/alice', + 'is_similar': True, + 'found': True, + 'status': copy.deepcopy(GOOD_RESULT), + }, + } + return { + 'username': 'alice', + 'generated_at': '2024-01-02 10:00', + 'brief': 'Search returned 1 account', + 'countries_tuple_list': [('us', 1)], + 'interests_tuple_list': [('photo', 1)], + 'first_seen': '2023-01-01', + 'results': [('alice', 'username', data)], + } + + +def test_save_markdown_report(): + filename = 'report_test.md' + context = _markdown_context_with_rich_ids() + save_markdown_report(filename, context, run_info={'sites_count': 100, 'flags': '--top-sites 100'}) + with open(filename) as f: + content = f.read() + assert '# Report by searching on username "alice"' in content + assert '## Summary' in content + assert '## Accounts found' in content + assert '### Github' in content + assert '[https://github.com/alice](https://github.com/alice)' in content + assert 'Ethical use' in content + assert '100 sites checked' in content + # image field must NOT appear in per-site listing + assert 'avatar.png' not in content + # list field rendered with join + assert 'alice, alicea' in content + # external url formatted as markdown link + assert '[https://example.com/profile](https://example.com/profile)' in content + + +def test_save_markdown_report_minimal_context(): + """No run_info, no first_seen — exercise the fallback branches.""" + filename = 'report_test_min.md' + context = { + 'username': 'bob', + 'brief': 'nothing found', + 'results': [], + } + save_markdown_report(filename, context) + with open(filename) as f: + content = f.read() + assert '# Report by searching on username "bob"' in content + assert '## Summary' in content + + +def test_get_plaintext_report_minimal(): + """Minimal context without countries/interests.""" + context = { + 'brief': 'Nothing to report.', + 'interests_tuple_list': [], + 'countries_tuple_list': [], + } + out = get_plaintext_report(context) + assert 'Nothing to report.' in out + assert 'Countries:' not in out + assert 'Interests' not in out