Cloudflare Bypass call added

maigret/checking.py

@@ -431,6 +431,11 @@ def make_site_result(
 
     # workaround to prevent slash errors
     url = re.sub("(?<!:)/+", "/", url)
+    url_probe = site.url_probe
+
+    if 'cloudflare' in site.tags:
+        url_probe = 'http://localhost:8000/html?url=' + url
+        logger.info(f"Using cloudflare proxy for {site.name}")
 
     # always clearweb_checker for now
     checker = options["checkers"][site.protocol]
@@ -472,7 +477,6 @@ def make_site_result(
     else:
         # URL of user on site (if it exists)
         results_site["url_user"] = url
-        url_probe = site.url_probe
         if url_probe is None:
             # Probe URL is normal one seen by people out on the web.
             url_probe = url

maigret/resources/data.json

@@ -3149,7 +3149,8 @@
         },
         "ChaturBate": {
             "tags": [
-                "us"
+                "us",
+                "cloudflare"
             ],
             "checkType": "status_code",
             "alexaRank": 62,
@@ -3329,7 +3330,8 @@
         "CloudflareCommunity": {
             "tags": [
                 "forum",
-                "tech"
+                "tech",
+                "cloudflare"
             ],
             "engine": "Discourse",
             "alexaRank": 976,
@@ -4203,7 +4205,8 @@
         "Discogs": {
             "tags": [
                 "music",
-                "us"
+                "us",
+                "cloudflare"
             ],
             "checkType": "status_code",
             "alexaRank": 1040,
@@ -4919,7 +4922,8 @@
         "Etsy": {
             "tags": [
                 "shopping",
-                "us"
+                "us",
+                "cloudflare"
             ],
             "errors": {
                 "Sanctions Policy": "Site censorship",
@@ -5037,7 +5041,8 @@
             "usernameUnclaimed": "noonewouldeverusethis7",
             "alexaRank": 240,
             "tags": [
-                "design"
+                "design",
+                "cloudflare"
             ]
         },
         "8tracks.com": {
@@ -6271,7 +6276,8 @@
             "tags": [
                 "art",
                 "photo",
-                "stock"
+                "stock",
+                "cloudflare"
             ],
             "checkType": "status_code",
             "alexaRank": 147,
@@ -8668,7 +8674,8 @@
         "Kickstarter": {
             "tags": [
                 "finance",
-                "us"
+                "us",
+                "cloudflare"
             ],
             "checkType": "status_code",
             "alexaRank": 609,
@@ -12022,7 +12029,8 @@
         },
         "Patreon": {
             "tags": [
-                "finance"
+                "finance",
+                "cloudflare"
             ],
             "checkType": "status_code",
             "alexaRank": 304,
@@ -13515,7 +13523,8 @@
         "Redbubble": {
             "tags": [
                 "shopping",
-                "us"
+                "us",
+                "cloudflare"
             ],
             "checkType": "status_code",
             "alexaRank": 925,
@@ -15059,7 +15068,8 @@
         "SourceForge": {
             "tags": [
                 "coding",
-                "us"
+                "us",
+                "cloudflare"
             ],
             "checkType": "message",
             "alexaRank": 444,
@@ -16911,9 +16921,13 @@
         "Twitch": {
             "tags": [
                 "streaming",
-                "us"
+                "us",
+                "cloudflare"
             ],
-            "urlProbe": "https://twitchtracker.com/{username}",
+            "headers": {
+                "cookies": "_gat=1; _ga_4ZM72D0Y59=GS1.2.1734305902.1.0.1734305902.0.0.0; _ga=GA1.2.1051951095.1734305902; _gid=GA1.2.30506583.1734305902; cf_clearance=xo5uTOkBRsAYb4so4QSu1h8tlFcFJyYSA2SBEHyYA2U-1734305900-1.2.1.1-l9mQ677uPsBenPceAasuW_ZVBqRgQqy4df.13gRl6y4aFBf._3bLo1c3.uVZOXwMxL_iVN.EvEHEBiNczBNMM6riJrVWgiLx1O1jGRbhIiGP.tsomZgyl_bNupNbWxZNzHy454hC0iUigDrE5jkJJoazDRJNc5532wj9nT.U9DDBxW3RplVCdj4x5sMt3K3IXADYvAGabBQnzvS3rEr_w66KClwAehy69tWHVSPDkc.ww7QnxdDItYqmtL8bz9IScdouTAvU_MWK6oxvxcLc6GQFCQZnoToeX8Fgeui2flhV.kXXjEQ4NjypxSFakcCPIysHZOUjKfv93.W9Vfl7id.Y8DUpsmxEPVOpfcrGY6YvFtk6yJhvUQryJftS5b7E5P5jVPW_pPlMWSTWL9IaysG7INm6ZjDyjsVG7OBJIUujSSMlyoKiR8sv0L2ueHt6",
+                "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36"
+            },
             "checkType": "status_code",
             "alexaRank": 34,
             "urlMain": "https://www.twitch.tv/",
@@ -17079,7 +17093,7 @@
         },
         "Udemy": {
             "tags": [
-                "in"
+                "cloudflare"
             ],
             "checkType": "response_url",
             "alexaRank": 131,
@@ -17477,7 +17491,7 @@
                 "method": "vimeo"
             },
             "headers": {
-                "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE3MzQxMTc1NDAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbCwianRpIjoiNDc4Y2ZhZGUtZjI0Yy00MDVkLTliYWItN2RlNGEzNGM4MzI5In0.guN7Fg8dqq7EYdckrJ-6Rdkj_5MOl6FaC4YUSOceDpU"
+                "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE3NDI3Mjg3NDAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbCwianRpIjoiZWI1ZjY1OTAtZGNiMC00MDQ5LTkwODctZmU3ZGFjNjAyMmZmIn0.wrdGR5HPqB_o4f2EHL1h0vVldfmUOvpvPKm-y88X7KU"
             },
             "urlProbe": "https://api.vimeo.com/users/{username}?fields=name%2Cgender%2Cbio%2Curi%2Clink%2Cbackground_video%2Clocation_details%2Cpictures%2Cverified%2Cmetadata.public_videos.total%2Cavailable_for_hire%2Ccan_work_remotely%2Cmetadata.connections.videos.total%2Cmetadata.connections.albums.total%2Cmetadata.connections.followers.total%2Cmetadata.connections.following.total%2Cmetadata.public_videos.total%2Cmetadata.connections.vimeo_experts.is_enrolled%2Ctotal_collection_count%2Ccreated_time%2Cprofile_preferences%2Cmembership%2Cclients%2Cskills%2Cproject_types%2Crates%2Ccategories%2Cis_expert%2Cprofile_discovery%2Cwebsites%2Ccontact_emails&fetch_user_profile=1",
             "checkType": "status_code",
@@ -27244,7 +27258,8 @@
         },
         "upwork.com": {
             "tags": [
-                "us"
+                "us",
+                "cloudflare"
             ],
             "engine": "engine404",
             "urlMain": "https://upwork.com",

sites.md

@@ -3141,7 +3141,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://pubg.op.gg) [OP.GG [PUBG] (https://pubg.op.gg)](https://pubg.op.gg)*: top 100M, gaming*
 1. ![](https://www.google.com/s2/favicons?domain=https://valorant.op.gg) [OP.GG [Valorant] (https://valorant.op.gg)](https://valorant.op.gg)*: top 100M, gaming*
 
-The list was updated at (2024-12-13)
+The list was updated at (2024-12-16)
 ## Statistics
 
 Enabled/total sites: 2684/3137 = 85.56%