mirror/maigret
1
0
Fork 0
mirror of https://github.com/soxoj/maigret.git synced 2026-05-07 14:34:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Cloudflare Bypass call added

Browse Source
This commit is contained in:
Soxoj
2025-03-23 12:08:37 +01:00
parent 5381a134f0
commit a163278f89
3 changed files with 36 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
maigret/checking.py
+5 -1
View File
@@ -431,6 +431,11 @@ def make_site_result(
# workaround to prevent slash errors
url = re.sub("(?<!:)/+", "/", url)
url_probe = site.url_probe
if 'cloudflare' in site.tags:
url_probe = 'http://localhost:8000/html?url=' + url
logger.info(f"Using cloudflare proxy for {site.name}")
# always clearweb_checker for now
checker = options["checkers"][site.protocol]
@@ -472,7 +477,6 @@ def make_site_result(
else:
# URL of user on site (if it exists)
results_site["url_user"] = url
url_probe = site.url_probe
if url_probe is None:
# Probe URL is normal one seen by people out on the web.
url_probe = url
maigret/resources/data.json
+30 -15
View File
@@ -3149,7 +3149,8 @@
},
"ChaturBate": {
"tags": [
"us"
"us",
"cloudflare"
],
"checkType": "status_code",
"alexaRank": 62,
@@ -3329,7 +3330,8 @@
"CloudflareCommunity": {
"tags": [
"forum",
"tech"
"tech",
"cloudflare"
],
"engine": "Discourse",
"alexaRank": 976,
@@ -4203,7 +4205,8 @@
"Discogs": {
"tags": [
"music",
"us"
"us",
"cloudflare"
],
"checkType": "status_code",
"alexaRank": 1040,
@@ -4919,7 +4922,8 @@
"Etsy": {
"tags": [
"shopping",
"us"
"us",
"cloudflare"
],
"errors": {
"Sanctions Policy": "Site censorship",
@@ -5037,7 +5041,8 @@
"usernameUnclaimed": "noonewouldeverusethis7",
"alexaRank": 240,
"tags": [
"design"
"design",
"cloudflare"
]
},
"8tracks.com": {
@@ -6271,7 +6276,8 @@
"tags": [
"art",
"photo",
"stock"
"stock",
"cloudflare"
],
"checkType": "status_code",
"alexaRank": 147,
@@ -8668,7 +8674,8 @@
"Kickstarter": {
"tags": [
"finance",
"us"
"us",
"cloudflare"
],
"checkType": "status_code",
"alexaRank": 609,
@@ -12022,7 +12029,8 @@
},
"Patreon": {
"tags": [
"finance"
"finance",
"cloudflare"
],
"checkType": "status_code",
"alexaRank": 304,
@@ -13515,7 +13523,8 @@
"Redbubble": {
"tags": [
"shopping",
"us"
"us",
"cloudflare"
],
"checkType": "status_code",
"alexaRank": 925,
@@ -15059,7 +15068,8 @@
"SourceForge": {
"tags": [
"coding",
"us"
"us",
"cloudflare"
],
"checkType": "message",
"alexaRank": 444,
@@ -16911,9 +16921,13 @@
"Twitch": {
"tags": [
"streaming",
"us"
"us",
"cloudflare"
],
"urlProbe": "https://twitchtracker.com/{username}",
"headers": {
"cookies": "_gat=1; _ga_4ZM72D0Y59=GS1.2.1734305902.1.0.1734305902.0.0.0; _ga=GA1.2.1051951095.1734305902; _gid=GA1.2.30506583.1734305902; cf_clearance=xo5uTOkBRsAYb4so4QSu1h8tlFcFJyYSA2SBEHyYA2U-1734305900-1.2.1.1-l9mQ677uPsBenPceAasuW_ZVBqRgQqy4df.13gRl6y4aFBf._3bLo1c3.uVZOXwMxL_iVN.EvEHEBiNczBNMM6riJrVWgiLx1O1jGRbhIiGP.tsomZgyl_bNupNbWxZNzHy454hC0iUigDrE5jkJJoazDRJNc5532wj9nT.U9DDBxW3RplVCdj4x5sMt3K3IXADYvAGabBQnzvS3rEr_w66KClwAehy69tWHVSPDkc.ww7QnxdDItYqmtL8bz9IScdouTAvU_MWK6oxvxcLc6GQFCQZnoToeX8Fgeui2flhV.kXXjEQ4NjypxSFakcCPIysHZOUjKfv93.W9Vfl7id.Y8DUpsmxEPVOpfcrGY6YvFtk6yJhvUQryJftS5b7E5P5jVPW_pPlMWSTWL9IaysG7INm6ZjDyjsVG7OBJIUujSSMlyoKiR8sv0L2ueHt6",
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36"
},
"checkType": "status_code",
"alexaRank": 34,
"urlMain": "https://www.twitch.tv/",
@@ -17079,7 +17093,7 @@
},
"Udemy": {
"tags": [
"in"
"cloudflare"
],
"checkType": "response_url",
"alexaRank": 131,
@@ -17477,7 +17491,7 @@
"method": "vimeo"
},
"headers": {
"Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE3MzQxMTc1NDAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbCwianRpIjoiNDc4Y2ZhZGUtZjI0Yy00MDVkLTliYWItN2RlNGEzNGM4MzI5In0.guN7Fg8dqq7EYdckrJ-6Rdkj_5MOl6FaC4YUSOceDpU"
"Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE3NDI3Mjg3NDAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbCwianRpIjoiZWI1ZjY1OTAtZGNiMC00MDQ5LTkwODctZmU3ZGFjNjAyMmZmIn0.wrdGR5HPqB_o4f2EHL1h0vVldfmUOvpvPKm-y88X7KU"
},
"urlProbe": "https://api.vimeo.com/users/{username}?fields=name%2Cgender%2Cbio%2Curi%2Clink%2Cbackground_video%2Clocation_details%2Cpictures%2Cverified%2Cmetadata.public_videos.total%2Cavailable_for_hire%2Ccan_work_remotely%2Cmetadata.connections.videos.total%2Cmetadata.connections.albums.total%2Cmetadata.connections.followers.total%2Cmetadata.connections.following.total%2Cmetadata.public_videos.total%2Cmetadata.connections.vimeo_experts.is_enrolled%2Ctotal_collection_count%2Ccreated_time%2Cprofile_preferences%2Cmembership%2Cclients%2Cskills%2Cproject_types%2Crates%2Ccategories%2Cis_expert%2Cprofile_discovery%2Cwebsites%2Ccontact_emails&fetch_user_profile=1",
"checkType": "status_code",
@@ -27244,7 +27258,8 @@
},
"upwork.com": {
"tags": [
"us"
"us",
"cloudflare"
],
"engine": "engine404",
"urlMain": "https://upwork.com",
sites.md
+1 -1
View File
@@ -3141,7 +3141,7 @@ Rank data fetched from Alexa by domains.
1. ![](https://www.google.com/s2/favicons?domain=https://pubg.op.gg) [OP.GG [PUBG] (https://pubg.op.gg)](https://pubg.op.gg)*: top 100M, gaming*
1. ![](https://www.google.com/s2/favicons?domain=https://valorant.op.gg) [OP.GG [Valorant] (https://valorant.op.gg)](https://valorant.op.gg)*: top 100M, gaming*
The list was updated at (2024-12-13)
The list was updated at (2024-12-16)
## Statistics
Enabled/total sites: 2684/3137 = 85.56%