mirror/maigret
1
0
Fork 0
mirror of https://github.com/soxoj/maigret.git synced 2026-05-06 14:08:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #30 from soxoj/vimeo

Browse Source 
Vimeo activation, tags updates
This commit is contained in:
soxoj
2021-01-16 15:58:04 +03:00
committed by GitHub
parent fd236dd162 b9da54f71a
commit a5ce906243
3 changed files with 52 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
maigret/activation.py
+9
View File
@@ -10,3 +10,12 @@ class ParsingActivator:
j = r.json()
guest_token = j[site.activation['src']]
site.headers['x-guest-token'] = guest_token
@staticmethod
def vimeo(site, logger):
headers = dict(site.headers)
if 'Authorization' in headers:
del headers['Authorization']
r = requests.get(site.activation['url'], headers=headers)
jwt_token = r.json()['jwt']
site.headers['Authorization'] = 'jwt ' + jwt_token
maigret/maigret.py
+2 -2
View File
@@ -182,7 +182,7 @@ def process_site_result(response, query_notify, logger, results_info, site: Maig
error_text, site_error_text = detect_error_page(html_text, status_code, failure_errors,
site.ignore_403)
if site.activation:
if site.activation and html_text:
is_need_activation = any([s for s in site.activation['marks'] if s in html_text])
if is_need_activation:
method = site.activation['method']
@@ -196,8 +196,8 @@ def process_site_result(response, query_notify, logger, results_info, site: Maig
# presense flags
# True by default
presense_flags = site.presense_strs
is_presense_detected = False
if html_text:
is_presense_detected = False
if not presense_flags:
is_presense_detected = True
site.stats['presense_flag'] = None
maigret/resources/data.json
+41 -48
View File
@@ -309,8 +309,7 @@
"500px": {
"tags": [
"global",
"in",
"photos"
"photo"
],
"errors": {
"INTERNAL_SERVER_ERROR": "Site error",
@@ -1004,14 +1003,17 @@
},
"Aptoide": {
"tags": [
"in"
"apps"
],
"regexCheck": "^[^\\.]+$",
"checkType": "status_code",
"presenseStrs": [
"BreadcrumbList"
],
"alexaRank": 6653,
"url": "https://{username}.en.aptoide.com/",
"urlMain": "https://en.aptoide.com/",
"usernameClaimed": "blue",
"usernameClaimed": "kodi",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"Aqa": {
@@ -1552,10 +1554,8 @@
"Badoo": {
"disabled": true,
"tags": [
"br",
"de",
"pl",
"social"
"global",
"dating"
],
"checkType": "status_code",
"alexaRank": 2343,
@@ -3224,9 +3224,8 @@
"DeviantART": {
"tags": [
"global",
"images",
"photos",
"us"
"photo",
"art"
],
"regexCheck": "^[a-zA-Z][a-zA-Z0-9_-]*$",
"checkType": "status_code",
@@ -3984,12 +3983,8 @@
},
"EyeEm": {
"tags": [
"de",
"global",
"in",
"it",
"photos",
"sd"
"photo"
],
"checkType": "message",
"absenceStrs": "Not Found (404) | EyeEm",
@@ -4085,9 +4080,7 @@
},
"Facebook": {
"tags": [
"global",
"social",
"us"
"global"
],
"regexCheck": "^[a-zA-Z0-9_\\.]{3,49}(?<!\\.com|\\.org|\\.net)$",
"checkType": "status_code",
@@ -9857,8 +9850,7 @@
"tags": [
"global",
"instagram",
"photos",
"us"
"photo"
],
"checkType": "message",
"absenceStrs": [
@@ -9919,9 +9911,8 @@
},
"Pinterest": {
"tags": [
"images",
"photos",
"us"
"art",
"photo"
],
"checkType": "status_code",
"alexaRank": 152,
@@ -10872,8 +10863,7 @@
"Reddit": {
"tags": [
"news",
"social",
"us"
"discussions"
],
"checkType": "status_code",
"presenseStrs": [
@@ -13372,9 +13362,7 @@
},
"TryHackMe": {
"tags": [
"be",
"in",
"us"
"hacking"
],
"checkType": "status_code",
"alexaRank": 35003,
@@ -13386,8 +13374,7 @@
"Tumblr": {
"tags": [
"blogs",
"global",
"us"
"global"
],
"regexCheck": "^[^\\.]+$",
"checkType": "status_code",
@@ -13410,7 +13397,7 @@
},
"Twitch": {
"tags": [
"us"
"streaming"
],
"urlProbe": "https://m.twitch.tv/{username}",
"checkType": "status_code",
@@ -13422,9 +13409,7 @@
},
"Twitter": {
"tags": [
"global",
"social",
"us"
"global"
],
"headers": {
"sec-ch-ua": "Google Chrome\";v=\"87\", \" Not;A Brand\";v=\"99\", \"Chromium\";v=\"87\"",
@@ -13614,8 +13599,7 @@
"VK": {
"tags": [
"global",
"ru",
"social"
"ru"
],
"checkType": "response_url",
"alexaRank": 23,
@@ -13788,9 +13772,20 @@
},
"Vimeo": {
"tags": [
"us",
"global",
"video"
],
"headers": {
"Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MTA4MDIxODAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.x1M4yvCF7fqekRpFfgxjtevGuEZkTP6HK8FXAxlAAQk"
},
"activation": {
"url": "https://vimeo.com/_rv/viewer",
"marks": [
"Something strange occurred. Please contact the app owners."
],
"method": "vimeo"
},
"urlProbe": "https://api.vimeo.com/users/{username}?fields=name%2Cgender%2Cbio%2Curi%2Clink%2Cbackground_video%2Clocation_details%2Cpictures%2Cverified%2Cmetadata.public_videos.total%2Cavailable_for_hire%2Ccan_work_remotely%2Cmetadata.connections.videos.total%2Cmetadata.connections.albums.total%2Cmetadata.connections.followers.total%2Cmetadata.connections.following.total%2Cmetadata.public_videos.total%2Ctotal_collection_count%2Ccreated_time%2Cprofile_preferences%2Cmembership%2Cclients%2Cskills%2Cproject_types%2Crates%2Ccategories&fetch_user_profile=1",
"checkType": "status_code",
"alexaRank": 158,
"url": "https://vimeo.com/{username}",
@@ -14063,7 +14058,7 @@
"Wattpad": {
"tags": [
"in",
"social"
"reading"
],
"checkType": "message",
"absenceStrs": "userError-404",
@@ -14109,9 +14104,9 @@
},
"We Heart It": {
"tags": [
"in",
"photos",
"us"
"photo",
"global",
"blogs"
],
"checkType": "message",
"absenceStrs": "Oops! You've landed on a moving target!",
@@ -15471,8 +15466,8 @@
},
"devRant": {
"tags": [
"in",
"social"
"global",
"coding"
],
"checkType": "response_url",
"alexaRank": 133141,
@@ -16165,8 +16160,7 @@
},
"geocaching": {
"tags": [
"de",
"social"
"hobby"
],
"checkType": "status_code",
"alexaRank": 17327,
@@ -17718,8 +17712,7 @@
},
"social.tchncs.de": {
"tags": [
"in",
"vn"
"de"
],
"checkType": "status_code",
"alexaRank": 441382,